topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(auth) Make row permissions take effect during the translate sql phase and refactor the auth code (#1368)

Browse Source 
Co-authored-by: lxwcodemonkey
This commit is contained in:
LXW
2024-07-07 20:46:53 +08:00
committed by GitHub
parent 08ae27ab43
commit 9911e6772c
10 changed files with 163 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthResGrp.java
View File

@@ -1,11 +0,0 @@
package com.tencent.supersonic.auth.api.authorization.pojo;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class AuthResGrp {
private List<AuthRes> group = new ArrayList<>();
}

3
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.auth.api.authorization.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import lombok.Data;
import lombok.ToString;
import org.springframework.util.CollectionUtils;
@@ -15,8 +14,6 @@ public class QueryAuthResReq {
private List<String> departmentIds = new ArrayList<>();
private List<AuthRes> resources;
private Long modelId;
private List<Long> modelIds;

8
auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/response/AuthorizedResourceResp.java
View File

@@ -1,16 +1,16 @@
package com.tencent.supersonic.auth.api.authorization.response;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
@Data
public class AuthorizedResourceResp {
private List<AuthResGrp> resources = new ArrayList<>();
private List<AuthRes> authResList = new ArrayList<>();
private List<DimensionFilter> filters = new ArrayList<>();
}

61
auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java
View File

@@ -1,19 +1,16 @@
package com.tencent.supersonic.auth.authorization.service;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.service.UserService;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -79,53 +76,35 @@ public class AuthServiceImpl implements AuthService {
@Override
public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) {
if (CollectionUtils.isEmpty(req.getModelIds())) {
return new AuthorizedResourceResp();
}
Set<String> userOrgIds = userService.getUserAllOrgId(user.getName());
List<AuthGroup> groups = getAuthGroups(req.getModelIds(), user.getName(), new ArrayList<>(userOrgIds));
AuthorizedResourceResp resource = new AuthorizedResourceResp();
Map<Long, List<AuthGroup>> authGroupsByModelId = groups.stream()
.collect(Collectors.groupingBy(AuthGroup::getModelId));
Map<Long, List<AuthRes>> reqAuthRes = req.getResources().stream()
.collect(Collectors.groupingBy(AuthRes::getModelId));
for (Long modelId : reqAuthRes.keySet()) {
List<AuthRes> reqResourcesList = reqAuthRes.get(modelId);
AuthResGrp rg = new AuthResGrp();
for (Long modelId : req.getModelIds()) {
if (authGroupsByModelId.containsKey(modelId)) {
List<AuthGroup> authGroups = authGroupsByModelId.get(modelId);
for (AuthRes reqRes : reqResourcesList) {
for (AuthGroup authRuleGroup : authGroups) {
List<AuthRule> authRules = authRuleGroup.getAuthRules();
List<String> allAuthItems = new ArrayList<>();
authRules.forEach(authRule -> allAuthItems.addAll(authRule.resourceNames()));
if (allAuthItems.contains(reqRes.getName())) {
rg.getGroup().add(reqRes);
for (AuthGroup authRuleGroup : authGroups) {
List<AuthRule> authRules = authRuleGroup.getAuthRules();
for (AuthRule authRule : authRules) {
for (String resBizName : authRule.resourceNames()) {
resource.getAuthResList().add(new AuthRes(modelId, resBizName));
}
}
}
}
if (!CollectionUtils.isEmpty(rg.getGroup())) {
resource.getResources().add(rg);
}
}
if (!CollectionUtils.isEmpty(req.getModelIds())) {
List<AuthGroup> authGroups = Lists.newArrayList();
for (Long modelId : authGroupsByModelId.keySet()) {
authGroups.addAll(authGroupsByModelId.getOrDefault(modelId, Lists.newArrayList()));
}
if (!CollectionUtils.isEmpty(authGroups)) {
for (AuthGroup group : authGroups) {
if (group.getDimensionFilters() != null
&& group.getDimensionFilters().stream().anyMatch(expr ->
!StringUtils.isEmpty(expr))) {
DimensionFilter df = new DimensionFilter();
df.setDescription(group.getDimensionFilterDescription());
df.setExpressions(group.getDimensionFilters());
resource.getFilters().add(df);
}
}
Set<Map.Entry<Long, List<AuthGroup>>> entries = authGroupsByModelId.entrySet();
for (Map.Entry<Long, List<AuthGroup>> entry : entries) {
List<AuthGroup> authGroups = entry.getValue();
for (AuthGroup authGroup : authGroups) {
DimensionFilter df = new DimensionFilter();
df.setDescription(authGroup.getDimensionFilterDescription());
df.setExpressions(authGroup.getDimensionFilters());
resource.getFilters().add(df);
}
}
return resource;
@@ -134,11 +113,11 @@ public class AuthServiceImpl implements AuthService {
private List<AuthGroup> getAuthGroups(List<Long> modelIds, String userName, List<String> departmentIds) {
List<AuthGroup> groups = load().stream()
.filter(group -> {
if (CollectionUtils.isEmpty(modelIds) || !modelIds.contains(group.getModelId())) {
if (!modelIds.contains(group.getModelId())) {
return false;
}
if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers()
.contains(userName)) {
if (!CollectionUtils.isEmpty(group.getAuthorizedUsers())
&& group.getAuthorizedUsers().contains(userName)) {
return true;
}
for (String departmentId : departmentIds) {

18
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java
View File

@@ -9,7 +9,9 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
@@ -58,4 +60,20 @@ public class SemanticSchemaResp {
.findFirst().orElse(null);
}
public Set<String> getNameFromBizNames(Set<String> bizNames) {
Set<String> names = new HashSet<>();
for (String bizName : bizNames) {
DimSchemaResp dimSchemaResp = getDimension(bizName);
if (dimSchemaResp != null) {
names.add(dimSchemaResp.getName());
continue;
}
MetricSchemaResp metricSchemaResp = getMetric(bizName);
if (metricSchemaResp != null) {
names.add(metricSchemaResp.getName());
}
}
return names;
}
}

443
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -1,42 +1,32 @@
package com.tencent.supersonic.headless.server.aspect;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.request.TranslateSqlReq;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.web.service.DimensionService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.web.service.ModelService;
import com.tencent.supersonic.headless.server.web.service.SchemaService;
import com.tencent.supersonic.headless.server.web.service.DataSetService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -46,45 +36,29 @@ import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@Order(1)
@Slf4j
public class S2DataPermissionAspect {
private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat(
new SimpleDateFormat(Constants.DAY_FORMAT));
@Autowired
private QueryStructUtils queryStructUtils;
@Autowired
private DimensionService dimensionService;
@Autowired
private ModelService modelService;
@Autowired
private SchemaService schemaService;
@Autowired
private DataSetService dataSetService;
@Autowired
private AuthService authService;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)")
@@ -93,8 +67,19 @@ public class S2DataPermissionAspect {
@Around("s2PermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
//1. check args
Object[] objects = joinPoint.getArgs();
SemanticQueryReq queryReq = (SemanticQueryReq) objects[0];
boolean needQueryData = true;
SemanticQueryReq queryReq = null;
if (objects[0] instanceof SemanticQueryReq) {
queryReq = (SemanticQueryReq) objects[0];
} else if (objects[0] instanceof TranslateSqlReq) {
queryReq = (SemanticQueryReq) ((TranslateSqlReq<?>) objects[0]).getQueryReq();
needQueryData = false;
}
if (queryReq == null) {
throw new InvalidArgumentException("queryReq is not Invalid");
}
if (!queryReq.isNeedAuth()) {
log.info("needAuth is false, there is no need to check permissions.");
return joinPoint.proceed();
@@ -103,179 +88,86 @@ public class S2DataPermissionAspect {
if (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
List<Long> modelIds = getModelsInDataSet(queryReq);
// determine whether admin of the model
if (doModelAdmin(user, modelIds)) {
SemanticSchemaResp semanticSchemaResp = getSemanticSchemaResp(queryReq);
List<Long> modelIds = getModelIds(semanticSchemaResp);
//2. determine whether admin of the model
if (checkModelAdmin(user, modelIds)) {
return joinPoint.proceed();
}
// determine whether the subject field is visible
doModelVisible(user, modelIds);
//3. determine whether the model is visible to cur user
checkModelVisible(user, modelIds);
//4. get permissions auth to cur user
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds);
//5. check col permission
if (needQueryData) {
checkColPermission(queryReq, authorizedResource, modelIds, semanticSchemaResp);
}
//6. check row permission
checkRowPermission(queryReq, authorizedResource);
//7. add hint to user
Object result = joinPoint.proceed();
if (result instanceof SemanticQueryResp) {
addHint(modelIds, (SemanticQueryResp) result, authorizedResource);
}
return result;
}
private void checkColPermission(SemanticQueryReq semanticQueryReq, AuthorizedResourceResp authorizedResource,
List<Long> modelIds, SemanticSchemaResp semanticSchemaResp) {
// get high sensitive fields in query
Set<String> bizNamesInQueryReq = getBizNameInQueryReq(semanticQueryReq, semanticSchemaResp);
Set<String> sensitiveBizNamesByModel = getHighSensitiveBizNamesByModelId(semanticSchemaResp);
Set<String> sensitiveBizNameInQuery = bizNamesInQueryReq.parallelStream()
.filter(sensitiveBizNamesByModel::contains).collect(Collectors.toSet());
//get high sensitive field cur user has been authed
Set<String> sensitiveBizNameUserAuthed = authorizedResource.getAuthResList()
.stream().map(AuthRes::getName).collect(Collectors.toSet());
sensitiveBizNameInQuery.removeAll(sensitiveBizNameUserAuthed);
if (!CollectionUtils.isEmpty(sensitiveBizNameInQuery)) {
Set<String> sensitiveResNames = semanticSchemaResp.getNameFromBizNames(sensitiveBizNameInQuery);
List<String> modelAdmin = modelService.getModelAdmin(modelIds.get(0));
String message = String.format("存在以下敏感资源:%s您暂无权限，请联系管理员%s申请",
sensitiveResNames, modelAdmin);
throw new InvalidPermissionException(message);
}
}
private void checkRowPermission(SemanticQueryReq queryReq,
AuthorizedResourceResp authorizedResource) {
if (queryReq instanceof QuerySqlReq) {
return checkSqlPermission(joinPoint, (QuerySqlReq) queryReq);
doRowPermission((QuerySqlReq) queryReq, authorizedResource);
}
if (queryReq instanceof QueryStructReq) {
return checkStructPermission(joinPoint, (QueryStructReq) queryReq);
doRowPermission((QueryStructReq) queryReq, authorizedResource);
}
}
private Set<String> getBizNameInQueryReq(SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) {
if (queryReq instanceof QuerySqlReq) {
return queryStructUtils.getBizNameFromSql((QuerySqlReq) queryReq, semanticSchemaResp);
}
if (queryReq instanceof QueryStructReq) {
return queryStructUtils.getBizNameFromStruct((QueryStructReq) queryReq);
}
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
}
private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySqlReq)
throws Throwable {
Object[] objects = joinPoint.getArgs();
User user = (User) objects[1];
// fetch data permission meta information
private SemanticSchemaResp getSemanticSchemaResp(SemanticQueryReq semanticQueryReq) {
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(querySqlReq.getModelIds());
filter.setDataSetId(querySqlReq.getDataSetId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
filter.setModelIds(semanticQueryReq.getModelIds());
filter.setDataSetId(semanticQueryReq.getDataSetId());
return schemaService.fetchSemanticSchema(filter);
}
private List<Long> getModelIds(SemanticSchemaResp semanticSchemaResp) {
return semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp);
log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInDataSet, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq);
// row permission pre-filter
doRowPermission(querySqlReq, authorizedResource);
// proceed
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
}
// if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
private void doFilterCheckLogic(QuerySqlReq querySqlReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySqlReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = Lists.newArrayList(querySqlReq.getModelIds());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
String modelNameCn = Constants.EMPTY;
if (!CollectionUtils.isEmpty(modelInfos)) {
modelNameCn = modelInfos.get(0).getName();
}
MetaFilter metaFilter = new MetaFilter(modelIds);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(metaFilter);
String finalDomainNameCn = modelNameCn;
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
ModelResp modelResp = modelInfos.get(0);
List<String> admins = modelService.getModelAdmin(modelResp.getId());
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
}
}
private void doFilterCheckLogic(Set<String> resAuthName, Set<String> sensitiveResReq,
List<Long> modelIdInDataSet, QueryStructReq queryStructReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
ModelFilter modelFilter = new ModelFilter(false, modelIdInDataSet);
Map<Long, ModelResp> modelRespMap = modelService.getModelMap(modelFilter);
List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIdInDataSet));
dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
.forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
if (!CollectionUtils.isEmpty(need2Apply)) {
List<String> admins = modelService.getModelAdmin(modelIdInDataSet.get(0));
log.info("in doFilterLogic, need2Apply:{}", need2Apply);
String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
throw new InvalidPermissionException(message);
}
}
public Object checkStructPermission(ProceedingJoinPoint point, QueryStructReq queryStructReq) throws Throwable {
Object[] args = point.getArgs();
User user = (User) args[1];
// fetch data permission meta information
SchemaFilterReq filter = new SchemaFilterReq();
filter.setModelIds(queryStructReq.getModelIds());
filter.setDataSetId(queryStructReq.getDataSetId());
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
.map(ModelResp::getId).collect(Collectors.toList());
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
Set<String> sensitiveResReq = res4Privilege.parallelStream()
.filter(sensitiveResByModel::contains).collect(Collectors.toSet());
log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInDataSet, sensitiveResReq);
// query user privilege info
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
modelIdInDataSet, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(resAuthSet, sensitiveResReq, modelIdInDataSet, queryStructReq);
// row permission pre-filter
doRowPermission(queryStructReq, authorizedResource);
// proceed
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
log.info("sensitiveResReq is empty");
return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
}
// if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) {
return true;
}
log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
return false;
}
private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
@@ -343,7 +235,7 @@ public class S2DataPermissionAspect {
}
public boolean doModelAdmin(User user, List<Long> modelIds) {
public boolean checkModelAdmin(User user, List<Long> modelIds) {
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
@@ -353,13 +245,14 @@ public class S2DataPermissionAspect {
}
}
public void doModelVisible(User user, List<Long> modelIds) {
public void checkModelVisible(User user, List<Long> modelIds) {
List<Long> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE)
.stream().map(ModelResp::getId).collect(Collectors.toList());
modelIds.removeAll(modelListVisible);
if (!CollectionUtils.isEmpty(modelIds)) {
List<Long> modelIdCopied = new ArrayList<>(modelIds);
modelIdCopied.removeAll(modelListVisible);
if (!CollectionUtils.isEmpty(modelIdCopied)) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(modelIds);
metaFilter.setIds(modelIdCopied);
List<ModelResp> modelResps = modelService.getModelList(metaFilter);
ModelResp modelResp = modelResps.stream().findFirst().orElse(null);
if (modelResp == null) {
@@ -370,7 +263,7 @@ public class S2DataPermissionAspect {
}
}
public Set<String> getHighSensitiveColsByModelId(SemanticSchemaResp semanticSchemaResp) {
public Set<String> getHighSensitiveBizNamesByModelId(SemanticSchemaResp semanticSchemaResp) {
Set<String> highSensitiveCols = new HashSet<>();
if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp ->
@@ -385,16 +278,12 @@ public class S2DataPermissionAspect {
return highSensitiveCols;
}
public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res)));
public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds) {
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelIds(modelIds);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds,
authorizedResource);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}",
user.getName(), modelIds, authorizedResource);
return authorizedResource;
}
@@ -403,142 +292,15 @@ public class S2DataPermissionAspect {
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, List<Long> modelIds) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> modelIds.contains(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName()));
}
});
log.info("resAuthName:{}", resAuthName);
return resAuthName;
}
public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns,
List<Long> modelIds,
AuthorizedResourceResp authResource) {
addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set<String> need2Apply) {
log.debug("start desensitizationData logic");
if (CollectionUtils.isEmpty(need2Apply)) {
log.info("user has all sensitiveRes");
return raw;
}
List<QueryColumn> columns = raw.getColumns();
boolean doDesensitization = false;
for (QueryColumn queryColumn : columns) {
for (String sensitiveCol : need2Apply) {
if (queryColumn.getNameEn().contains(sensitiveCol)) {
doDesensitization = true;
break;
}
}
}
if (!doDesensitization) {
return raw;
}
SemanticQueryResp queryResultWithColumns = raw;
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
log.warn("deepCopyResult: ", e);
}
addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
return queryResultWithColumns;
}
private void addAuthorizedSchemaInfo(List<QueryColumn> columns, Set<String> need2Apply) {
if (CollectionUtils.isEmpty(need2Apply)) {
return;
}
columns.stream().forEach(col -> {
if (need2Apply.contains(getName(col.getNameEn()))) {
col.setAuthorized(false);
}
});
}
private String getName(String nameEn) {
Pattern pattern = Pattern.compile("\\((.*?)\\)");
Matcher matcher = pattern.matcher(nameEn);
if (matcher.find()) {
return matcher.group(1).replaceAll("`", "");
}
return nameEn;
}
private void desensitizationInternal(List<Map<String, Object>> result, Set<String> need2Apply) {
log.info("start desensitizationInternal logic");
for (int i = 0; i < result.size(); i++) {
Map<String, Object> row = result.get(i);
Map<String, Object> newRow = new HashMap<>();
for (String col : row.keySet()) {
boolean sensitive = false;
for (String sensitiveCol : need2Apply) {
if (col.contains(sensitiveCol)) {
sensitive = true;
break;
}
}
if (sensitive) {
newRow.put(col, "******");
} else {
newRow.put(col, row.get(col));
}
}
result.set(i, newRow);
}
}
private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
BeanUtils.copyProperties(raw, queryResultWithColumns);
List<QueryColumn> columns = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getColumns())) {
String columnsStr = MAPPER.writeValueAsString(raw.getColumns());
columns = MAPPER.readValue(columnsStr, new TypeReference<List<QueryColumn>>() {
});
queryResultWithColumns.setColumns(columns);
}
queryResultWithColumns.setColumns(columns);
List<Map<String, Object>> resultData = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getResultList())) {
for (Map<String, Object> line : raw.getResultList()) {
Map<String, Object> newLine = new HashMap<>();
newLine.putAll(line);
resultData.add(newLine);
}
}
queryResultWithColumns.setResultList(resultData);
return queryResultWithColumns;
}
public void addPromptInfoInfo(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
public void addHint(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
if (CollectionUtils.isEmpty(filters)) {
return;
}
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
if (!CollectionUtils.isEmpty(need2Apply)) {
String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
}
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
ModelResp modelResp = modelService.getModel(modelIds.get(0));
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
@@ -550,20 +312,9 @@ public class S2DataPermissionAspect {
});
String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
log.info("queryResultWithColumns:{}", queryResultWithColumns);
}
}
private List<Long> getModelsInDataSet(SemanticQueryReq queryReq) {
List<Long> modelIds = queryReq.getModelIds();
if (queryReq.getDataSetId() != null) {
DataSetResp dataSetResp = dataSetService.getDataSet(queryReq.getDataSetId());
modelIds = dataSetResp.getAllModels();
}
return modelIds;
}
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -227,6 +227,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryByReq(querySqlReq, user);
}
@S2DataPermission
@Override
public <T> TranslateResp translate(TranslateSqlReq<T> translateSqlReq, User user) throws Exception {
T queryReq = translateSqlReq.getQueryReq();

48
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -114,28 +114,21 @@ public class QueryStructUtils {
return metricIds;
}
public Set<String> getResNameEn(QueryStructReq queryStructCmd) {
public Set<String> getBizNameFromStruct(QueryStructReq queryStructReq) {
Set<String> resNameEnSet = new HashSet<>();
queryStructCmd.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn()));
resNameEnSet.addAll(queryStructCmd.getGroups());
queryStructCmd.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn()));
sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
queryStructReq.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn()));
resNameEnSet.addAll(queryStructReq.getGroups());
queryStructReq.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn()));
sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
return resNameEnSet;
}
public Set<String> getResName(QuerySqlReq querySqlReq) {
Set<String> resNameSet = SqlSelectHelper.getAllFields(querySqlReq.getSql())
.stream().collect(Collectors.toSet());
return resNameSet;
return new HashSet<>(SqlSelectHelper.getAllFields(querySqlReq.getSql()));
}
public Set<String> getResNameEnExceptInternalCol(QueryStructReq queryStructCmd) {
Set<String> resNameEnSet = getResNameEn(queryStructCmd);
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySqlReq,
SemanticSchemaResp semanticSchemaResp) {
public Set<String> getBizNameFromSql(QuerySqlReq querySqlReq,
SemanticSchemaResp semanticSchemaResp) {
Set<String> resNameSet = getResName(querySqlReq);
Set<String> resNameEnSet = new HashSet<>();
if (semanticSchemaResp != null) {
@@ -155,23 +148,6 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getFilterResNameEn(QueryStructReq queryStructCmd) {
Set<String> resNameEnSet = new HashSet<>();
sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
return resNameEnSet;
}
public Set<String> getFilterResNameEnExceptInternalCol(QueryStructReq queryStructCmd) {
Set<String> resNameEnSet = getFilterResNameEn(queryStructCmd);
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) {
String sql = querySqlReq.getSql();
Set<String> resNameEnSet = SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) {
List<Long> dimensionIds = getDimensionIds(queryStructCmd);
List<Long> metricIds = getMetricIds(queryStructCmd);
@@ -281,13 +257,5 @@ public class QueryStructUtils {
return null;
}
public List<String> getDateCol() {
return dateModeUtils.getDateCol();
}
public String getVariablePrefix() {
return variablePrefix;
}
}

15
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java
View File

@@ -101,18 +101,23 @@ public class QueryBySqlTest extends BaseTest {
@Test
public void testAuthorization_sensitive_metric() throws Exception {
User tom = DataUtils.getUserTom();
assertThrows(InvalidPermissionException.class,
() -> queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom));
}
@Test
public void testAuthorization_sensitive_metric_jack() throws Exception {
User jack = DataUtils.getUserJack();
SemanticQueryResp semanticQueryResp =
queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom);
Assertions.assertEquals(false, semanticQueryResp.getColumns().get(0).getAuthorized());
Assertions.assertEquals("******",
semanticQueryResp.getResultList().get(0).get("SUM(stay_hours)"));
queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计", jack);
Assertions.assertTrue(semanticQueryResp.getResultList().size() > 0);
}
@Test
public void testAuthorization_row_permission() throws Exception {
User tom = DataUtils.getUserTom();
SemanticQueryResp semanticQueryResp =
queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom);
queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计 WHERE department ='HR'", tom);
Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());
Assertions.assertTrue(semanticQueryResp.getSql().contains("user_name = 'tom'"));
}

11
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java
View File

@@ -111,15 +111,14 @@ public class QueryByStructTest extends BaseTest {
}
@Test
public void testAuthorization_sensitive_metric() throws Exception {
public void testAuthorization_sensitive_metric() {
User tom = DataUtils.getUserTom();
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("stay_hours");
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom);
Assertions.assertEquals(false, semanticQueryResp.getColumns().get(1).getAuthorized());
Assertions.assertEquals("******", semanticQueryResp.getResultList().get(0).get("stay_hours"));
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"), aggregator);
assertThrows(InvalidPermissionException.class,
() -> semanticLayerService.queryByReq(queryStructReq, tom));
}
@Test
@@ -127,7 +126,7 @@ public class QueryByStructTest extends BaseTest {
User tom = DataUtils.getUserTom();
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("stay_hours");
aggregator.setColumn("pv");
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom);
Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());