From 9911e6772c4a8826f7e4ab3c877d9a06d3eff87b Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Sun, 7 Jul 2024 20:46:53 +0800 Subject: [PATCH] (improvement)(auth) Make row permissions take effect during the translate sql phase and refactor the auth code (#1368) Co-authored-by: lxwcodemonkey --- .../api/authorization/pojo/AuthResGrp.java | 11 - .../request/QueryAuthResReq.java | 3 - .../response/AuthorizedResourceResp.java | 8 +- .../service/AuthServiceImpl.java | 61 +-- .../api/pojo/response/SemanticSchemaResp.java | 18 + .../server/aspect/S2DataPermissionAspect.java | 443 ++++-------------- .../service/impl/S2SemanticLayerService.java | 1 + .../server/utils/QueryStructUtils.java | 48 +- .../supersonic/headless/QueryBySqlTest.java | 15 +- .../headless/QueryByStructTest.java | 11 +- 10 files changed, 163 insertions(+), 456 deletions(-) delete mode 100644 auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthResGrp.java diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthResGrp.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthResGrp.java deleted file mode 100644 index f99cb0258..000000000 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthResGrp.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tencent.supersonic.auth.api.authorization.pojo; - -import java.util.ArrayList; -import java.util.List; -import lombok.Data; - -@Data -public class AuthResGrp { - - private List group = new ArrayList<>(); -} diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java index 250b63689..336ed65e5 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.auth.api.authorization.request; import com.google.common.collect.Lists; -import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; import lombok.Data; import lombok.ToString; import org.springframework.util.CollectionUtils; @@ -15,8 +14,6 @@ public class QueryAuthResReq { private List departmentIds = new ArrayList<>(); - private List resources; - private Long modelId; private List modelIds; diff --git a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/response/AuthorizedResourceResp.java b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/response/AuthorizedResourceResp.java index 059d3ce39..7d682e109 100644 --- a/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/response/AuthorizedResourceResp.java +++ b/auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/response/AuthorizedResourceResp.java @@ -1,16 +1,16 @@ package com.tencent.supersonic.auth.api.authorization.response; -import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp; +import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter; +import lombok.Data; + import java.util.ArrayList; import java.util.List; -import lombok.Data; - @Data public class AuthorizedResourceResp { - private List resources = new ArrayList<>(); + private List authResList = new ArrayList<>(); private List filters = new ArrayList<>(); } diff --git a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java index 008d6e970..6e022c105 100644 --- a/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java +++ b/auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java @@ -1,19 +1,16 @@ package com.tencent.supersonic.auth.authorization.service; -import com.google.common.collect.Lists; import com.google.gson.Gson; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.service.UserService; import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; -import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule; import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter; import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq; import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp; import com.tencent.supersonic.auth.api.authorization.service.AuthService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -79,53 +76,35 @@ public class AuthServiceImpl implements AuthService { @Override public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) { + if (CollectionUtils.isEmpty(req.getModelIds())) { + return new AuthorizedResourceResp(); + } Set userOrgIds = userService.getUserAllOrgId(user.getName()); List groups = getAuthGroups(req.getModelIds(), user.getName(), new ArrayList<>(userOrgIds)); AuthorizedResourceResp resource = new AuthorizedResourceResp(); Map> authGroupsByModelId = groups.stream() .collect(Collectors.groupingBy(AuthGroup::getModelId)); - Map> reqAuthRes = req.getResources().stream() - .collect(Collectors.groupingBy(AuthRes::getModelId)); - - for (Long modelId : reqAuthRes.keySet()) { - List reqResourcesList = reqAuthRes.get(modelId); - AuthResGrp rg = new AuthResGrp(); + for (Long modelId : req.getModelIds()) { if (authGroupsByModelId.containsKey(modelId)) { List authGroups = authGroupsByModelId.get(modelId); - for (AuthRes reqRes : reqResourcesList) { - for (AuthGroup authRuleGroup : authGroups) { - List authRules = authRuleGroup.getAuthRules(); - List allAuthItems = new ArrayList<>(); - authRules.forEach(authRule -> allAuthItems.addAll(authRule.resourceNames())); - - if (allAuthItems.contains(reqRes.getName())) { - rg.getGroup().add(reqRes); + for (AuthGroup authRuleGroup : authGroups) { + List authRules = authRuleGroup.getAuthRules(); + for (AuthRule authRule : authRules) { + for (String resBizName : authRule.resourceNames()) { + resource.getAuthResList().add(new AuthRes(modelId, resBizName)); } - } } } - if (!CollectionUtils.isEmpty(rg.getGroup())) { - resource.getResources().add(rg); - } } - - if (!CollectionUtils.isEmpty(req.getModelIds())) { - List authGroups = Lists.newArrayList(); - for (Long modelId : authGroupsByModelId.keySet()) { - authGroups.addAll(authGroupsByModelId.getOrDefault(modelId, Lists.newArrayList())); - } - if (!CollectionUtils.isEmpty(authGroups)) { - for (AuthGroup group : authGroups) { - if (group.getDimensionFilters() != null - && group.getDimensionFilters().stream().anyMatch(expr -> - !StringUtils.isEmpty(expr))) { - DimensionFilter df = new DimensionFilter(); - df.setDescription(group.getDimensionFilterDescription()); - df.setExpressions(group.getDimensionFilters()); - resource.getFilters().add(df); - } - } + Set>> entries = authGroupsByModelId.entrySet(); + for (Map.Entry> entry : entries) { + List authGroups = entry.getValue(); + for (AuthGroup authGroup : authGroups) { + DimensionFilter df = new DimensionFilter(); + df.setDescription(authGroup.getDimensionFilterDescription()); + df.setExpressions(authGroup.getDimensionFilters()); + resource.getFilters().add(df); } } return resource; @@ -134,11 +113,11 @@ public class AuthServiceImpl implements AuthService { private List getAuthGroups(List modelIds, String userName, List departmentIds) { List groups = load().stream() .filter(group -> { - if (CollectionUtils.isEmpty(modelIds) || !modelIds.contains(group.getModelId())) { + if (!modelIds.contains(group.getModelId())) { return false; } - if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers() - .contains(userName)) { + if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) + && group.getAuthorizedUsers().contains(userName)) { return true; } for (String departmentId : departmentIds) { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index 922ba84eb..6477e08f1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -9,7 +9,9 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.apache.commons.lang3.StringUtils; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; @@ -58,4 +60,20 @@ public class SemanticSchemaResp { .findFirst().orElse(null); } + public Set getNameFromBizNames(Set bizNames) { + Set names = new HashSet<>(); + for (String bizName : bizNames) { + DimSchemaResp dimSchemaResp = getDimension(bizName); + if (dimSchemaResp != null) { + names.add(dimSchemaResp.getName()); + continue; + } + MetricSchemaResp metricSchemaResp = getMetric(bizName); + if (metricSchemaResp != null) { + names.add(metricSchemaResp.getName()); + } + } + return names; + } + } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index d748bcb1a..b65b8850b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -1,42 +1,32 @@ package com.tencent.supersonic.headless.server.aspect; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes; -import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp; import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter; import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq; import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp; import com.tencent.supersonic.auth.api.authorization.service.AuthService; -import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.QueryAuthorization; -import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; -import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.api.pojo.request.TranslateSqlReq; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; -import com.tencent.supersonic.headless.server.pojo.ModelFilter; -import com.tencent.supersonic.headless.server.web.service.DimensionService; +import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.web.service.ModelService; import com.tencent.supersonic.headless.server.web.service.SchemaService; -import com.tencent.supersonic.headless.server.web.service.DataSetService; -import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -46,45 +36,29 @@ import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Pointcut; -import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.StringJoiner; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import static com.tencent.supersonic.common.pojo.Constants.MINUS; @Component @Aspect -@Order(1) @Slf4j public class S2DataPermissionAspect { - private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat( - new SimpleDateFormat(Constants.DAY_FORMAT)); - @Autowired private QueryStructUtils queryStructUtils; @Autowired - private DimensionService dimensionService; - @Autowired private ModelService modelService; @Autowired private SchemaService schemaService; @Autowired - private DataSetService dataSetService; - @Autowired private AuthService authService; @Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)") @@ -93,8 +67,19 @@ public class S2DataPermissionAspect { @Around("s2PermissionCheck()") public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { + //1. check args Object[] objects = joinPoint.getArgs(); - SemanticQueryReq queryReq = (SemanticQueryReq) objects[0]; + boolean needQueryData = true; + SemanticQueryReq queryReq = null; + if (objects[0] instanceof SemanticQueryReq) { + queryReq = (SemanticQueryReq) objects[0]; + } else if (objects[0] instanceof TranslateSqlReq) { + queryReq = (SemanticQueryReq) ((TranslateSqlReq) objects[0]).getQueryReq(); + needQueryData = false; + } + if (queryReq == null) { + throw new InvalidArgumentException("queryReq is not Invalid"); + } if (!queryReq.isNeedAuth()) { log.info("needAuth is false, there is no need to check permissions."); return joinPoint.proceed(); @@ -103,179 +88,86 @@ public class S2DataPermissionAspect { if (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) { throw new RuntimeException("please provide user information"); } - List modelIds = getModelsInDataSet(queryReq); - // determine whether admin of the model - if (doModelAdmin(user, modelIds)) { + SemanticSchemaResp semanticSchemaResp = getSemanticSchemaResp(queryReq); + List modelIds = getModelIds(semanticSchemaResp); + + //2. determine whether admin of the model + if (checkModelAdmin(user, modelIds)) { return joinPoint.proceed(); } - // determine whether the subject field is visible - doModelVisible(user, modelIds); + //3. determine whether the model is visible to cur user + checkModelVisible(user, modelIds); + //4. get permissions auth to cur user + AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds); + + //5. check col permission + if (needQueryData) { + checkColPermission(queryReq, authorizedResource, modelIds, semanticSchemaResp); + } + //6. check row permission + checkRowPermission(queryReq, authorizedResource); + + //7. add hint to user + Object result = joinPoint.proceed(); + if (result instanceof SemanticQueryResp) { + addHint(modelIds, (SemanticQueryResp) result, authorizedResource); + } + return result; + } + + private void checkColPermission(SemanticQueryReq semanticQueryReq, AuthorizedResourceResp authorizedResource, + List modelIds, SemanticSchemaResp semanticSchemaResp) { + // get high sensitive fields in query + Set bizNamesInQueryReq = getBizNameInQueryReq(semanticQueryReq, semanticSchemaResp); + Set sensitiveBizNamesByModel = getHighSensitiveBizNamesByModelId(semanticSchemaResp); + Set sensitiveBizNameInQuery = bizNamesInQueryReq.parallelStream() + .filter(sensitiveBizNamesByModel::contains).collect(Collectors.toSet()); + + //get high sensitive field cur user has been authed + Set sensitiveBizNameUserAuthed = authorizedResource.getAuthResList() + .stream().map(AuthRes::getName).collect(Collectors.toSet()); + sensitiveBizNameInQuery.removeAll(sensitiveBizNameUserAuthed); + if (!CollectionUtils.isEmpty(sensitiveBizNameInQuery)) { + Set sensitiveResNames = semanticSchemaResp.getNameFromBizNames(sensitiveBizNameInQuery); + List modelAdmin = modelService.getModelAdmin(modelIds.get(0)); + String message = String.format("存在以下敏感资源:%s您暂无权限，请联系管理员%s申请", + sensitiveResNames, modelAdmin); + throw new InvalidPermissionException(message); + } + } + + private void checkRowPermission(SemanticQueryReq queryReq, + AuthorizedResourceResp authorizedResource) { if (queryReq instanceof QuerySqlReq) { - return checkSqlPermission(joinPoint, (QuerySqlReq) queryReq); + doRowPermission((QuerySqlReq) queryReq, authorizedResource); } if (queryReq instanceof QueryStructReq) { - return checkStructPermission(joinPoint, (QueryStructReq) queryReq); + doRowPermission((QueryStructReq) queryReq, authorizedResource); + } + } + + private Set getBizNameInQueryReq(SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) { + if (queryReq instanceof QuerySqlReq) { + return queryStructUtils.getBizNameFromSql((QuerySqlReq) queryReq, semanticSchemaResp); + } + if (queryReq instanceof QueryStructReq) { + return queryStructUtils.getBizNameFromStruct((QueryStructReq) queryReq); } throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); } - private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySqlReq) - throws Throwable { - Object[] objects = joinPoint.getArgs(); - User user = (User) objects[1]; - // fetch data permission meta information + private SemanticSchemaResp getSemanticSchemaResp(SemanticQueryReq semanticQueryReq) { SchemaFilterReq filter = new SchemaFilterReq(); - filter.setModelIds(querySqlReq.getModelIds()); - filter.setDataSetId(querySqlReq.getDataSetId()); - SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); - List modelIdInDataSet = semanticSchemaResp.getModelResps().stream() + filter.setModelIds(semanticQueryReq.getModelIds()); + filter.setDataSetId(semanticQueryReq.getDataSetId()); + return schemaService.fetchSemanticSchema(filter); + } + + private List getModelIds(SemanticSchemaResp semanticSchemaResp) { + return semanticSchemaResp.getModelResps().stream() .map(ModelResp::getId).collect(Collectors.toList()); - Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp); - log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege); - - Set sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp); - Set sensitiveResReq = res4Privilege.parallelStream() - .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); - - // query user privilege info - AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInDataSet, sensitiveResReq); - // get sensitiveRes that user has privilege - Set resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet); - - // if sensitive fields without permission are involved in filter, thrown an exception - doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq); - - // row permission pre-filter - doRowPermission(querySqlReq, authorizedResource); - - // proceed - SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); - - if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { - // if sensitiveRes is empty - log.info("sensitiveResReq is empty"); - return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource); - } - - // if the column has no permission, hit * - Set need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req)) - .collect(Collectors.toSet()); - log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet); - SemanticQueryResp queryResultAfterDesensitization = - desensitizationData(queryResultWithColumns, need2Apply); - addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply); - - return queryResultAfterDesensitization; - } - - private void doFilterCheckLogic(QuerySqlReq querySqlReq, Set resAuthName, - Set sensitiveResReq) { - Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySqlReq); - Set need2Apply = resFilterSet.stream() - .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); - Set nameCnSet = new HashSet<>(); - - List modelIds = Lists.newArrayList(querySqlReq.getModelIds()); - ModelFilter modelFilter = new ModelFilter(); - modelFilter.setModelIds(modelIds); - List modelInfos = modelService.getModelList(modelFilter); - String modelNameCn = Constants.EMPTY; - if (!CollectionUtils.isEmpty(modelInfos)) { - modelNameCn = modelInfos.get(0).getName(); - } - MetaFilter metaFilter = new MetaFilter(modelIds); - List dimensionDescList = dimensionService.getDimensions(metaFilter); - String finalDomainNameCn = modelNameCn; - dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName())) - .forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName())); - - if (!CollectionUtils.isEmpty(need2Apply)) { - ModelResp modelResp = modelInfos.get(0); - List admins = modelService.getModelAdmin(modelResp.getId()); - log.info("in doFilterLogic, need2Apply:{}", need2Apply); - String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins); - throw new InvalidPermissionException(message); - } - } - - private void doFilterCheckLogic(Set resAuthName, Set sensitiveResReq, - List modelIdInDataSet, QueryStructReq queryStructReq) { - Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq); - Set need2Apply = resFilterSet.stream() - .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); - Set nameCnSet = new HashSet<>(); - ModelFilter modelFilter = new ModelFilter(false, modelIdInDataSet); - Map modelRespMap = modelService.getModelMap(modelFilter); - List dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIdInDataSet)); - dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName())) - .forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName())); - - if (!CollectionUtils.isEmpty(need2Apply)) { - List admins = modelService.getModelAdmin(modelIdInDataSet.get(0)); - log.info("in doFilterLogic, need2Apply:{}", need2Apply); - String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins); - throw new InvalidPermissionException(message); - } - } - - public Object checkStructPermission(ProceedingJoinPoint point, QueryStructReq queryStructReq) throws Throwable { - Object[] args = point.getArgs(); - User user = (User) args[1]; - // fetch data permission meta information - SchemaFilterReq filter = new SchemaFilterReq(); - filter.setModelIds(queryStructReq.getModelIds()); - filter.setDataSetId(queryStructReq.getDataSetId()); - SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); - List modelIdInDataSet = semanticSchemaResp.getModelResps().stream() - .map(ModelResp::getId).collect(Collectors.toList()); - Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq); - log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege); - - Set sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp); - Set sensitiveResReq = res4Privilege.parallelStream() - .filter(sensitiveResByModel::contains).collect(Collectors.toSet()); - log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInDataSet, sensitiveResReq); - - // query user privilege info - AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, - modelIdInDataSet, sensitiveResReq); - // get sensitiveRes that user has privilege - Set resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet); - - // if sensitive fields without permission are involved in filter, thrown an exception - doFilterCheckLogic(resAuthSet, sensitiveResReq, modelIdInDataSet, queryStructReq); - - // row permission pre-filter - doRowPermission(queryStructReq, authorizedResource); - - // proceed - SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed(); - - if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { - // if sensitiveRes is empty - log.info("sensitiveResReq is empty"); - return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource); - } - - // if the column has no permission, hit * - Set need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req)) - .collect(Collectors.toSet()); - SemanticQueryResp queryResultAfterDesensitization = - desensitizationData(queryResultWithColumns, need2Apply); - addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply); - - return queryResultAfterDesensitization; - - } - - public boolean allSensitiveResReqIsOk(Set sensitiveResReq, Set resAuthSet) { - if (resAuthSet.containsAll(sensitiveResReq)) { - return true; - } - log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet); - return false; } private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) { @@ -343,7 +235,7 @@ public class S2DataPermissionAspect { } - public boolean doModelAdmin(User user, List modelIds) { + public boolean checkModelAdmin(User user, List modelIds) { List modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN); if (CollectionUtils.isEmpty(modelListAdmin)) { return false; @@ -353,13 +245,14 @@ public class S2DataPermissionAspect { } } - public void doModelVisible(User user, List modelIds) { + public void checkModelVisible(User user, List modelIds) { List modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE) .stream().map(ModelResp::getId).collect(Collectors.toList()); - modelIds.removeAll(modelListVisible); - if (!CollectionUtils.isEmpty(modelIds)) { + List modelIdCopied = new ArrayList<>(modelIds); + modelIdCopied.removeAll(modelListVisible); + if (!CollectionUtils.isEmpty(modelIdCopied)) { MetaFilter metaFilter = new MetaFilter(); - metaFilter.setIds(modelIds); + metaFilter.setIds(modelIdCopied); List modelResps = modelService.getModelList(metaFilter); ModelResp modelResp = modelResps.stream().findFirst().orElse(null); if (modelResp == null) { @@ -370,7 +263,7 @@ public class S2DataPermissionAspect { } } - public Set getHighSensitiveColsByModelId(SemanticSchemaResp semanticSchemaResp) { + public Set getHighSensitiveBizNamesByModelId(SemanticSchemaResp semanticSchemaResp) { Set highSensitiveCols = new HashSet<>(); if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) { semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp -> @@ -385,16 +278,12 @@ public class S2DataPermissionAspect { return highSensitiveCols; } - public AuthorizedResourceResp getAuthorizedResource(User user, List modelIds, - Set sensitiveResReq) { - List resourceReqList = new ArrayList<>(); - sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res))); + public AuthorizedResourceResp getAuthorizedResource(User user, List modelIds) { QueryAuthResReq queryAuthResReq = new QueryAuthResReq(); - queryAuthResReq.setResources(resourceReqList); queryAuthResReq.setModelIds(modelIds); AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user); - log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds, - authorizedResource); + log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", + user.getName(), modelIds, authorizedResource); return authorizedResource; } @@ -403,142 +292,15 @@ public class S2DataPermissionAspect { return authService.queryAuthorizedResources(queryAuthResReq, user); } - public Set getAuthResNameSet(AuthorizedResourceResp authorizedResource, List modelIds) { - Set resAuthName = new HashSet<>(); - List authResGrpList = authorizedResource.getResources(); - authResGrpList.stream().forEach(authResGrp -> { - List cols = authResGrp.getGroup(); - if (!CollectionUtils.isEmpty(cols)) { - cols.stream().filter(col -> modelIds.contains(col.getModelId())) - .forEach(col -> resAuthName.add(col.getName())); - } - - }); - log.info("resAuthName:{}", resAuthName); - return resAuthName; - } - - public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns, - List modelIds, - AuthorizedResourceResp authResource) { - addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet()); - return resultWithColumns; - } - - public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set need2Apply) { - log.debug("start desensitizationData logic"); - if (CollectionUtils.isEmpty(need2Apply)) { - log.info("user has all sensitiveRes"); - return raw; - } - - List columns = raw.getColumns(); - - boolean doDesensitization = false; - for (QueryColumn queryColumn : columns) { - for (String sensitiveCol : need2Apply) { - if (queryColumn.getNameEn().contains(sensitiveCol)) { - doDesensitization = true; - break; - } - } - } - if (!doDesensitization) { - return raw; - } - - SemanticQueryResp queryResultWithColumns = raw; - try { - queryResultWithColumns = deepCopyResult(raw); - } catch (Exception e) { - log.warn("deepCopyResult: ", e); - } - addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply); - desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply); - return queryResultWithColumns; - } - - private void addAuthorizedSchemaInfo(List columns, Set need2Apply) { - if (CollectionUtils.isEmpty(need2Apply)) { - return; - } - columns.stream().forEach(col -> { - if (need2Apply.contains(getName(col.getNameEn()))) { - col.setAuthorized(false); - } - }); - } - - private String getName(String nameEn) { - Pattern pattern = Pattern.compile("\\((.*?)\\)"); - Matcher matcher = pattern.matcher(nameEn); - if (matcher.find()) { - return matcher.group(1).replaceAll("`", ""); - } - return nameEn; - } - - private void desensitizationInternal(List> result, Set need2Apply) { - log.info("start desensitizationInternal logic"); - for (int i = 0; i < result.size(); i++) { - Map row = result.get(i); - Map newRow = new HashMap<>(); - for (String col : row.keySet()) { - boolean sensitive = false; - for (String sensitiveCol : need2Apply) { - if (col.contains(sensitiveCol)) { - sensitive = true; - break; - } - } - if (sensitive) { - newRow.put(col, "******"); - } else { - newRow.put(col, row.get(col)); - } - } - result.set(i, newRow); - } - } - - private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception { - SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); - BeanUtils.copyProperties(raw, queryResultWithColumns); - - List columns = new ArrayList<>(); - if (!CollectionUtils.isEmpty(raw.getColumns())) { - String columnsStr = MAPPER.writeValueAsString(raw.getColumns()); - columns = MAPPER.readValue(columnsStr, new TypeReference>() { - }); - queryResultWithColumns.setColumns(columns); - } - queryResultWithColumns.setColumns(columns); - - List> resultData = new ArrayList<>(); - if (!CollectionUtils.isEmpty(raw.getResultList())) { - for (Map line : raw.getResultList()) { - Map newLine = new HashMap<>(); - newLine.putAll(line); - resultData.add(newLine); - } - } - queryResultWithColumns.setResultList(resultData); - return queryResultWithColumns; - } - - public void addPromptInfoInfo(List modelIds, SemanticQueryResp queryResultWithColumns, - AuthorizedResourceResp authorizedResource, Set need2Apply) { + public void addHint(List modelIds, SemanticQueryResp queryResultWithColumns, + AuthorizedResourceResp authorizedResource) { List filters = authorizedResource.getFilters(); - if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) { + if (CollectionUtils.isEmpty(filters)) { return; } List admins = modelService.getModelAdmin(modelIds.get(0)); - if (!CollectionUtils.isEmpty(need2Apply)) { - String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins); - queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo)); - } + if (!CollectionUtils.isEmpty(filters)) { - log.debug("dimensionFilters:{}", filters); ModelResp modelResp = modelService.getModel(modelIds.get(0)); List exprList = new ArrayList<>(); List descList = new ArrayList<>(); @@ -550,20 +312,9 @@ public class S2DataPermissionAspect { }); String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s"; String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins); - queryResultWithColumns.setQueryAuthorization( new QueryAuthorization(modelResp.getName(), exprList, descList, message)); - log.info("queryResultWithColumns:{}", queryResultWithColumns); } } - private List getModelsInDataSet(SemanticQueryReq queryReq) { - List modelIds = queryReq.getModelIds(); - if (queryReq.getDataSetId() != null) { - DataSetResp dataSetResp = dataSetService.getDataSet(queryReq.getDataSetId()); - modelIds = dataSetResp.getAllModels(); - } - return modelIds; - } - } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index f906c31b6..e8c690b17 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -227,6 +227,7 @@ public class S2SemanticLayerService implements SemanticLayerService { return queryByReq(querySqlReq, user); } + @S2DataPermission @Override public TranslateResp translate(TranslateSqlReq translateSqlReq, User user) throws Exception { T queryReq = translateSqlReq.getQueryReq(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index 0d387f218..39f1af6eb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -114,28 +114,21 @@ public class QueryStructUtils { return metricIds; } - public Set getResNameEn(QueryStructReq queryStructCmd) { + public Set getBizNameFromStruct(QueryStructReq queryStructReq) { Set resNameEnSet = new HashSet<>(); - queryStructCmd.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn())); - resNameEnSet.addAll(queryStructCmd.getGroups()); - queryStructCmd.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn())); - sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col)); + queryStructReq.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn())); + resNameEnSet.addAll(queryStructReq.getGroups()); + queryStructReq.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn())); + sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col)); return resNameEnSet; } public Set getResName(QuerySqlReq querySqlReq) { - Set resNameSet = SqlSelectHelper.getAllFields(querySqlReq.getSql()) - .stream().collect(Collectors.toSet()); - return resNameSet; + return new HashSet<>(SqlSelectHelper.getAllFields(querySqlReq.getSql())); } - public Set getResNameEnExceptInternalCol(QueryStructReq queryStructCmd) { - Set resNameEnSet = getResNameEn(queryStructCmd); - return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); - } - - public Set getResNameEnExceptInternalCol(QuerySqlReq querySqlReq, - SemanticSchemaResp semanticSchemaResp) { + public Set getBizNameFromSql(QuerySqlReq querySqlReq, + SemanticSchemaResp semanticSchemaResp) { Set resNameSet = getResName(querySqlReq); Set resNameEnSet = new HashSet<>(); if (semanticSchemaResp != null) { @@ -155,23 +148,6 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public Set getFilterResNameEn(QueryStructReq queryStructCmd) { - Set resNameEnSet = new HashSet<>(); - sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col)); - return resNameEnSet; - } - - public Set getFilterResNameEnExceptInternalCol(QueryStructReq queryStructCmd) { - Set resNameEnSet = getFilterResNameEn(queryStructCmd); - return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); - } - - public Set getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) { - String sql = querySqlReq.getSql(); - Set resNameEnSet = SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); - return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); - } - public ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) { List dimensionIds = getDimensionIds(queryStructCmd); List metricIds = getMetricIds(queryStructCmd); @@ -281,13 +257,5 @@ public class QueryStructUtils { return null; } - public List getDateCol() { - return dateModeUtils.getDateCol(); - } - - public String getVariablePrefix() { - return variablePrefix; - } - } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java index 307353eb3..4723537b6 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java @@ -101,18 +101,23 @@ public class QueryBySqlTest extends BaseTest { @Test public void testAuthorization_sensitive_metric() throws Exception { User tom = DataUtils.getUserTom(); + assertThrows(InvalidPermissionException.class, + () -> queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom)); + } + + @Test + public void testAuthorization_sensitive_metric_jack() throws Exception { + User jack = DataUtils.getUserJack(); SemanticQueryResp semanticQueryResp = - queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom); - Assertions.assertEquals(false, semanticQueryResp.getColumns().get(0).getAuthorized()); - Assertions.assertEquals("******", - semanticQueryResp.getResultList().get(0).get("SUM(stay_hours)")); + queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计", jack); + Assertions.assertTrue(semanticQueryResp.getResultList().size() > 0); } @Test public void testAuthorization_row_permission() throws Exception { User tom = DataUtils.getUserTom(); SemanticQueryResp semanticQueryResp = - queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom); + queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计 WHERE department ='HR'", tom); Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage()); Assertions.assertTrue(semanticQueryResp.getSql().contains("user_name = 'tom'")); } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java index c7f4abe6c..f34c7a6b7 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java @@ -111,15 +111,14 @@ public class QueryByStructTest extends BaseTest { } @Test - public void testAuthorization_sensitive_metric() throws Exception { + public void testAuthorization_sensitive_metric() { User tom = DataUtils.getUserTom(); Aggregator aggregator = new Aggregator(); aggregator.setFunc(AggOperatorEnum.SUM); aggregator.setColumn("stay_hours"); - QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator); - SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom); - Assertions.assertEquals(false, semanticQueryResp.getColumns().get(1).getAuthorized()); - Assertions.assertEquals("******", semanticQueryResp.getResultList().get(0).get("stay_hours")); + QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"), aggregator); + assertThrows(InvalidPermissionException.class, + () -> semanticLayerService.queryByReq(queryStructReq, tom)); } @Test @@ -127,7 +126,7 @@ public class QueryByStructTest extends BaseTest { User tom = DataUtils.getUserTom(); Aggregator aggregator = new Aggregator(); aggregator.setFunc(AggOperatorEnum.SUM); - aggregator.setColumn("stay_hours"); + aggregator.setColumn("pv"); QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator); SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom); Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());