(improvement)(auth) Make row permissions take effect during the translate sql phase and refactor the auth code (#1368)

Co-authored-by: lxwcodemonkey

auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/pojo/AuthResGrp.java

@@ -1,11 +0,0 @@
-package com.tencent.supersonic.auth.api.authorization.pojo;
-
-import java.util.ArrayList;
-import java.util.List;
-import lombok.Data;
-
-@Data
-public class AuthResGrp {
-
-    private List<AuthRes> group = new ArrayList<>();
-}

auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/request/QueryAuthResReq.java

@@ -1,7 +1,6 @@
 package com.tencent.supersonic.auth.api.authorization.request;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
 import lombok.Data;
 import lombok.ToString;
 import org.springframework.util.CollectionUtils;
@@ -15,8 +14,6 @@ public class QueryAuthResReq {
 
     private List<String> departmentIds = new ArrayList<>();
 
-    private List<AuthRes> resources;
-
     private Long modelId;
 
     private List<Long> modelIds;

auth/api/src/main/java/com/tencent/supersonic/auth/api/authorization/response/AuthorizedResourceResp.java

@@ -1,16 +1,16 @@
 package com.tencent.supersonic.auth.api.authorization.response;
 
-import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
+import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
 import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
+import lombok.Data;
+
 import java.util.ArrayList;
 import java.util.List;
 
-import lombok.Data;
-
 @Data
 public class AuthorizedResourceResp {
 
-    private List<AuthResGrp> resources = new ArrayList<>();
+    private List<AuthRes> authResList = new ArrayList<>();
 
     private List<DimensionFilter> filters = new ArrayList<>();
 }

auth/authorization/src/main/java/com/tencent/supersonic/auth/authorization/service/AuthServiceImpl.java

@@ -1,19 +1,16 @@
 package com.tencent.supersonic.auth.authorization.service;
 
-import com.google.common.collect.Lists;
 import com.google.gson.Gson;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.service.UserService;
 import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
 import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
-import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
 import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
 import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
 import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
 import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
 import com.tencent.supersonic.auth.api.authorization.service.AuthService;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -79,66 +76,48 @@ public class AuthServiceImpl implements AuthService {
 
     @Override
     public AuthorizedResourceResp queryAuthorizedResources(QueryAuthResReq req, User user) {
+        if (CollectionUtils.isEmpty(req.getModelIds())) {
+            return new AuthorizedResourceResp();
+        }
         Set<String> userOrgIds = userService.getUserAllOrgId(user.getName());
         List<AuthGroup> groups = getAuthGroups(req.getModelIds(), user.getName(), new ArrayList<>(userOrgIds));
         AuthorizedResourceResp resource = new AuthorizedResourceResp();
         Map<Long, List<AuthGroup>> authGroupsByModelId = groups.stream()
                 .collect(Collectors.groupingBy(AuthGroup::getModelId));
-        Map<Long, List<AuthRes>> reqAuthRes = req.getResources().stream()
+        for (Long modelId : req.getModelIds()) {
-                .collect(Collectors.groupingBy(AuthRes::getModelId));
-
-        for (Long modelId : reqAuthRes.keySet()) {
-            List<AuthRes> reqResourcesList = reqAuthRes.get(modelId);
-            AuthResGrp rg = new AuthResGrp();
             if (authGroupsByModelId.containsKey(modelId)) {
                 List<AuthGroup> authGroups = authGroupsByModelId.get(modelId);
-                for (AuthRes reqRes : reqResourcesList) {
                 for (AuthGroup authRuleGroup : authGroups) {
                     List<AuthRule> authRules = authRuleGroup.getAuthRules();
-                        List<String> allAuthItems = new ArrayList<>();
+                    for (AuthRule authRule : authRules) {
-                        authRules.forEach(authRule -> allAuthItems.addAll(authRule.resourceNames()));
+                        for (String resBizName : authRule.resourceNames()) {
-
+                            resource.getAuthResList().add(new AuthRes(modelId, resBizName));
-                        if (allAuthItems.contains(reqRes.getName())) {
-                            rg.getGroup().add(reqRes);
-                        }
-
                         }
                     }
                 }
-            if (!CollectionUtils.isEmpty(rg.getGroup())) {
-                resource.getResources().add(rg);
             }
         }
-
+        Set<Map.Entry<Long, List<AuthGroup>>> entries = authGroupsByModelId.entrySet();
-        if (!CollectionUtils.isEmpty(req.getModelIds())) {
+        for (Map.Entry<Long, List<AuthGroup>> entry : entries) {
-            List<AuthGroup> authGroups = Lists.newArrayList();
+            List<AuthGroup> authGroups = entry.getValue();
-            for (Long modelId : authGroupsByModelId.keySet()) {
+            for (AuthGroup authGroup : authGroups) {
-                authGroups.addAll(authGroupsByModelId.getOrDefault(modelId, Lists.newArrayList()));
-            }
-            if (!CollectionUtils.isEmpty(authGroups)) {
-                for (AuthGroup group : authGroups) {
-                    if (group.getDimensionFilters() != null
-                            && group.getDimensionFilters().stream().anyMatch(expr ->
-                            !StringUtils.isEmpty(expr))) {
                 DimensionFilter df = new DimensionFilter();
-                        df.setDescription(group.getDimensionFilterDescription());
+                df.setDescription(authGroup.getDimensionFilterDescription());
-                        df.setExpressions(group.getDimensionFilters());
+                df.setExpressions(authGroup.getDimensionFilters());
                 resource.getFilters().add(df);
             }
         }
-            }
-        }
         return resource;
     }
 
     private List<AuthGroup> getAuthGroups(List<Long> modelIds, String userName, List<String> departmentIds) {
         List<AuthGroup> groups = load().stream()
                 .filter(group -> {
-                    if (CollectionUtils.isEmpty(modelIds) || !modelIds.contains(group.getModelId())) {
+                    if (!modelIds.contains(group.getModelId())) {
                         return false;
                     }
-                    if (!CollectionUtils.isEmpty(group.getAuthorizedUsers()) && group.getAuthorizedUsers()
+                    if (!CollectionUtils.isEmpty(group.getAuthorizedUsers())
-                            .contains(userName)) {
+                            && group.getAuthorizedUsers().contains(userName)) {
                         return true;
                     }
                     for (String departmentId : departmentIds) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java

@@ -9,7 +9,9 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.apache.commons.lang3.StringUtils;
 
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
 
@@ -58,4 +60,20 @@ public class SemanticSchemaResp {
                 .findFirst().orElse(null);
     }
 
+    public Set<String> getNameFromBizNames(Set<String> bizNames) {
+        Set<String> names = new HashSet<>();
+        for (String bizName : bizNames) {
+            DimSchemaResp dimSchemaResp = getDimension(bizName);
+            if (dimSchemaResp != null) {
+                names.add(dimSchemaResp.getName());
+                continue;
+            }
+            MetricSchemaResp metricSchemaResp = getMetric(bizName);
+            if (metricSchemaResp != null) {
+                names.add(metricSchemaResp.getName());
+            }
+        }
+        return names;
+    }
+
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java

@@ -1,42 +1,32 @@
 package com.tencent.supersonic.headless.server.aspect;
 
-import com.fasterxml.jackson.core.type.TypeReference;
+
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
-import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
 import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
 import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
 import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
 import com.tencent.supersonic.auth.api.authorization.service.AuthService;
-import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
 import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.common.pojo.QueryAuthorization;
-import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
 import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
-import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq;
 import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
+import com.tencent.supersonic.headless.api.pojo.request.TranslateSqlReq;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
-import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
-import com.tencent.supersonic.headless.server.pojo.ModelFilter;
+import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
-import com.tencent.supersonic.headless.server.web.service.DimensionService;
 import com.tencent.supersonic.headless.server.web.service.ModelService;
 import com.tencent.supersonic.headless.server.web.service.SchemaService;
-import com.tencent.supersonic.headless.server.web.service.DataSetService;
-import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.Expression;
@@ -46,45 +36,29 @@ import org.aspectj.lang.ProceedingJoinPoint;
 import org.aspectj.lang.annotation.Around;
 import org.aspectj.lang.annotation.Aspect;
 import org.aspectj.lang.annotation.Pointcut;
-import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import static com.tencent.supersonic.common.pojo.Constants.MINUS;
 
 @Component
 @Aspect
-@Order(1)
 @Slf4j
 public class S2DataPermissionAspect {
 
-    private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat(
-            new SimpleDateFormat(Constants.DAY_FORMAT));
-
     @Autowired
     private QueryStructUtils queryStructUtils;
     @Autowired
-    private DimensionService dimensionService;
-    @Autowired
     private ModelService modelService;
     @Autowired
     private SchemaService schemaService;
     @Autowired
-    private DataSetService dataSetService;
-    @Autowired
     private AuthService authService;
 
     @Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)")
@@ -93,8 +67,19 @@ public class S2DataPermissionAspect {
 
     @Around("s2PermissionCheck()")
     public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
+        //1. check args
         Object[] objects = joinPoint.getArgs();
-        SemanticQueryReq queryReq = (SemanticQueryReq) objects[0];
+        boolean needQueryData = true;
+        SemanticQueryReq queryReq = null;
+        if (objects[0] instanceof SemanticQueryReq) {
+            queryReq = (SemanticQueryReq) objects[0];
+        } else if (objects[0] instanceof TranslateSqlReq) {
+            queryReq = (SemanticQueryReq) ((TranslateSqlReq<?>) objects[0]).getQueryReq();
+            needQueryData = false;
+        }
+        if (queryReq == null) {
+            throw new InvalidArgumentException("queryReq is not Invalid");
+        }
         if (!queryReq.isNeedAuth()) {
             log.info("needAuth is false, there is no need to check permissions.");
             return joinPoint.proceed();
@@ -103,179 +88,86 @@ public class S2DataPermissionAspect {
         if (Objects.isNull(user) || StringUtils.isEmpty(user.getName())) {
             throw new RuntimeException("please provide user information");
         }
-        List<Long> modelIds = getModelsInDataSet(queryReq);
 
-        // determine whether admin of the model
+        SemanticSchemaResp semanticSchemaResp = getSemanticSchemaResp(queryReq);
-        if (doModelAdmin(user, modelIds)) {
+        List<Long> modelIds = getModelIds(semanticSchemaResp);
+
+        //2. determine whether admin of the model
+        if (checkModelAdmin(user, modelIds)) {
             return joinPoint.proceed();
         }
-        // determine whether the subject field is visible
+        //3. determine whether the model is visible to cur user
-        doModelVisible(user, modelIds);
+        checkModelVisible(user, modelIds);
 
+        //4. get permissions auth to cur user
+        AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIds);
+
+        //5. check col permission
+        if (needQueryData) {
+            checkColPermission(queryReq, authorizedResource, modelIds, semanticSchemaResp);
+        }
+        //6. check row permission
+        checkRowPermission(queryReq, authorizedResource);
+
+        //7. add hint to user
+        Object result = joinPoint.proceed();
+        if (result instanceof SemanticQueryResp) {
+            addHint(modelIds, (SemanticQueryResp) result, authorizedResource);
+        }
+        return result;
+    }
+
+    private void checkColPermission(SemanticQueryReq semanticQueryReq, AuthorizedResourceResp authorizedResource,
+                                    List<Long> modelIds, SemanticSchemaResp semanticSchemaResp) {
+        // get high sensitive fields in query
+        Set<String> bizNamesInQueryReq = getBizNameInQueryReq(semanticQueryReq, semanticSchemaResp);
+        Set<String> sensitiveBizNamesByModel = getHighSensitiveBizNamesByModelId(semanticSchemaResp);
+        Set<String> sensitiveBizNameInQuery = bizNamesInQueryReq.parallelStream()
+                .filter(sensitiveBizNamesByModel::contains).collect(Collectors.toSet());
+
+        //get high sensitive field cur user has been authed
+        Set<String> sensitiveBizNameUserAuthed = authorizedResource.getAuthResList()
+                .stream().map(AuthRes::getName).collect(Collectors.toSet());
+        sensitiveBizNameInQuery.removeAll(sensitiveBizNameUserAuthed);
+        if (!CollectionUtils.isEmpty(sensitiveBizNameInQuery)) {
+            Set<String> sensitiveResNames = semanticSchemaResp.getNameFromBizNames(sensitiveBizNameInQuery);
+            List<String> modelAdmin = modelService.getModelAdmin(modelIds.get(0));
+            String message = String.format("存在以下敏感资源:%s您暂无权限，请联系管理员%s申请",
+                    sensitiveResNames, modelAdmin);
+            throw new InvalidPermissionException(message);
+        }
+    }
+
+    private void checkRowPermission(SemanticQueryReq queryReq,
+                                      AuthorizedResourceResp authorizedResource) {
         if (queryReq instanceof QuerySqlReq) {
-            return checkSqlPermission(joinPoint, (QuerySqlReq) queryReq);
+            doRowPermission((QuerySqlReq) queryReq, authorizedResource);
         }
         if (queryReq instanceof QueryStructReq) {
-            return checkStructPermission(joinPoint, (QueryStructReq) queryReq);
+            doRowPermission((QueryStructReq) queryReq, authorizedResource);
+        }
+    }
+
+    private Set<String> getBizNameInQueryReq(SemanticQueryReq queryReq, SemanticSchemaResp semanticSchemaResp) {
+        if (queryReq instanceof QuerySqlReq) {
+            return queryStructUtils.getBizNameFromSql((QuerySqlReq) queryReq, semanticSchemaResp);
+        }
+        if (queryReq instanceof QueryStructReq) {
+            return queryStructUtils.getBizNameFromStruct((QueryStructReq) queryReq);
         }
         throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
     }
 
-    private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySqlReq)
+    private SemanticSchemaResp getSemanticSchemaResp(SemanticQueryReq semanticQueryReq) {
-            throws Throwable {
-        Object[] objects = joinPoint.getArgs();
-        User user = (User) objects[1];
-        // fetch data permission meta information
         SchemaFilterReq filter = new SchemaFilterReq();
-        filter.setModelIds(querySqlReq.getModelIds());
+        filter.setModelIds(semanticQueryReq.getModelIds());
-        filter.setDataSetId(querySqlReq.getDataSetId());
+        filter.setDataSetId(semanticQueryReq.getDataSetId());
-        SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
+        return schemaService.fetchSemanticSchema(filter);
-        List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
+    }
+
+    private List<Long> getModelIds(SemanticSchemaResp semanticSchemaResp) {
+        return semanticSchemaResp.getModelResps().stream()
                 .map(ModelResp::getId).collect(Collectors.toList());
-        Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, semanticSchemaResp);
-        log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
-
-        Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
-        Set<String> sensitiveResReq = res4Privilege.parallelStream()
-                .filter(sensitiveResByModel::contains).collect(Collectors.toSet());
-
-        // query user privilege info
-        AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelIdInDataSet, sensitiveResReq);
-        // get sensitiveRes that user has privilege
-        Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
-
-        // if sensitive fields without permission are involved in filter, thrown an exception
-        doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq);
-
-        // row permission pre-filter
-        doRowPermission(querySqlReq, authorizedResource);
-
-        // proceed
-        SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
-
-        if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
-            // if sensitiveRes is empty
-            log.info("sensitiveResReq is empty");
-            return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
-        }
-
-        // if the column has no permission, hit *
-        Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
-                .collect(Collectors.toSet());
-        log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
-        SemanticQueryResp queryResultAfterDesensitization =
-                desensitizationData(queryResultWithColumns, need2Apply);
-        addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
-
-        return queryResultAfterDesensitization;
-    }
-
-    private void doFilterCheckLogic(QuerySqlReq querySqlReq, Set<String> resAuthName,
-            Set<String> sensitiveResReq) {
-        Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySqlReq);
-        Set<String> need2Apply = resFilterSet.stream()
-                .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
-        Set<String> nameCnSet = new HashSet<>();
-
-        List<Long> modelIds = Lists.newArrayList(querySqlReq.getModelIds());
-        ModelFilter modelFilter = new ModelFilter();
-        modelFilter.setModelIds(modelIds);
-        List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
-        String modelNameCn = Constants.EMPTY;
-        if (!CollectionUtils.isEmpty(modelInfos)) {
-            modelNameCn = modelInfos.get(0).getName();
-        }
-        MetaFilter metaFilter = new MetaFilter(modelIds);
-        List<DimensionResp> dimensionDescList = dimensionService.getDimensions(metaFilter);
-        String finalDomainNameCn = modelNameCn;
-        dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
-                .forEach(dim -> nameCnSet.add(finalDomainNameCn + MINUS + dim.getName()));
-
-        if (!CollectionUtils.isEmpty(need2Apply)) {
-            ModelResp modelResp = modelInfos.get(0);
-            List<String> admins = modelService.getModelAdmin(modelResp.getId());
-            log.info("in doFilterLogic, need2Apply:{}", need2Apply);
-            String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
-            throw new InvalidPermissionException(message);
-        }
-    }
-
-    private void doFilterCheckLogic(Set<String> resAuthName, Set<String> sensitiveResReq,
-                                    List<Long> modelIdInDataSet, QueryStructReq queryStructReq) {
-        Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryStructReq);
-        Set<String> need2Apply = resFilterSet.stream()
-                .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
-        Set<String> nameCnSet = new HashSet<>();
-        ModelFilter modelFilter = new ModelFilter(false, modelIdInDataSet);
-        Map<Long, ModelResp> modelRespMap = modelService.getModelMap(modelFilter);
-        List<DimensionResp> dimensionDescList = dimensionService.getDimensions(new MetaFilter(modelIdInDataSet));
-        dimensionDescList.stream().filter(dim -> need2Apply.contains(dim.getBizName()))
-                .forEach(dim -> nameCnSet.add(modelRespMap.get(dim.getModelId()).getName() + MINUS + dim.getName()));
-
-        if (!CollectionUtils.isEmpty(need2Apply)) {
-            List<String> admins = modelService.getModelAdmin(modelIdInDataSet.get(0));
-            log.info("in doFilterLogic, need2Apply:{}", need2Apply);
-            String message = String.format("您没有以下维度%s权限, 请联系管理员%s开通", nameCnSet, admins);
-            throw new InvalidPermissionException(message);
-        }
-    }
-
-    public Object checkStructPermission(ProceedingJoinPoint point, QueryStructReq queryStructReq) throws Throwable {
-        Object[] args = point.getArgs();
-        User user = (User) args[1];
-        // fetch data permission meta information
-        SchemaFilterReq filter = new SchemaFilterReq();
-        filter.setModelIds(queryStructReq.getModelIds());
-        filter.setDataSetId(queryStructReq.getDataSetId());
-        SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
-        List<Long> modelIdInDataSet = semanticSchemaResp.getModelResps().stream()
-                .map(ModelResp::getId).collect(Collectors.toList());
-        Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
-        log.info("modelId:{}, res4Privilege:{}", modelIdInDataSet, res4Privilege);
-
-        Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(semanticSchemaResp);
-        Set<String> sensitiveResReq = res4Privilege.parallelStream()
-                .filter(sensitiveResByModel::contains).collect(Collectors.toSet());
-        log.info("this query domainId:{}, sensitiveResReq:{}", modelIdInDataSet, sensitiveResReq);
-
-        // query user privilege info
-        AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
-                modelIdInDataSet, sensitiveResReq);
-        // get sensitiveRes that user has privilege
-        Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInDataSet);
-
-        // if sensitive fields without permission are involved in filter, thrown an exception
-        doFilterCheckLogic(resAuthSet, sensitiveResReq, modelIdInDataSet, queryStructReq);
-
-        // row permission pre-filter
-        doRowPermission(queryStructReq, authorizedResource);
-
-        // proceed
-        SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed();
-
-        if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
-            // if sensitiveRes is empty
-            log.info("sensitiveResReq is empty");
-            return getQueryResultWithColumns(queryResultWithColumns, modelIdInDataSet, authorizedResource);
-        }
-
-        // if the column has no permission, hit *
-        Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
-                .collect(Collectors.toSet());
-        SemanticQueryResp queryResultAfterDesensitization =
-                desensitizationData(queryResultWithColumns, need2Apply);
-        addPromptInfoInfo(modelIdInDataSet, queryResultAfterDesensitization, authorizedResource, need2Apply);
-
-        return queryResultAfterDesensitization;
-
-    }
-
-    public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
-        if (resAuthSet.containsAll(sensitiveResReq)) {
-            return true;
-        }
-        log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
-        return false;
     }
 
     private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
@@ -343,7 +235,7 @@ public class S2DataPermissionAspect {
 
     }
 
-    public boolean doModelAdmin(User user, List<Long> modelIds) {
+    public boolean checkModelAdmin(User user, List<Long> modelIds) {
         List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
         if (CollectionUtils.isEmpty(modelListAdmin)) {
             return false;
@@ -353,13 +245,14 @@ public class S2DataPermissionAspect {
         }
     }
 
-    public void doModelVisible(User user, List<Long> modelIds) {
+    public void checkModelVisible(User user, List<Long> modelIds) {
         List<Long> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE)
                 .stream().map(ModelResp::getId).collect(Collectors.toList());
-        modelIds.removeAll(modelListVisible);
+        List<Long> modelIdCopied = new ArrayList<>(modelIds);
-        if (!CollectionUtils.isEmpty(modelIds)) {
+        modelIdCopied.removeAll(modelListVisible);
+        if (!CollectionUtils.isEmpty(modelIdCopied)) {
             MetaFilter metaFilter = new MetaFilter();
-            metaFilter.setIds(modelIds);
+            metaFilter.setIds(modelIdCopied);
             List<ModelResp> modelResps = modelService.getModelList(metaFilter);
             ModelResp modelResp = modelResps.stream().findFirst().orElse(null);
             if (modelResp == null) {
@@ -370,7 +263,7 @@ public class S2DataPermissionAspect {
         }
     }
 
-    public Set<String> getHighSensitiveColsByModelId(SemanticSchemaResp semanticSchemaResp) {
+    public Set<String> getHighSensitiveBizNamesByModelId(SemanticSchemaResp semanticSchemaResp) {
         Set<String> highSensitiveCols = new HashSet<>();
         if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
             semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp ->
@@ -385,16 +278,12 @@ public class S2DataPermissionAspect {
         return highSensitiveCols;
     }
 
-    public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds,
+    public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds) {
-                                                        Set<String> sensitiveResReq) {
-        List<AuthRes> resourceReqList = new ArrayList<>();
-        sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res)));
         QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
-        queryAuthResReq.setResources(resourceReqList);
         queryAuthResReq.setModelIds(modelIds);
         AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
-        log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds,
+        log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}",
-                authorizedResource);
+                user.getName(), modelIds, authorizedResource);
         return authorizedResource;
     }
 
@@ -403,142 +292,15 @@ public class S2DataPermissionAspect {
         return authService.queryAuthorizedResources(queryAuthResReq, user);
     }
 
-    public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, List<Long> modelIds) {
+    public void addHint(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
-        Set<String> resAuthName = new HashSet<>();
+                                  AuthorizedResourceResp authorizedResource) {
-        List<AuthResGrp> authResGrpList = authorizedResource.getResources();
-        authResGrpList.stream().forEach(authResGrp -> {
-            List<AuthRes> cols = authResGrp.getGroup();
-            if (!CollectionUtils.isEmpty(cols)) {
-                cols.stream().filter(col -> modelIds.contains(col.getModelId()))
-                        .forEach(col -> resAuthName.add(col.getName()));
-            }
-
-        });
-        log.info("resAuthName:{}", resAuthName);
-        return resAuthName;
-    }
-
-    public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns,
-                                                       List<Long> modelIds,
-                                                       AuthorizedResourceResp authResource) {
-        addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
-        return resultWithColumns;
-    }
-
-    public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set<String> need2Apply) {
-        log.debug("start desensitizationData logic");
-        if (CollectionUtils.isEmpty(need2Apply)) {
-            log.info("user has all sensitiveRes");
-            return raw;
-        }
-
-        List<QueryColumn> columns = raw.getColumns();
-
-        boolean doDesensitization = false;
-        for (QueryColumn queryColumn : columns) {
-            for (String sensitiveCol : need2Apply) {
-                if (queryColumn.getNameEn().contains(sensitiveCol)) {
-                    doDesensitization = true;
-                    break;
-                }
-            }
-        }
-        if (!doDesensitization) {
-            return raw;
-        }
-
-        SemanticQueryResp queryResultWithColumns = raw;
-        try {
-            queryResultWithColumns = deepCopyResult(raw);
-        } catch (Exception e) {
-            log.warn("deepCopyResult: ", e);
-        }
-        addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
-        desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
-        return queryResultWithColumns;
-    }
-
-    private void addAuthorizedSchemaInfo(List<QueryColumn> columns, Set<String> need2Apply) {
-        if (CollectionUtils.isEmpty(need2Apply)) {
-            return;
-        }
-        columns.stream().forEach(col -> {
-            if (need2Apply.contains(getName(col.getNameEn()))) {
-                col.setAuthorized(false);
-            }
-        });
-    }
-
-    private String getName(String nameEn) {
-        Pattern pattern = Pattern.compile("\\((.*?)\\)");
-        Matcher matcher = pattern.matcher(nameEn);
-        if (matcher.find()) {
-            return matcher.group(1).replaceAll("`", "");
-        }
-        return nameEn;
-    }
-
-    private void desensitizationInternal(List<Map<String, Object>> result, Set<String> need2Apply) {
-        log.info("start desensitizationInternal logic");
-        for (int i = 0; i < result.size(); i++) {
-            Map<String, Object> row = result.get(i);
-            Map<String, Object> newRow = new HashMap<>();
-            for (String col : row.keySet()) {
-                boolean sensitive = false;
-                for (String sensitiveCol : need2Apply) {
-                    if (col.contains(sensitiveCol)) {
-                        sensitive = true;
-                        break;
-                    }
-                }
-                if (sensitive) {
-                    newRow.put(col, "******");
-                } else {
-                    newRow.put(col, row.get(col));
-                }
-            }
-            result.set(i, newRow);
-        }
-    }
-
-    private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception {
-        SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
-        BeanUtils.copyProperties(raw, queryResultWithColumns);
-
-        List<QueryColumn> columns = new ArrayList<>();
-        if (!CollectionUtils.isEmpty(raw.getColumns())) {
-            String columnsStr = MAPPER.writeValueAsString(raw.getColumns());
-            columns = MAPPER.readValue(columnsStr, new TypeReference<List<QueryColumn>>() {
-            });
-            queryResultWithColumns.setColumns(columns);
-        }
-        queryResultWithColumns.setColumns(columns);
-
-        List<Map<String, Object>> resultData = new ArrayList<>();
-        if (!CollectionUtils.isEmpty(raw.getResultList())) {
-            for (Map<String, Object> line : raw.getResultList()) {
-                Map<String, Object> newLine = new HashMap<>();
-                newLine.putAll(line);
-                resultData.add(newLine);
-            }
-        }
-        queryResultWithColumns.setResultList(resultData);
-        return queryResultWithColumns;
-    }
-
-    public void addPromptInfoInfo(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
-                                  AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
         List<DimensionFilter> filters = authorizedResource.getFilters();
-        if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
+        if (CollectionUtils.isEmpty(filters)) {
             return;
         }
         List<String> admins = modelService.getModelAdmin(modelIds.get(0));
-        if (!CollectionUtils.isEmpty(need2Apply)) {
+
-            String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
-            queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
-        }
         if (!CollectionUtils.isEmpty(filters)) {
-            log.debug("dimensionFilters:{}", filters);
             ModelResp modelResp = modelService.getModel(modelIds.get(0));
             List<String> exprList = new ArrayList<>();
             List<String> descList = new ArrayList<>();
@@ -550,20 +312,9 @@ public class S2DataPermissionAspect {
             });
             String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
             String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
-
             queryResultWithColumns.setQueryAuthorization(
                     new QueryAuthorization(modelResp.getName(), exprList, descList, message));
-            log.info("queryResultWithColumns:{}", queryResultWithColumns);
         }
     }
 
-    private List<Long> getModelsInDataSet(SemanticQueryReq queryReq) {
-        List<Long> modelIds = queryReq.getModelIds();
-        if (queryReq.getDataSetId() != null) {
-            DataSetResp dataSetResp = dataSetService.getDataSet(queryReq.getDataSetId());
-            modelIds = dataSetResp.getAllModels();
-        }
-        return modelIds;
-    }
-
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java

@@ -227,6 +227,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
         return queryByReq(querySqlReq, user);
     }
 
+    @S2DataPermission
     @Override
     public <T> TranslateResp translate(TranslateSqlReq<T> translateSqlReq, User user) throws Exception {
         T queryReq = translateSqlReq.getQueryReq();

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java

@@ -114,27 +114,20 @@ public class QueryStructUtils {
         return metricIds;
     }
 
-    public Set<String> getResNameEn(QueryStructReq queryStructCmd) {
+    public Set<String> getBizNameFromStruct(QueryStructReq queryStructReq) {
         Set<String> resNameEnSet = new HashSet<>();
-        queryStructCmd.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn()));
+        queryStructReq.getAggregators().stream().forEach(agg -> resNameEnSet.add(agg.getColumn()));
-        resNameEnSet.addAll(queryStructCmd.getGroups());
+        resNameEnSet.addAll(queryStructReq.getGroups());
-        queryStructCmd.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn()));
+        queryStructReq.getOrders().stream().forEach(order -> resNameEnSet.add(order.getColumn()));
-        sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
+        sqlFilterUtils.getFiltersCol(queryStructReq.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
         return resNameEnSet;
     }
 
     public Set<String> getResName(QuerySqlReq querySqlReq) {
-        Set<String> resNameSet = SqlSelectHelper.getAllFields(querySqlReq.getSql())
+        return new HashSet<>(SqlSelectHelper.getAllFields(querySqlReq.getSql()));
-                .stream().collect(Collectors.toSet());
-        return resNameSet;
     }
 
-    public Set<String> getResNameEnExceptInternalCol(QueryStructReq queryStructCmd) {
+    public Set<String> getBizNameFromSql(QuerySqlReq querySqlReq,
-        Set<String> resNameEnSet = getResNameEn(queryStructCmd);
-        return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
-    }
-
-    public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySqlReq,
                                         SemanticSchemaResp semanticSchemaResp) {
         Set<String> resNameSet = getResName(querySqlReq);
         Set<String> resNameEnSet = new HashSet<>();
@@ -155,23 +148,6 @@ public class QueryStructUtils {
         return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
     }
 
-    public Set<String> getFilterResNameEn(QueryStructReq queryStructCmd) {
-        Set<String> resNameEnSet = new HashSet<>();
-        sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
-        return resNameEnSet;
-    }
-
-    public Set<String> getFilterResNameEnExceptInternalCol(QueryStructReq queryStructCmd) {
-        Set<String> resNameEnSet = getFilterResNameEn(queryStructCmd);
-        return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
-    }
-
-    public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) {
-        String sql = querySqlReq.getSql();
-        Set<String> resNameEnSet = SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
-        return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
-    }
-
     public ItemDateResp getItemDateResp(QueryStructReq queryStructCmd) {
         List<Long> dimensionIds = getDimensionIds(queryStructCmd);
         List<Long> metricIds = getMetricIds(queryStructCmd);
@@ -281,13 +257,5 @@ public class QueryStructUtils {
         return null;
     }
 
-    public List<String> getDateCol() {
-        return dateModeUtils.getDateCol();
-    }
-
-    public String getVariablePrefix() {
-        return variablePrefix;
-    }
-
 }
 

launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java

@@ -101,18 +101,23 @@ public class QueryBySqlTest extends BaseTest {
     @Test
     public void testAuthorization_sensitive_metric() throws Exception {
         User tom = DataUtils.getUserTom();
+        assertThrows(InvalidPermissionException.class,
+                () -> queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计  WHERE department ='HR'", tom));
+    }
+
+    @Test
+    public void testAuthorization_sensitive_metric_jack() throws Exception {
+        User jack = DataUtils.getUserJack();
         SemanticQueryResp semanticQueryResp =
-                queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计  WHERE department ='HR'", tom);
+                queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计", jack);
-        Assertions.assertEquals(false, semanticQueryResp.getColumns().get(0).getAuthorized());
+        Assertions.assertTrue(semanticQueryResp.getResultList().size() > 0);
-        Assertions.assertEquals("******",
-                semanticQueryResp.getResultList().get(0).get("SUM(stay_hours)"));
     }
 
     @Test
     public void testAuthorization_row_permission() throws Exception {
         User tom = DataUtils.getUserTom();
         SemanticQueryResp semanticQueryResp =
-                queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计  WHERE department ='HR'", tom);
+                queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计  WHERE department ='HR'", tom);
         Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());
         Assertions.assertTrue(semanticQueryResp.getSql().contains("user_name = 'tom'"));
     }

launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java

@@ -111,15 +111,14 @@ public class QueryByStructTest extends BaseTest {
     }
 
     @Test
-    public void testAuthorization_sensitive_metric() throws Exception {
+    public void testAuthorization_sensitive_metric() {
         User tom = DataUtils.getUserTom();
         Aggregator aggregator = new Aggregator();
         aggregator.setFunc(AggOperatorEnum.SUM);
         aggregator.setColumn("stay_hours");
-        QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
+        QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"), aggregator);
-        SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom);
+        assertThrows(InvalidPermissionException.class,
-        Assertions.assertEquals(false, semanticQueryResp.getColumns().get(1).getAuthorized());
+                () -> semanticLayerService.queryByReq(queryStructReq, tom));
-        Assertions.assertEquals("******", semanticQueryResp.getResultList().get(0).get("stay_hours"));
     }
 
     @Test
@@ -127,7 +126,7 @@ public class QueryByStructTest extends BaseTest {
         User tom = DataUtils.getUserTom();
         Aggregator aggregator = new Aggregator();
         aggregator.setFunc(AggOperatorEnum.SUM);
-        aggregator.setColumn("stay_hours");
+        aggregator.setColumn("pv");
         QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
         SemanticQueryResp semanticQueryResp = semanticLayerService.queryByReq(queryStructReq1, tom);
         Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());