topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Add data permission integrating test (#721)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-02-05 18:37:50 +08:00
committed by GitHub
parent 8a342eb32a
commit 74d0ec2b23
9 changed files with 358 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java
View File

@@ -1,262 +0,0 @@
package com.tencent.supersonic.headless.server.aspect;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.server.service.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@Slf4j
public class AuthCheckBaseAspect {
private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat(
new SimpleDateFormat(Constants.DAY_FORMAT));
@Autowired
private AuthService authService;
@Autowired
private ModelService modelService;
public boolean doModelAdmin(User user, List<Long> modelIds) {
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
} else {
Set<Long> modelAdmins = modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet());
return !CollectionUtils.isEmpty(modelAdmins) && modelAdmins.containsAll(modelIds);
}
}
public void doModelVisible(User user, List<Long> modelIds) {
Boolean visible = true;
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE);
if (CollectionUtils.isEmpty(modelListVisible)) {
visible = false;
} else {
Set<Long> modelVisibles = modelListVisible.stream().map(ModelResp::getId).collect(Collectors.toSet());
if (!CollectionUtils.isEmpty(modelVisibles) && !modelVisibles.containsAll(modelIds)) {
visible = false;
}
}
if (!visible) {
ModelResp modelResp = modelService.getModel(modelIds.get(0));
String modelName = modelResp.getName();
List<String> admins = modelService.getModelAdmin(modelResp.getId());
String message = String.format("您没有模型[%s]权限，请联系管理员%s开通", modelName, admins);
throw new InvalidPermissionException(message);
}
}
public Set<String> getHighSensitiveColsByModelId(SemanticSchemaResp semanticSchemaResp) {
Set<String> highSensitiveCols = new HashSet<>();
if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(dimSchemaResp.getSensitiveLevel()))
.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) {
semanticSchemaResp.getMetrics().stream().filter(metricSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(metricSchemaResp.getSensitiveLevel()))
.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
}
public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelIds(modelIds);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds,
authorizedResource);
return authorizedResource;
}
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
log.info("queryAuthResReq:{}", queryAuthResReq);
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, List<Long> modelIds) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> modelIds.contains(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName()));
}
});
log.info("resAuthName:{}", resAuthName);
return resAuthName;
}
public boolean allSensitiveResReqIsOk(Set<String> sensitiveResReq, Set<String> resAuthSet) {
if (resAuthSet.containsAll(sensitiveResReq)) {
return true;
}
log.info("sensitiveResReq:{}, resAuthSet:{}", sensitiveResReq, resAuthSet);
return false;
}
public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns,
List<Long> modelIds,
AuthorizedResourceResp authResource) {
addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set<String> need2Apply) {
log.debug("start desensitizationData logic");
if (CollectionUtils.isEmpty(need2Apply)) {
log.info("user has all sensitiveRes");
return raw;
}
List<QueryColumn> columns = raw.getColumns();
boolean doDesensitization = false;
for (QueryColumn queryColumn : columns) {
for (String sensitiveCol : need2Apply) {
if (queryColumn.getNameEn().contains(sensitiveCol)) {
doDesensitization = true;
break;
}
}
}
if (!doDesensitization) {
return raw;
}
SemanticQueryResp queryResultWithColumns = raw;
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
log.warn("deepCopyResult: ", e);
}
addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
return queryResultWithColumns;
}
private void addAuthorizedSchemaInfo(List<QueryColumn> columns, Set<String> need2Apply) {
if (CollectionUtils.isEmpty(need2Apply)) {
return;
}
columns.stream().forEach(col -> {
if (need2Apply.contains(col.getNameEn())) {
col.setAuthorized(false);
}
});
}
private void desensitizationInternal(List<Map<String, Object>> result, Set<String> need2Apply) {
log.info("start desensitizationInternal logic");
for (int i = 0; i < result.size(); i++) {
Map<String, Object> row = result.get(i);
Map<String, Object> newRow = new HashMap<>();
for (String col : row.keySet()) {
boolean sensitive = false;
for (String sensitiveCol : need2Apply) {
if (col.contains(sensitiveCol)) {
sensitive = true;
break;
}
}
if (sensitive) {
newRow.put(col, "******");
} else {
newRow.put(col, row.get(col));
}
}
result.set(i, newRow);
}
}
private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
BeanUtils.copyProperties(raw, queryResultWithColumns);
List<QueryColumn> columns = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getColumns())) {
String columnsStr = MAPPER.writeValueAsString(raw.getColumns());
columns = MAPPER.readValue(columnsStr, new TypeReference<List<QueryColumn>>() {
});
queryResultWithColumns.setColumns(columns);
}
queryResultWithColumns.setColumns(columns);
List<Map<String, Object>> resultData = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getResultList())) {
for (Map<String, Object> line : raw.getResultList()) {
Map<String, Object> newLine = new HashMap<>();
newLine.putAll(line);
resultData.add(newLine);
}
}
queryResultWithColumns.setResultList(resultData);
return queryResultWithColumns;
}
public void addPromptInfoInfo(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
return;
}
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
if (!CollectionUtils.isEmpty(need2Apply)) {
String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
}
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
ModelResp modelResp = modelService.getModel(modelIds.get(0));
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream().forEach(filter -> {
descList.add(filter.getDescription());
exprList.add(filter.getExpressions().toString());
});
String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
log.info("queryResultWithColumns:{}", queryResultWithColumns);
}
}
}

264
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -1,12 +1,24 @@
package com.tencent.supersonic.headless.server.aspect;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRes;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthResGrp;
import com.tencent.supersonic.auth.api.authorization.pojo.DimensionFilter;
import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.QueryAuthorization;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AuthType;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.util.jsqlparser.SqlAddHelper;
@@ -18,11 +30,13 @@ import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.ViewResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.ViewService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -33,18 +47,22 @@ import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@@ -52,7 +70,10 @@ import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Aspect
@Order(1)
@Slf4j
public class S2DataPermissionAspect extends AuthCheckBaseAspect {
public class S2DataPermissionAspect {
private static final ObjectMapper MAPPER = new ObjectMapper().setDateFormat(
new SimpleDateFormat(Constants.DAY_FORMAT));
@Autowired
private QueryStructUtils queryStructUtils;
@@ -60,10 +81,12 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
private DimensionService dimensionService;
@Autowired
private ModelService modelService;
@Value("${permission.data.enable:true}")
private Boolean permissionDataEnable;
@Autowired
private SchemaService schemaService;
@Autowired
private ViewService viewService;
@Autowired
private AuthService authService;
@Pointcut("@annotation(com.tencent.supersonic.headless.server.annotation.S2DataPermission)")
private void s2PermissionCheck() {
@@ -71,11 +94,6 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
@Around("s2PermissionCheck()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("s2 permission check!");
if (!permissionDataEnable) {
log.info("not to check permission!");
return joinPoint.proceed();
}
Object[] objects = joinPoint.getArgs();
SemanticQueryReq queryReq = (SemanticQueryReq) objects[0];
if (!queryReq.isNeedAuth()) {
@@ -86,13 +104,14 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
List<Long> modelIds = getModelsInView(queryReq);
// determine whether admin of the model
if (doModelAdmin(user, queryReq.getModelIds())) {
if (doModelAdmin(user, modelIds)) {
return joinPoint.proceed();
}
// determine whether the subject field is visible
doModelVisible(user, queryReq.getModelIds());
doModelVisible(user, modelIds);
if (queryReq instanceof QuerySqlReq) {
return checkSqlPermission(joinPoint, (QuerySqlReq) queryReq);
@@ -225,7 +244,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user,
modelIdInView, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, queryStructReq.getModelIds());
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIdInView);
// if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
@@ -326,4 +345,225 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
}
public boolean doModelAdmin(User user, List<Long> modelIds) {
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user, null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
} else {
Set<Long> modelAdmins = modelListAdmin.stream().map(ModelResp::getId).collect(Collectors.toSet());
return !CollectionUtils.isEmpty(modelAdmins) && modelAdmins.containsAll(modelIds);
}
}
public void doModelVisible(User user, List<Long> modelIds) {
List<Long> modelListVisible = modelService.getModelListWithAuth(user, null, AuthType.VISIBLE)
.stream().map(ModelResp::getId).collect(Collectors.toList());
modelIds.removeAll(modelListVisible);
if (!CollectionUtils.isEmpty(modelIds)) {
MetaFilter metaFilter = new MetaFilter();
metaFilter.setIds(modelIds);
List<ModelResp> modelResps = modelService.getModelList(metaFilter);
ModelResp modelResp = modelResps.stream().findFirst().orElse(null);
if (modelResp == null) {
throw new InvalidArgumentException("查询的模型不存在");
}
String message = String.format("您没有模型[%s]权限，请联系管理员%s开通", modelResp.getName(), modelResp.getAdmins());
throw new InvalidPermissionException(message);
}
}
public Set<String> getHighSensitiveColsByModelId(SemanticSchemaResp semanticSchemaResp) {
Set<String> highSensitiveCols = new HashSet<>();
if (!CollectionUtils.isEmpty(semanticSchemaResp.getDimensions())) {
semanticSchemaResp.getDimensions().stream().filter(dimSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(dimSchemaResp.getSensitiveLevel()))
.forEach(dim -> highSensitiveCols.add(dim.getBizName()));
}
if (!CollectionUtils.isEmpty(semanticSchemaResp.getMetrics())) {
semanticSchemaResp.getMetrics().stream().filter(metricSchemaResp ->
SensitiveLevelEnum.HIGH.getCode().equals(metricSchemaResp.getSensitiveLevel()))
.forEach(metric -> highSensitiveCols.add(metric.getBizName()));
}
return highSensitiveCols;
}
public AuthorizedResourceResp getAuthorizedResource(User user, List<Long> modelIds,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(modelIds.get(0), res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelIds(modelIds);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), modelIds,
authorizedResource);
return authorizedResource;
}
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
log.info("queryAuthResReq:{}", queryAuthResReq);
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
public Set<String> getAuthResNameSet(AuthorizedResourceResp authorizedResource, List<Long> modelIds) {
Set<String> resAuthName = new HashSet<>();
List<AuthResGrp> authResGrpList = authorizedResource.getResources();
authResGrpList.stream().forEach(authResGrp -> {
List<AuthRes> cols = authResGrp.getGroup();
if (!CollectionUtils.isEmpty(cols)) {
cols.stream().filter(col -> modelIds.contains(col.getModelId()))
.forEach(col -> resAuthName.add(col.getName()));
}
});
log.info("resAuthName:{}", resAuthName);
return resAuthName;
}
public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns,
List<Long> modelIds,
AuthorizedResourceResp authResource) {
addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set<String> need2Apply) {
log.debug("start desensitizationData logic");
if (CollectionUtils.isEmpty(need2Apply)) {
log.info("user has all sensitiveRes");
return raw;
}
List<QueryColumn> columns = raw.getColumns();
boolean doDesensitization = false;
for (QueryColumn queryColumn : columns) {
for (String sensitiveCol : need2Apply) {
if (queryColumn.getNameEn().contains(sensitiveCol)) {
doDesensitization = true;
break;
}
}
}
if (!doDesensitization) {
return raw;
}
SemanticQueryResp queryResultWithColumns = raw;
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
log.warn("deepCopyResult: ", e);
}
addAuthorizedSchemaInfo(queryResultWithColumns.getColumns(), need2Apply);
desensitizationInternal(queryResultWithColumns.getResultList(), need2Apply);
return queryResultWithColumns;
}
private void addAuthorizedSchemaInfo(List<QueryColumn> columns, Set<String> need2Apply) {
if (CollectionUtils.isEmpty(need2Apply)) {
return;
}
columns.stream().forEach(col -> {
if (need2Apply.contains(getName(col.getNameEn()))) {
col.setAuthorized(false);
}
});
}
private String getName(String nameEn) {
Pattern pattern = Pattern.compile("\\((.*?)\\)");
Matcher matcher = pattern.matcher(nameEn);
if (matcher.find()) {
return matcher.group(1).replaceAll("`", "");
}
return nameEn;
}
private void desensitizationInternal(List<Map<String, Object>> result, Set<String> need2Apply) {
log.info("start desensitizationInternal logic");
for (int i = 0; i < result.size(); i++) {
Map<String, Object> row = result.get(i);
Map<String, Object> newRow = new HashMap<>();
for (String col : row.keySet()) {
boolean sensitive = false;
for (String sensitiveCol : need2Apply) {
if (col.contains(sensitiveCol)) {
sensitive = true;
break;
}
}
if (sensitive) {
newRow.put(col, "******");
} else {
newRow.put(col, row.get(col));
}
}
result.set(i, newRow);
}
}
private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
BeanUtils.copyProperties(raw, queryResultWithColumns);
List<QueryColumn> columns = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getColumns())) {
String columnsStr = MAPPER.writeValueAsString(raw.getColumns());
columns = MAPPER.readValue(columnsStr, new TypeReference<List<QueryColumn>>() {
});
queryResultWithColumns.setColumns(columns);
}
queryResultWithColumns.setColumns(columns);
List<Map<String, Object>> resultData = new ArrayList<>();
if (!CollectionUtils.isEmpty(raw.getResultList())) {
for (Map<String, Object> line : raw.getResultList()) {
Map<String, Object> newLine = new HashMap<>();
newLine.putAll(line);
resultData.add(newLine);
}
}
queryResultWithColumns.setResultList(resultData);
return queryResultWithColumns;
}
public void addPromptInfoInfo(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {
return;
}
List<String> admins = modelService.getModelAdmin(modelIds.get(0));
if (!CollectionUtils.isEmpty(need2Apply)) {
String promptInfo = String.format("当前结果已经过脱敏处理， 申请权限请联系管理员%s", admins);
queryResultWithColumns.setQueryAuthorization(new QueryAuthorization(promptInfo));
}
if (!CollectionUtils.isEmpty(filters)) {
log.debug("dimensionFilters:{}", filters);
ModelResp modelResp = modelService.getModel(modelIds.get(0));
List<String> exprList = new ArrayList<>();
List<String> descList = new ArrayList<>();
filters.stream().forEach(filter -> {
descList.add(filter.getDescription());
exprList.add(filter.getExpressions().toString());
});
String promptInfo = "当前结果已经过行权限过滤，详细过滤条件如下:%s, 申请权限请联系管理员%s";
String message = String.format(promptInfo, CollectionUtils.isEmpty(descList) ? exprList : descList, admins);
queryResultWithColumns.setQueryAuthorization(
new QueryAuthorization(modelResp.getName(), exprList, descList, message));
log.info("queryResultWithColumns:{}", queryResultWithColumns);
}
}
private List<Long> getModelsInView(SemanticQueryReq queryReq) {
List<Long> modelIds = queryReq.getModelIds();
if (queryReq.getViewId() != null) {
ViewResp viewResp = viewService.getView(queryReq.getViewId());
modelIds = viewResp.getAllModels();
}
return modelIds;
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -144,12 +144,12 @@ public class QueryStructUtils {
List<MetricSchemaResp> metrics = semanticSchemaResp.getMetrics();
List<DimSchemaResp> dimensions = semanticSchemaResp.getDimensions();
metrics.stream().forEach(o -> {
if (resNameSet.contains(o.getName())) {
if (resNameSet.contains(o.getName()) || resNameSet.contains(o.getBizName())) {
resNameEnSet.add(o.getBizName());
}
});
dimensions.stream().forEach(o -> {
if (resNameSet.contains(o.getName())) {
if (resNameSet.contains(o.getName()) || resNameSet.contains(o.getBizName())) {
resNameEnSet.add(o.getBizName());
}
});

7
launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
View File

@@ -528,14 +528,9 @@ public class ModelDemoDataLoader {
public void addAuthGroup_2() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId(3L);
authGroupReq.setName("tom_sales_permission");
authGroupReq.setName("tom_row_permission");
List<AuthRule> authRules = new ArrayList<>();
AuthRule authRule = new AuthRule();
authRule.setMetrics(Collections.singletonList("stay_hours"));
authRule.setDimensions(Collections.singletonList("page"));
authRules.add(authRule);
authGroupReq.setAuthRules(authRules);
authGroupReq.setDimensionFilters(Collections.singletonList("user_name = 'tom'"));
authGroupReq.setDimensionFilterDescription("用户名='tom'");

31
launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.headless;
import static java.time.LocalDate.now;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
@@ -16,11 +14,14 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.util.DataUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import static java.time.LocalDate.now;
public class BaseTest extends BaseApplication {
@@ -74,4 +75,26 @@ public class BaseTest extends BaseApplication {
queryStructReq.setOrders(orders);
return queryStructReq;
}
protected QueryStructReq buildQueryStructReq(List<String> groups,
Aggregator aggregator) {
QueryStructReq queryStructReq = new QueryStructReq();
for (Long modelId : DataUtils.getMetricAgentIModelIds()) {
queryStructReq.addModelId(modelId);
}
queryStructReq.setQueryType(QueryType.METRIC);
queryStructReq.setAggregators(Arrays.asList(aggregator));
if (CollectionUtils.isNotEmpty(groups)) {
queryStructReq.setGroups(groups);
}
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateMode.BETWEEN);
dateConf.setEndDate(now().plusDays(0).toString());
dateConf.setStartDate(now().plusDays(-365).toString());
queryStructReq.setDateInfo(dateConf);
return queryStructReq;
}
}

34
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryBySqlTest.java
View File

@@ -1,16 +1,17 @@
package com.tencent.supersonic.headless;
import static java.time.LocalDate.now;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.util.DataUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import static java.time.LocalDate.now;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
public class QueryBySqlTest extends BaseTest {
@Test
@@ -89,10 +90,29 @@ public class QueryBySqlTest extends BaseTest {
}
@Test
public void testAuthorization() {
User alice = new User(2L, "alice", "alice", "alice@email", 0);
public void testAuthorization_model() {
User alice = DataUtils.getUserAlice();
assertThrows(InvalidPermissionException.class,
() -> queryBySql("SELECT SUM(pv) FROM 超音数PVUV统计 WHERE department ='HR'", alice));
}
@Test
public void testAuthorization_sensitive_metric() throws Exception {
User tom = DataUtils.getUserTom();
SemanticQueryResp semanticQueryResp =
queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom);
Assertions.assertEquals(false, semanticQueryResp.getColumns().get(0).getAuthorized());
Assertions.assertEquals("******",
semanticQueryResp.getResultList().get(0).get("SUM(stay_hours)"));
}
@Test
public void testAuthorization_row_permission() throws Exception {
User tom = DataUtils.getUserTom();
SemanticQueryResp semanticQueryResp =
queryBySql("SELECT SUM(stay_hours) FROM 停留时长统计 WHERE department ='HR'", tom);
Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());
Assertions.assertTrue(semanticQueryResp.getSql().contains("`user_name` = 'tom'"));
}
}

43
launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java
View File

@@ -1,22 +1,26 @@
package com.tencent.supersonic.headless;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.util.DataUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
public class QueryByStructTest extends BaseTest {
@@ -89,10 +93,35 @@ public class QueryByStructTest extends BaseTest {
}
@Test
public void testAuthorization() {
public void testAuthorization_model() {
User alice = new User(2L, "alice", "alice", "alice@email", 0);
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"));
assertThrows(InvalidPermissionException.class,
() -> queryService.queryByReq(queryStructReq1, alice));
}
@Test
public void testAuthorization_sensitive_metric() throws Exception {
User tom = DataUtils.getUserTom();
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("stay_hours");
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryStructReq1, tom);
Assertions.assertEquals(false, semanticQueryResp.getColumns().get(1).getAuthorized());
Assertions.assertEquals("******", semanticQueryResp.getResultList().get(0).get("stay_hours"));
}
@Test
public void testAuthorization_row_permission() throws Exception {
User tom = DataUtils.getUserTom();
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("stay_hours");
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"), aggregator);
SemanticQueryResp semanticQueryResp = queryService.queryByReq(queryStructReq1, tom);
Assertions.assertNotNull(semanticQueryResp.getQueryAuthorization().getMessage());
Assertions.assertTrue(semanticQueryResp.getSql().contains("`user_name` = 'tom'"));
}
}

9
launchers/standalone/src/test/java/com/tencent/supersonic/headless/SchemaAuthTest.java
View File

@@ -45,6 +45,15 @@ public class SchemaAuthTest extends BaseTest {
modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()));
}
@Test
public void test_getVisibleModelList_alice() {
User user = DataUtils.getUserAlice();
List<ModelResp> modelResps = modelService.getModelListWithAuth(user, 0L, AuthType.VISIBLE);
List<Long> expectedModelIds = Lists.newArrayList(1L, 4L);
Assertions.assertEquals(expectedModelIds,
modelResps.stream().map(ModelResp::getId).collect(Collectors.toList()));
}
@Test
public void test_getViewList_alice() {
User user = DataUtils.getUserAlice();

4
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -37,6 +37,10 @@ public class DataUtils {
return User.get(2L, "jack");
}
public static User getUserTom() {
return User.get(3L, "tom");
}
public static QueryReq getQueryContextReq(Integer id, String query) {
QueryReq queryContextReq = new QueryReq();
queryContextReq.setQueryText(query);