topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(semantic) perfect dsl permission (#161)

Browse Source
This commit is contained in:
mainmain
2023-09-27 21:01:44 +08:00
committed by GitHub
parent 6047c787b3
commit e688422ec3
10 changed files with 233 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -59,6 +59,12 @@ public class QueryController {
return queryService.queryByStructWithAuth(queryStructReq, user);
}
@PostMapping("/queryStatement")
public Object queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
Object result = queryService.queryByQueryStatement(queryStatement);
return result;
}
@PostMapping("/struct/parse")
public SqlParserResp parseByStruct(@RequestBody ParseSqlReq parseSqlReq,
HttpServletRequest request,

19
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/AuthCommonService.java
View File

@@ -155,9 +155,11 @@ public class AuthCommonService {
boolean doDesensitization = false;
for (QueryColumn queryColumn : columns) {
if (need2Apply.contains(queryColumn.getNameEn())) {
doDesensitization = true;
break;
for (String sensitiveCol : need2Apply) {
if (queryColumn.getNameEn().contains(sensitiveCol)) {
doDesensitization = true;
break;
}
}
}
if (!doDesensitization) {
@@ -192,8 +194,15 @@ public class AuthCommonService {
Map<String, Object> row = result.get(i);
Map<String, Object> newRow = new HashMap<>();
for (String col : row.keySet()) {
if (need2Apply.contains(col)) {
newRow.put(col, "****");
boolean sensitive = false;
for (String sensitiveCol : need2Apply) {
if (col.contains(sensitiveCol)) {
sensitive = true;
break;
}
}
if (sensitive) {
newRow.put(col, "******");
} else {
newRow.put(col, row.get(col));
}

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
View File

@@ -10,6 +10,8 @@ import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import java.util.List;
public interface QueryService {
@@ -25,6 +27,8 @@ public interface QueryService {
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
Object queryByQueryStatement(QueryStatement queryStatement);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
<T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -77,11 +77,17 @@ public class QueryServiceImpl implements QueryService {
} catch (Exception e) {
log.info("convertToQueryStatement has a exception:{}", e.toString());
}
log.info("queryStatement:{}", queryStatement);
QueryResultWithSchemaResp results = semanticQueryEngine.execute(queryStatement);
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return results;
}
public Object queryByQueryStatement(QueryStatement queryStatement) {
QueryResultWithSchemaResp results = semanticQueryEngine.execute(queryStatement);
return results;
}
private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = new ArrayList<>();

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DslDataAspect.java
View File

@@ -84,7 +84,7 @@ public class DslDataAspect {
authCommonService.doModelVisible(user, modelId);
// 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryDslReq);
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryDslReq, user);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
Set<String> sensitiveResByModel = authCommonService.getHighSensitiveColsByModelId(modelId);
@@ -117,6 +117,7 @@ public class DslDataAspect {
// 6.if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
QueryResultWithSchemaResp queryResultAfterDesensitization = authCommonService
.desensitizationData(queryResultWithColumns, need2Apply);
authCommonService.addPromptInfoInfo(modelId, queryResultAfterDesensitization, authorizedResource, need2Apply);

40
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.Aggregator;
@@ -9,9 +10,13 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
@@ -25,8 +30,11 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.assertj.core.util.Lists;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -42,6 +50,8 @@ public class QueryStructUtils {
private final Catalog catalog;
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
@Autowired
private SchemaService schemaService;
public QueryStructUtils(
DateUtils dateUtils,
@@ -147,17 +157,37 @@ public class QueryStructUtils {
sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col));
return resNameEnSet;
}
public Set<String> getResNameEn(QueryDslReq queryDslReq) {
Set<String> resNameEnSet = SqlParserSelectHelper.getAllFields(queryDslReq.getSql())
public Set<String> getResName(QueryDslReq queryDslReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryDslReq.getSql())
.stream().collect(Collectors.toSet());
return resNameEnSet;
return resNameSet;
}
public Set<String> getResNameEnExceptInternalCol(QueryStructReq queryStructCmd) {
Set<String> resNameEnSet = getResNameEn(queryStructCmd);
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getResNameEnExceptInternalCol(QueryDslReq queryDslReq) {
Set<String> resNameEnSet = getResNameEn(queryDslReq);
public Set<String> getResNameEnExceptInternalCol(QueryDslReq queryDslReq, User user) {
Set<String> resNameSet = getResName(queryDslReq);
Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryDslReq.getModelId());
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
List<MetricSchemaResp> metrics = modelSchemaRespList.get(0).getMetrics();
List<DimSchemaResp> dimensions = modelSchemaRespList.get(0).getDimensions();
metrics.stream().forEach(o -> {
if (resNameSet.contains(o.getName())) {
resNameEnSet.add(o.getBizName());
}
});
dimensions.stream().forEach(o -> {
if (resNameSet.contains(o.getName())) {
resNameEnSet.add(o.getBizName());
}
});
}
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}