topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](supersonic) based on version 0.7.2 (#34)

Browse Source 
Co-authored-by: zuopengge <hwzuopengge@tencent.com>
This commit is contained in:
mainmain
2023-08-20 17:30:35 +08:00
committed by GitHub
parent c93e60ced7
commit cf1b5336c3
122 changed files with 4045 additions and 1075 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java
View File

@@ -416,7 +416,7 @@ public class CalculateAggConverter implements SemanticConverter {
}
private static String getLimit(QueryStructReq queryStructCmd) {
if (queryStructCmd.getLimit() > 0) {
if (queryStructCmd != null && queryStructCmd.getLimit() > 0) {
return " limit " + String.valueOf(queryStructCmd.getLimit());
}
return "";

21
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java
View File

@@ -23,8 +23,7 @@ import org.springframework.util.CollectionUtils;
@Slf4j
public class ParserDefaultConverter implements SemanticConverter {
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
private final CalculateAggConverter calculateCoverterAgg;
private final QueryStructUtils queryStructUtils;
@@ -69,7 +68,7 @@ public class ParserDefaultConverter implements SemanticConverter {
// todo tmp delete
// support detail query
if (queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
String internalMetricName = generateInternalMetricName(catalog, queryStructCmd);
String internalMetricName = queryStructUtils.generateInternalMetricName(queryStructCmd.getModelId(), queryStructCmd.getGroups());
sqlCommend.getMetrics().add(internalMetricName);
}
@@ -77,21 +76,5 @@ public class ParserDefaultConverter implements SemanticConverter {
}
public String generateInternalMetricName(Catalog catalog, QueryStructReq queryStructCmd) {
String internalMetricNamePrefix = "";
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
log.warn("group is empty!");
} else {
String group = queryStructCmd.getGroups().get(0).equalsIgnoreCase("sys_imp_date")
? queryStructCmd.getGroups().get(1) : queryStructCmd.getGroups().get(0);
DimensionResp dimension = catalog.getDimension(group, queryStructCmd.getModelId());
String datasourceBizName = dimension.getDatasourceBizName();
if (Strings.isNotEmpty(datasourceBizName)) {
internalMetricNamePrefix = datasourceBizName + UNDERLINE;
}
}
String internalMetricName = internalMetricNamePrefix + internalMetricNameSuffix;
return internalMetricName;
}
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java
View File

@@ -50,7 +50,7 @@ public class QueryController {
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
return queryService.queryByStruct(queryStructReq, user, request);
return queryService.queryByStructWithAuth(queryStructReq, user);
}
@PostMapping("/struct/parse")

7
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.semantic.query.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.semantic.api.model.pojo.QueryStat;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
@@ -9,22 +8,18 @@ import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
public interface QueryService {
Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user, HttpServletRequest request)
QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user)
throws Exception;
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception;
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
List<QueryStat> getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend);
}

12
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java
View File

@@ -22,7 +22,7 @@ import com.tencent.supersonic.semantic.query.utils.StatUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
@@ -110,8 +110,8 @@ public class QueryServiceImpl implements QueryService {
@Override
@DataPermission
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user, HttpServletRequest request)
throws Exception {
@SneakyThrows
public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) {
return queryByStruct(queryStructCmd, user);
}
@@ -171,12 +171,6 @@ public class QueryServiceImpl implements QueryService {
return statInfos;
}
@Override
public List<QueryStat> getQueryStatInfoWithoutCache(ItemUseReq itemUseCommend) {
return statUtils.getQueryStatInfoWithoutCache(itemUseCommend);
}
private boolean isCache(QueryStructReq queryStructCmd) {
if (!cacheEnable) {
return false;

45
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java
View File

@@ -90,11 +90,15 @@ public class DataPermissionAOP {
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("lease provide user information");
}
//1. determine whether admin of the model
if (doModelAdmin(user, queryStructReq)) {
return point.proceed();
}
// 1. determine whether the subject field is visible
doDomainVisible(user, queryStructReq);
// 2. determine whether the subject field is visible
doModelVisible(user, queryStructReq);
// 2. fetch data permission meta information
// 3. fetch data permission meta information
Long modelId = queryStructReq.getModelId();
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryStructReq);
log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);
@@ -105,18 +109,17 @@ public class DataPermissionAOP {
log.info("this query domainId:{}, sensitiveResReq:{}", modelId, sensitiveResReq);
// query user privilege info
HttpServletRequest request = (HttpServletRequest) args[2];
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, request, modelId, sensitiveResReq);
AuthorizedResourceResp authorizedResource = getAuthorizedResource(user, modelId, sensitiveResReq);
// get sensitiveRes that user has privilege
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, queryStructReq.getModelId());
// 3.if sensitive fields without permission are involved in filter, thrown an exception
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryStructReq, resAuthSet, sensitiveResReq);
// 4.row permission pre-filter
// 5.row permission pre-filter
doRowPermission(queryStructReq, authorizedResource);
// 5.proceed
// 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) point.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
@@ -136,7 +139,19 @@ public class DataPermissionAOP {
}
private void doDomainVisible(User user, QueryStructReq queryStructCmd) {
private boolean doModelAdmin(User user, QueryStructReq queryStructCmd) {
Long modelId = queryStructCmd.getModelId();
List<ModelResp> modelListAdmin = modelService.getModelListWithAuth(user.getName(), null, AuthType.ADMIN);
if (CollectionUtils.isEmpty(modelListAdmin)) {
return false;
} else {
Map<Long, List<ModelResp>> id2modelResp = modelListAdmin.stream()
.collect(Collectors.groupingBy(SchemaItem::getId));
return !CollectionUtils.isEmpty(id2modelResp) && id2modelResp.containsKey(modelId);
}
}
private void doModelVisible(User user, QueryStructReq queryStructCmd) {
Boolean visible = true;
Long domainId = queryStructCmd.getModelId();
List<ModelResp> modelListVisible = modelService.getModelListWithAuth(user.getName(), null, AuthType.VISIBLE);
@@ -251,15 +266,14 @@ public class DataPermissionAOP {
return resAuthName;
}
private AuthorizedResourceResp getAuthorizedResource(User user, HttpServletRequest request, Long domainId,
private AuthorizedResourceResp getAuthorizedResource(User user, Long domainId,
Set<String> sensitiveResReq) {
List<AuthRes> resourceReqList = new ArrayList<>();
sensitiveResReq.stream().forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res)));
sensitiveResReq.forEach(res -> resourceReqList.add(new AuthRes(domainId.toString(), res)));
QueryAuthResReq queryAuthResReq = new QueryAuthResReq();
queryAuthResReq.setUser(user.getName());
queryAuthResReq.setResources(resourceReqList);
queryAuthResReq.setModelId(domainId + "");
AuthorizedResourceResp authorizedResource = fetchAuthRes(request, queryAuthResReq);
AuthorizedResourceResp authorizedResource = fetchAuthRes(queryAuthResReq, user);
log.info("user:{}, domainId:{}, after queryAuthorizedResources:{}", user.getName(), domainId,
authorizedResource);
return authorizedResource;
@@ -396,10 +410,9 @@ public class DataPermissionAOP {
}
private AuthorizedResourceResp fetchAuthRes(HttpServletRequest request, QueryAuthResReq queryAuthResReq) {
log.info("Authorization:{}", request.getHeader("Authorization"));
private AuthorizedResourceResp fetchAuthRes(QueryAuthResReq queryAuthResReq, User user) {
log.info("queryAuthResReq:{}", queryAuthResReq);
return authService.queryAuthorizedResources(queryAuthResReq, request);
return authService.queryAuthorizedResources(queryAuthResReq, user);
}
}

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java
View File

@@ -35,7 +35,8 @@ public class DimValueAspect {
private DimensionService dimensionService;
@Around("execution(* com.tencent.supersonic.semantic.query.rest.QueryController.queryByStruct(..))" +
" || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStruct(..))")
" || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStruct(..))" +
" || execution(* com.tencent.supersonic.semantic.query.service.QueryService.queryByStructWithAuth(..))")
public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapEnable) {

10
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryReqConverter.java
View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.semantic.model.domain.ModelService;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@@ -28,6 +29,9 @@ public class QueryReqConverter {
@Autowired
private SemanticQueryEngine parserService;
@Autowired
private QueryStructUtils queryStructUtils;
public QueryStatement convert(QueryDslReq databaseReq, List<ModelSchemaResp> domainSchemas) throws Exception {
List<MetricTable> tables = new ArrayList<>();
@@ -60,6 +64,12 @@ public class QueryReqConverter {
}
metricTable.setDimensions(new ArrayList<>(collect));
metricTable.setAlias(tableName.toLowerCase());
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>(Arrays.asList(
queryStructUtils.generateInternalMetricName(databaseReq.getModelId(),
metricTable.getDimensions()))));
}
tables.add(metricTable);
ParseSqlReq result = new ParseSqlReq();

32
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -1,16 +1,19 @@
package com.tencent.supersonic.semantic.query.utils;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.model.pojo.ItemDateFilter;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
import com.tencent.supersonic.semantic.model.domain.Catalog;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -19,9 +22,12 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -32,6 +38,8 @@ public class QueryStructUtils {
private final SqlFilterUtils sqlFilterUtils;
private final Catalog catalog;
@Value("${internal.metric.cnt.suffix:internal_cnt}")
private String internalMetricNameSuffix;
public static Set<String> internalCols = new HashSet<>(
Arrays.asList("dayno", "plat_sys_var", "sys_imp_date", "sys_imp_week", "sys_imp_month"));
@@ -157,5 +165,23 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public String generateInternalMetricName(Long modelId, List<String> groups) {
String internalMetricNamePrefix = "";
if (CollectionUtils.isEmpty(groups)) {
log.warn("group is empty!");
} else {
String group = groups.get(0).equalsIgnoreCase("sys_imp_date")
? groups.get(1) : groups.get(0);
DimensionResp dimension = catalog.getDimension(group, modelId);
String datasourceBizName = dimension.getDatasourceBizName();
if (Strings.isNotEmpty(datasourceBizName)) {
internalMetricNamePrefix = datasourceBizName + UNDERLINE;
}
}
String internalMetricName = internalMetricNamePrefix + internalMetricNameSuffix;
return internalMetricName;
}
}

24
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryUtils.java
View File

@@ -3,11 +3,11 @@ package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
import com.tencent.supersonic.semantic.api.model.response.MetricResp;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
@@ -67,8 +67,8 @@ public class QueryUtils {
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, Long modelId) {
List<MetricResp> metricDescList = catalog.getMetrics(modelId);
List<DimensionResp> dimensionDescList = catalog.getDimensions(modelId);
Map<String, MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a, (k1, k2) -> k1));
Map<String,MetricResp> metricRespMap =
metricDescList.stream().collect(Collectors.toMap(MetricResp::getBizName, a -> a,(k1, k2)->k1));
Map<String, String> namePair = new HashMap<>();
Map<String, String> nameTypePair = new HashMap<>();
addSysTimeDimension(namePair, nameTypePair);
@@ -92,13 +92,27 @@ public class QueryUtils {
if (nameTypePair.containsKey(nameEn)) {
column.setShowType(nameTypePair.get(nameEn));
}
if (metricRespMap.containsKey(nameEn)) {
if (!nameTypePair.containsKey(nameEn) && isNumberType(column.getType())) {
column.setShowType("NUMBER");
}
if(metricRespMap.containsKey(nameEn)){
column.setDataFormatType(metricRespMap.get(nameEn).getDataFormatType());
column.setDataFormat(metricRespMap.get(nameEn).getDataFormat());
}
});
}
private boolean isNumberType(String type) {
if (StringUtils.isBlank(type)) {
return false;
}
if (type.equalsIgnoreCase("int") || type.equalsIgnoreCase("bigint")
|| type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")) {
return true;
}
return false;
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns,
QueryMultiStructReq queryMultiStructCmd) {
List<Aggregator> aggregators = queryMultiStructCmd.getQueryStructReqs().stream()