topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless] Fix the null pointer issue when handling execution permissions and retrieving recommended metrics (#1780)

Browse Source
This commit is contained in:
lexluo09
2024-10-10 23:22:48 +08:00
committed by GitHub
parent 484861571f
commit 0fe7f0e1a7
4 changed files with 72 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java
View File

@@ -43,6 +43,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT;
@@ -51,7 +52,9 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT;
import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT;
/** Add ratio queries for metric queries. */
/**
* Add ratio queries for metric queries.
*/
@Slf4j
public class MetricRatioProcessor implements ExecuteResultProcessor {
@@ -71,10 +74,8 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo,
QueryResult queryResult) {
Set<String> resultMetricNames = getResultMetricNames(queryResult);
Set<String> resultMetricNames = new HashSet<>();
queryResult.getQueryColumns().stream().forEach(
c -> resultMetricNames.addAll(SqlSelectHelper.getColumnFromExpr(c.getNameEn())));
Optional<SchemaElement> ratioMetric = semanticParseInfo.getMetrics().stream()
.filter(m -> resultMetricNames.contains(m.getBizName())).findFirst();
@@ -92,17 +93,20 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
if (!lastDayOp.isPresent()) {
return new AggregateInfo();
}
Optional<Map<String, Object>> lastValue = queryResult.getQueryResults().stream()
.filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst();
MetricInfo metricInfo = new MetricInfo();
metricInfo.setStatistics(new HashMap<>());
if (lastValue.isPresent()
&& lastValue.get().containsKey(ratioMetric.get().getBizName())) {
lastValue.ifPresent(value -> {
if (value.containsKey(ratioMetric.get().getBizName())) {
DecimalFormat df = new DecimalFormat("#.####");
metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName())));
metricInfo.setValue(df.format(value.get(ratioMetric.get().getBizName())));
}
metricInfo.setDate(lastValue.get().get(dateField).toString());
metricInfo.setDate(value.get(dateField).toString());
});
CompletableFuture<MetricInfo> metricInfoRoll =
CompletableFuture.supplyAsync(() -> queryRatio(user, semanticParseInfo,
@@ -110,11 +114,14 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
CompletableFuture<MetricInfo> metricInfoOver =
CompletableFuture.supplyAsync(() -> queryRatio(user, semanticParseInfo,
ratioMetric.get(), AggOperatorEnum.RATIO_OVER, queryResult));
CompletableFuture.allOf(metricInfoRoll, metricInfoOver);
CompletableFuture.allOf(metricInfoRoll, metricInfoOver).join();
metricInfo.setName(metricInfoRoll.get().getName());
metricInfo.setValue(metricInfoRoll.get().getValue());
metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics());
metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics());
aggregateInfo.getMetricInfos().add(metricInfo);
} catch (Exception e) {
log.error("queryRatio error {}", e);
@@ -122,6 +129,15 @@ public class MetricRatioProcessor implements ExecuteResultProcessor {
return aggregateInfo;
}
private static Set<String> getResultMetricNames(QueryResult queryResult) {
if (queryResult.getQueryColumns() == null) {
return new HashSet<>();
}
return queryResult.getQueryColumns().stream()
.flatMap(c -> SqlSelectHelper.getColumnFromExpr(c.getNameEn()).stream())
.collect(Collectors.toSet());
}
@SneakyThrows
private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo,
SchemaElement metric, AggOperatorEnum aggOperatorEnum, QueryResult queryResult) {

77
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -43,6 +43,7 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
@@ -180,64 +181,66 @@ public class S2DataPermissionAspect {
private void doRowPermission(QuerySqlReq querySqlReq,
AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) {
authorizedResource.getFilters().stream()
.forEach(filter -> dimensionFilters.addAll(filter.getExpressions()));
}
log.debug("Start doRowPermission logic");
if (CollectionUtils.isEmpty(dimensionFilters)) {
log.debug("dimensionFilters is empty");
if (CollectionUtils.isEmpty(authorizedResource.getFilters())) {
log.debug("authorizedResource.getFilters() is empty");
return;
}
List<String> dimensionFilters = authorizedResource.getFilters().stream()
.flatMap(filter -> filter.getExpressions().stream()).filter(StringUtils::isNotBlank)
.collect(Collectors.toList());
if (dimensionFilters.isEmpty()) {
log.debug("Dimension filters are empty");
return;
}
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
StringJoiner joiner = new StringJoiner(" OR ");
dimensionFilters.stream().filter(
filter -> StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim()))
.forEach(filter -> joiner.add(" ( " + filter + " ) "));
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlAddHelper.addWhere(querySqlReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());
querySqlReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());
String originalSql = querySqlReq.getSql();
String modifiedSql = SqlAddHelper.addWhere(originalSql, expression);
log.info("Before doRowPermission, querySqlReq: {}", originalSql);
querySqlReq.setSql(modifiedSql);
log.info("After doRowPermission, querySqlReq: {}", modifiedSql);
}
} catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
} catch (JSQLParserException e) {
log.error("JSQLParser encountered an exception: {}", e.toString());
}
}
private void doRowPermission(QueryStructReq queryStructReq,
AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) {
authorizedResource.getFilters().stream()
.forEach(filter -> dimensionFilters.addAll(filter.getExpressions()));
}
if (CollectionUtils.isEmpty(dimensionFilters)) {
if (CollectionUtils.isEmpty(authorizedResource.getFilters())) {
log.debug("authorizedResource.getFilters() is empty");
return;
}
List<String> dimensionFilters = authorizedResource.getFilters().stream()
.flatMap(filter -> filter.getExpressions().stream()).filter(StringUtils::isNotBlank)
.collect(Collectors.toList());
if (dimensionFilters.isEmpty()) {
log.debug("dimensionFilters is empty");
return;
}
dimensionFilters.stream().forEach(filter -> {
if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) {
joiner.add(" ( " + filter + " ) ");
}
});
StringJoiner joiner = new StringJoiner(" OR ");
dimensionFilters.forEach(filter -> joiner.add(" ( " + filter + " ) "));
if (StringUtils.isNotEmpty(joiner.toString())) {
String joinedFilters = joiner.toString();
if (StringUtils.isNotEmpty(joinedFilters)) {
log.info("before doRowPermission, queryStructReq:{}", queryStructReq);
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString());
List<Filter> filters =
Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>()
: queryStructReq.getOriginalFilter();
Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joinedFilters);
List<Filter> filters = Optional.ofNullable(queryStructReq.getOriginalFilter())
.orElseGet(ArrayList::new);
filters.add(filter);
queryStructReq.setDimensionFilters(filters);
log.info("after doRowPermission, queryStructReq:{}", queryStructReq);

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java
View File

@@ -11,6 +11,7 @@ import org.springframework.beans.BeanUtils;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
public class DomainConvert {
@@ -26,6 +27,9 @@ public class DomainConvert {
}
public static DomainResp convert(DomainDO domainDO, Map<Long, String> domainFullPathMap) {
if (Objects.isNull(domainDO)) {
return null;
}
DomainResp domainResp = new DomainResp();
BeanUtils.copyProperties(domainDO, domainResp);
domainResp.setFullPath(domainFullPathMap.get(domainDO.getId()));

2
launchers/standalone/src/main/resources/hanlp.properties
View File

@@ -1,2 +1,2 @@
root=.
CustomDictionaryPath=data/dictionary/custom/DimValue_1_1.txt;data/dictionary/custom/DimValue_1_2.txt;
CustomDictionaryPath=data/dictionary/custom/dic_value_1_DIMENSION_1.txt