diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java index 501193169..473af1fab 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java @@ -43,6 +43,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; @@ -51,7 +52,9 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; -/** Add ratio queries for metric queries. */ +/** + * Add ratio queries for metric queries. + */ @Slf4j public class MetricRatioProcessor implements ExecuteResultProcessor { @@ -71,10 +74,8 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) { + Set resultMetricNames = getResultMetricNames(queryResult); - Set resultMetricNames = new HashSet<>(); - queryResult.getQueryColumns().stream().forEach( - c -> resultMetricNames.addAll(SqlSelectHelper.getColumnFromExpr(c.getNameEn()))); Optional ratioMetric = semanticParseInfo.getMetrics().stream() .filter(m -> resultMetricNames.contains(m.getBizName())).findFirst(); @@ -92,17 +93,20 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { if (!lastDayOp.isPresent()) { return new AggregateInfo(); } + Optional> lastValue = queryResult.getQueryResults().stream() .filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst(); MetricInfo metricInfo = new MetricInfo(); metricInfo.setStatistics(new HashMap<>()); - if (lastValue.isPresent() - && lastValue.get().containsKey(ratioMetric.get().getBizName())) { - DecimalFormat df = new DecimalFormat("#.####"); - metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName()))); - } - metricInfo.setDate(lastValue.get().get(dateField).toString()); + + lastValue.ifPresent(value -> { + if (value.containsKey(ratioMetric.get().getBizName())) { + DecimalFormat df = new DecimalFormat("#.####"); + metricInfo.setValue(df.format(value.get(ratioMetric.get().getBizName()))); + } + metricInfo.setDate(value.get(dateField).toString()); + }); CompletableFuture metricInfoRoll = CompletableFuture.supplyAsync(() -> queryRatio(user, semanticParseInfo, @@ -110,11 +114,14 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { CompletableFuture metricInfoOver = CompletableFuture.supplyAsync(() -> queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER, queryResult)); - CompletableFuture.allOf(metricInfoRoll, metricInfoOver); + + CompletableFuture.allOf(metricInfoRoll, metricInfoOver).join(); + metricInfo.setName(metricInfoRoll.get().getName()); metricInfo.setValue(metricInfoRoll.get().getValue()); metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics()); metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics()); + aggregateInfo.getMetricInfos().add(metricInfo); } catch (Exception e) { log.error("queryRatio error {}", e); @@ -122,6 +129,15 @@ public class MetricRatioProcessor implements ExecuteResultProcessor { return aggregateInfo; } + private static Set getResultMetricNames(QueryResult queryResult) { + if (queryResult.getQueryColumns() == null) { + return new HashSet<>(); + } + return queryResult.getQueryColumns().stream() + .flatMap(c -> SqlSelectHelper.getColumnFromExpr(c.getNameEn()).stream()) + .collect(Collectors.toSet()); + } + @SneakyThrows private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, AggOperatorEnum aggOperatorEnum, QueryResult queryResult) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index db75d99f4..b2cc16847 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.stream.Collectors; @@ -180,64 +181,66 @@ public class S2DataPermissionAspect { private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) { - log.debug("start doRowPermission logic"); - StringJoiner joiner = new StringJoiner(" OR "); - List dimensionFilters = new ArrayList<>(); - if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) { - authorizedResource.getFilters().stream() - .forEach(filter -> dimensionFilters.addAll(filter.getExpressions())); - } + log.debug("Start doRowPermission logic"); - if (CollectionUtils.isEmpty(dimensionFilters)) { - log.debug("dimensionFilters is empty"); + if (CollectionUtils.isEmpty(authorizedResource.getFilters())) { + log.debug("authorizedResource.getFilters() is empty"); + return; + } + List dimensionFilters = authorizedResource.getFilters().stream() + .flatMap(filter -> filter.getExpressions().stream()).filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + + if (dimensionFilters.isEmpty()) { + log.debug("Dimension filters are empty"); return; } - dimensionFilters.stream().forEach(filter -> { - if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) { - joiner.add(" ( " + filter + " ) "); - } - }); + StringJoiner joiner = new StringJoiner(" OR "); + dimensionFilters.stream().filter( + filter -> StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) + .forEach(filter -> joiner.add(" ( " + filter + " ) ")); + try { Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); if (StringUtils.isNotEmpty(joiner.toString())) { - String sql = SqlAddHelper.addWhere(querySqlReq.getSql(), expression); - log.info("before doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql()); - querySqlReq.setSql(sql); - log.info("after doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql()); + String originalSql = querySqlReq.getSql(); + String modifiedSql = SqlAddHelper.addWhere(originalSql, expression); + log.info("Before doRowPermission, querySqlReq: {}", originalSql); + querySqlReq.setSql(modifiedSql); + log.info("After doRowPermission, querySqlReq: {}", modifiedSql); } - } catch (JSQLParserException jsqlParserException) { - log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); + } catch (JSQLParserException e) { + log.error("JSQLParser encountered an exception: {}", e.toString()); } } private void doRowPermission(QueryStructReq queryStructReq, AuthorizedResourceResp authorizedResource) { log.debug("start doRowPermission logic"); - StringJoiner joiner = new StringJoiner(" OR "); - List dimensionFilters = new ArrayList<>(); - if (!CollectionUtils.isEmpty(authorizedResource.getFilters())) { - authorizedResource.getFilters().stream() - .forEach(filter -> dimensionFilters.addAll(filter.getExpressions())); - } - if (CollectionUtils.isEmpty(dimensionFilters)) { + if (CollectionUtils.isEmpty(authorizedResource.getFilters())) { + log.debug("authorizedResource.getFilters() is empty"); + return; + } + List dimensionFilters = authorizedResource.getFilters().stream() + .flatMap(filter -> filter.getExpressions().stream()).filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + + if (dimensionFilters.isEmpty()) { log.debug("dimensionFilters is empty"); return; } - dimensionFilters.stream().forEach(filter -> { - if (StringUtils.isNotEmpty(filter) && StringUtils.isNotEmpty(filter.trim())) { - joiner.add(" ( " + filter + " ) "); - } - }); + StringJoiner joiner = new StringJoiner(" OR "); + dimensionFilters.forEach(filter -> joiner.add(" ( " + filter + " ) ")); - if (StringUtils.isNotEmpty(joiner.toString())) { + String joinedFilters = joiner.toString(); + if (StringUtils.isNotEmpty(joinedFilters)) { log.info("before doRowPermission, queryStructReq:{}", queryStructReq); - Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joiner.toString()); - List filters = - Objects.isNull(queryStructReq.getOriginalFilter()) ? new ArrayList<>() - : queryStructReq.getOriginalFilter(); + Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, joinedFilters); + List filters = Optional.ofNullable(queryStructReq.getOriginalFilter()) + .orElseGet(ArrayList::new); filters.add(filter); queryStructReq.setDimensionFilters(filters); log.info("after doRowPermission, queryStructReq:{}", queryStructReq); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java index 9c766d74b..7233513cd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DomainConvert.java @@ -11,6 +11,7 @@ import org.springframework.beans.BeanUtils; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Objects; public class DomainConvert { @@ -26,6 +27,9 @@ public class DomainConvert { } public static DomainResp convert(DomainDO domainDO, Map domainFullPathMap) { + if (Objects.isNull(domainDO)) { + return null; + } DomainResp domainResp = new DomainResp(); BeanUtils.copyProperties(domainDO, domainResp); domainResp.setFullPath(domainFullPathMap.get(domainDO.getId())); diff --git a/launchers/standalone/src/main/resources/hanlp.properties b/launchers/standalone/src/main/resources/hanlp.properties index 50c40ca8f..c9aed76c2 100644 --- a/launchers/standalone/src/main/resources/hanlp.properties +++ b/launchers/standalone/src/main/resources/hanlp.properties @@ -1,2 +1,2 @@ root=. -CustomDictionaryPath=data/dictionary/custom/DimValue_1_1.txt;data/dictionary/custom/DimValue_1_2.txt; +CustomDictionaryPath=data/dictionary/custom/dic_value_1_DIMENSION_1.txt