From 1004f71ba44998e9f29f2b5416cbff3e1912997f Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Thu, 1 Feb 2024 22:32:48 +0800 Subject: [PATCH] [improvement][Chat] MetricRatio of Chat has been migrated to the processor, Rule and LLM queries now support unified ratio queries. (#711) --- .../api/pojo/request/DimensionValueReq.java | 1 - .../query/rule/metric/MetricFilterQuery.java | 4 +- .../query/rule/metric/MetricModelQuery.java | 7 +- .../rule/metric/MetricSemanticQuery.java | 225 +----------------- .../query/rule/metric/MetricTagQuery.java | 4 +- .../execute/MetricRatioProcessor.java | 219 +++++++++++++++++ .../main/resources/META-INF/spring.factories | 5 +- .../tencent/supersonic/ChatDemoLoader.java | 8 +- .../main/resources/META-INF/spring.factories | 3 +- 9 files changed, 245 insertions(+), 231 deletions(-) create mode 100644 chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java index c8934be89..77b50a8c6 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DimensionValueReq.java @@ -11,7 +11,6 @@ public class DimensionValueReq { @NotNull private Long elementID; - @NotNull private Long modelId; private String bizName; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java index 80b1c54ee..bf7b881a2 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java @@ -38,9 +38,7 @@ public class MetricFilterQuery extends MetricSemanticQuery { @Override public QueryResult execute(User user) { if (!isMultiStructQuery()) { - QueryResult queryResult = super.execute(user); - fillAggregateInfo(user, queryResult); - return queryResult; + return super.execute(user); } return super.multiStructExecute(user); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java index 1845fad98..fe6be3b99 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.chat.core.query.rule.metric; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; +import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import org.springframework.stereotype.Component; - -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL; -import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST; @Component public class MetricModelQuery extends MetricSemanticQuery { @@ -25,7 +25,6 @@ public class MetricModelQuery extends MetricSemanticQuery { @Override public QueryResult execute(User user) { QueryResult queryResult = super.execute(user); - fillAggregateInfo(user, queryResult); return queryResult; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java index 74e07251d..4aa69ac4d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java @@ -1,61 +1,21 @@ package com.tencent.supersonic.chat.core.query.rule.metric; -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.api.pojo.SchemaElement; -import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.ViewSchema; -import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; -import com.tencent.supersonic.chat.api.pojo.response.MetricInfo; -import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.core.config.AggregatorConfig; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; -import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; -import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.DateConf.DateMode; -import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.RatioOverType; -import com.tencent.supersonic.common.pojo.enums.TimeMode; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.common.util.DateUtils; -import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; - -import java.text.DecimalFormat; -import java.time.DayOfWeek; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.YearMonth; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED; import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; -import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; + +import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.chat.api.pojo.ViewSchema; +import com.tencent.supersonic.chat.core.pojo.ChatContext; +import com.tencent.supersonic.chat.core.pojo.QueryContext; +import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.enums.TimeMode; +import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import java.time.LocalDate; +import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class MetricSemanticQuery extends RuleSemanticQuery { @@ -99,163 +59,4 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { parseInfo.setDateInfo(dateInfo); } } - - public void fillAggregateInfo(User user, QueryResult queryResult) { - if (Objects.nonNull(queryResult)) { - SemanticQueryResp queryResp = new SemanticQueryResp(); - queryResp.setColumns(queryResult.getQueryColumns()); - queryResp.setResultList(queryResult.getQueryResults()); - AggregateInfo aggregateInfo = getAggregateInfo(user, parseInfo, queryResp); - queryResult.setAggregateInfo(aggregateInfo); - } - } - - public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, - SemanticQueryResp result) { - AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class); - - if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio()) { - return new AggregateInfo(); - } - List resultMetricNames = result.getColumns().stream().map(c -> c.getNameEn()) - .collect(Collectors.toList()); - Optional ratioMetric = semanticParseInfo.getMetrics().stream() - .filter(m -> resultMetricNames.contains(m.getBizName())).findFirst(); - if (ratioMetric.isPresent()) { - AggregateInfo aggregateInfo = new AggregateInfo(); - MetricInfo metricInfo = new MetricInfo(); - metricInfo.setStatistics(new HashMap<>()); - try { - String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); - - Optional lastDayOp = result.getResultList().stream().filter(r -> r.containsKey(dateField)) - .map(r -> r.get(dateField).toString()) - .sorted(Comparator.reverseOrder()).findFirst(); - if (!lastDayOp.isPresent()) { - return new AggregateInfo(); - } - Optional> lastValue = result.getResultList().stream() - .filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst(); - if (lastValue.isPresent() && lastValue.get().containsKey(ratioMetric.get().getBizName())) { - DecimalFormat df = new DecimalFormat("#.####"); - metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName()))); - } - metricInfo.setDate(lastValue.get().get(dateField).toString()); - - CompletableFuture metricInfoRoll = CompletableFuture - .supplyAsync(() -> { - return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL, - result); - }); - CompletableFuture metricInfoOver = CompletableFuture - .supplyAsync(() -> { - return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER, - result); - }); - CompletableFuture.allOf(metricInfoRoll, metricInfoOver); - metricInfo.setName(metricInfoRoll.get().getName()); - metricInfo.setValue(metricInfoRoll.get().getValue()); - metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics()); - metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics()); - aggregateInfo.getMetricInfos().add(metricInfo); - } catch (Exception e) { - log.error("queryRatio error {}", e); - } - return aggregateInfo; - } - return new AggregateInfo(); - } - - private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, - AggOperatorEnum aggOperatorEnum, SemanticQueryResp results) { - MetricInfo metricInfo = new MetricInfo(); - metricInfo.setStatistics(new HashMap<>()); - QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum); - DateConf dateInfo = semanticParseInfo.getDateInfo(); - String dateField = QueryReqBuilder.getDateField(dateInfo); - - queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField))); - queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, results)); - - SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); - - if (Objects.nonNull(queryResp) && !CollectionUtils.isEmpty(queryResp.getResultList())) { - - Map result = queryResp.getResultList().get(0); - Optional valueColumn = queryResp.getColumns().stream() - .filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst(); - if (valueColumn.isPresent()) { - - String valueField = String.format("%s_%s", valueColumn.get().getNameEn(), - aggOperatorEnum.getOperator()); - if (result.containsKey(valueColumn.get().getNameEn())) { - DecimalFormat df = new DecimalFormat("#.####"); - metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn()))); - } - String ratio = ""; - if (Objects.nonNull(result.get(valueField))) { - ratio = String.format("%.2f", - (Double.valueOf(result.get(valueField).toString()) * 100)) + "%"; - } - String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName(); - String statisticsOverName = RatioOverType.WEEK_ON_DAY.getShowName(); - if (MONTH.equals(dateInfo.getPeriod())) { - statisticsRollName = RatioOverType.MONTH_ON_MONTH.getShowName(); - statisticsOverName = RatioOverType.YEAR_ON_MONTH.getShowName(); - } - if (WEEK.equals(dateInfo.getPeriod())) { - statisticsRollName = RatioOverType.WEEK_ON_WEEK.getShowName(); - statisticsOverName = RatioOverType.MONTH_ON_WEEK.getShowName(); - } - metricInfo.getStatistics().put(aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? statisticsRollName - : statisticsOverName, - ratio); - } - metricInfo.setName(metric.getName()); - } - return metricInfo; - } - - private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo, - SemanticQueryResp results) { - String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); - Optional lastDayOp = results.getResultList().stream() - .map(r -> r.get(dateField).toString()) - .sorted(Comparator.reverseOrder()).findFirst(); - if (lastDayOp.isPresent()) { - String lastDay = lastDayOp.get(); - DateConf dateConf = new DateConf(); - dateConf.setPeriod(semanticParseInfo.getDateInfo().getPeriod()); - dateConf.setDateMode(DateMode.LIST); - List dayList = new ArrayList<>(); - dayList.add(lastDay); - String start = ""; - if (DAY.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { - DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay, - new String[]{DAY_FORMAT, DAY_FORMAT_INT}); - LocalDate end = LocalDate.parse(lastDay, formatter); - start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusDays(1).format(formatter) - : end.minusWeeks(1).format(formatter); - } - if (WEEK.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { - DateTimeFormatter formatter = DateUtils.getTimeFormatter(lastDay, - new String[]{TIMES_FORMAT, DAY_FORMAT, TIME_FORMAT, DAY_FORMAT_INT}); - LocalDateTime end = LocalDateTime.parse(lastDay, formatter); - start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusWeeks(1).format(formatter) - : end.minusMonths(1).with(DayOfWeek.MONDAY).format(formatter); - } - if (MONTH.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { - DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay, - new String[]{MONTH_FORMAT, MONTH_FORMAT_INT}); - YearMonth end = YearMonth.parse(lastDay, formatter); - start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusMonths(1).format(formatter) - : end.minusYears(1).format(formatter); - } - dayList.add(start); - dateConf.setDateList(dayList); - return dateConf; - - } - return semanticParseInfo.getDateInfo(); - } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java index 09c8939ec..c07ace227 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java @@ -40,9 +40,7 @@ public class MetricTagQuery extends MetricSemanticQuery { @Override public QueryResult execute(User user) { if (!isMultiStructQuery()) { - QueryResult queryResult = super.execute(user); - fillAggregateInfo(user, queryResult); - return queryResult; + return super.execute(user); } return super.multiStructExecute(user); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java new file mode 100644 index 000000000..408c4a561 --- /dev/null +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java @@ -0,0 +1,219 @@ +package com.tencent.supersonic.chat.server.processor.execute; + +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT; +import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.SchemaElement; +import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo; +import com.tencent.supersonic.chat.api.pojo.response.MetricInfo; +import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.core.config.AggregatorConfig; +import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter; +import com.tencent.supersonic.chat.core.utils.ComponentFactory; +import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.DateConf.DateMode; +import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.pojo.enums.RatioOverType; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.common.util.DateUtils; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import java.text.DecimalFormat; +import java.time.DayOfWeek; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.YearMonth; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + +/** + * Add ratio queries for metric queries. + */ +@Slf4j +public class MetricRatioProcessor implements ExecuteResultProcessor { + + private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer(); + + @Override + public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) { + + AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class); + if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) + || !aggregatorConfig.getEnableRatio() + || !QueryType.METRIC.equals(semanticParseInfo.getQueryType())) { + return; + } + AggregateInfo aggregateInfo = getAggregateInfo(queryReq.getUser(), semanticParseInfo, queryResult); + queryResult.setAggregateInfo(aggregateInfo); + } + + public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) { + + Set resultMetricNames = queryResult.getQueryColumns() + .stream().map(c -> c.getNameEn()).collect(Collectors.toSet()); + + Optional ratioMetric = semanticParseInfo.getMetrics().stream() + .filter(m -> resultMetricNames.contains(m.getBizName())).findFirst(); + + AggregateInfo aggregateInfo = new AggregateInfo(); + if (!ratioMetric.isPresent()) { + return aggregateInfo; + } + + try { + String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); + Optional lastDayOp = queryResult.getQueryResults().stream() + .filter(r -> r.containsKey(dateField)) + .map(r -> r.get(dateField).toString()) + .sorted(Comparator.reverseOrder()).findFirst(); + + if (!lastDayOp.isPresent()) { + return new AggregateInfo(); + } + Optional> lastValue = queryResult.getQueryResults().stream() + .filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst(); + + MetricInfo metricInfo = new MetricInfo(); + metricInfo.setStatistics(new HashMap<>()); + if (lastValue.isPresent() && lastValue.get().containsKey(ratioMetric.get().getBizName())) { + DecimalFormat df = new DecimalFormat("#.####"); + metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName()))); + } + metricInfo.setDate(lastValue.get().get(dateField).toString()); + + CompletableFuture metricInfoRoll = CompletableFuture.supplyAsync( + () -> queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL, + queryResult)); + CompletableFuture metricInfoOver = CompletableFuture.supplyAsync( + () -> queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER, + queryResult)); + CompletableFuture.allOf(metricInfoRoll, metricInfoOver); + metricInfo.setName(metricInfoRoll.get().getName()); + metricInfo.setValue(metricInfoRoll.get().getValue()); + metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics()); + metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics()); + aggregateInfo.getMetricInfos().add(metricInfo); + } catch (Exception e) { + log.error("queryRatio error {}", e); + } + return aggregateInfo; + } + + private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, + AggOperatorEnum aggOperatorEnum, QueryResult queryResult) { + + QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum); + String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); + queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField))); + queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult)); + + SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); + MetricInfo metricInfo = new MetricInfo(); + metricInfo.setStatistics(new HashMap<>()); + if (Objects.isNull(queryResp) || CollectionUtils.isEmpty(queryResp.getResultList())) { + return metricInfo; + } + + Map result = queryResp.getResultList().get(0); + Optional valueColumn = queryResp.getColumns().stream() + .filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst(); + + if (!valueColumn.isPresent()) { + return metricInfo; + } + String valueField = String.format("%s_%s", valueColumn.get().getNameEn(), aggOperatorEnum.getOperator()); + if (result.containsKey(valueColumn.get().getNameEn())) { + DecimalFormat df = new DecimalFormat("#.####"); + metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn()))); + } + String ratio = ""; + if (Objects.nonNull(result.get(valueField))) { + ratio = String.format("%.2f", + (Double.valueOf(result.get(valueField).toString()) * 100)) + "%"; + } + String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName(); + String statisticsOverName = RatioOverType.WEEK_ON_DAY.getShowName(); + if (MONTH.equals(semanticParseInfo.getDateInfo().getPeriod())) { + statisticsRollName = RatioOverType.MONTH_ON_MONTH.getShowName(); + statisticsOverName = RatioOverType.YEAR_ON_MONTH.getShowName(); + } + if (WEEK.equals(semanticParseInfo.getDateInfo().getPeriod())) { + statisticsRollName = RatioOverType.WEEK_ON_WEEK.getShowName(); + statisticsOverName = RatioOverType.MONTH_ON_WEEK.getShowName(); + } + metricInfo.getStatistics().put(aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) + ? statisticsRollName : statisticsOverName, ratio); + metricInfo.setName(metric.getName()); + return metricInfo; + } + + private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo, + QueryResult queryResult) { + String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); + + Optional lastDayOp = queryResult.getQueryResults() + .stream().map(r -> r.get(dateField).toString()) + .sorted(Comparator.reverseOrder()).findFirst(); + + if (!lastDayOp.isPresent()) { + return semanticParseInfo.getDateInfo(); + } + String lastDay = lastDayOp.get(); + DateConf dateConf = new DateConf(); + dateConf.setPeriod(semanticParseInfo.getDateInfo().getPeriod()); + dateConf.setDateMode(DateMode.LIST); + List dayList = new ArrayList<>(); + dayList.add(lastDay); + String start = ""; + if (DAY.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { + DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay, + new String[]{DAY_FORMAT, DAY_FORMAT_INT}); + LocalDate end = LocalDate.parse(lastDay, formatter); + start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusDays(1).format(formatter) + : end.minusWeeks(1).format(formatter); + } + if (WEEK.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { + DateTimeFormatter formatter = DateUtils.getTimeFormatter(lastDay, + new String[]{TIMES_FORMAT, DAY_FORMAT, TIME_FORMAT, DAY_FORMAT_INT}); + LocalDateTime end = LocalDateTime.parse(lastDay, formatter); + start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusWeeks(1).format(formatter) + : end.minusMonths(1).with(DayOfWeek.MONDAY).format(formatter); + } + if (MONTH.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) { + DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay, + new String[]{MONTH_FORMAT, MONTH_FORMAT_INT}); + YearMonth end = YearMonth.parse(lastDay, formatter); + start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusMonths(1).format(formatter) + : end.minusYears(1).format(formatter); + } + dayList.add(start); + dateConf.setDateList(dayList); + return dateConf; + } + +} diff --git a/launchers/chat/src/main/resources/META-INF/spring.factories b/launchers/chat/src/main/resources/META-INF/spring.factories index a0a46b0ec..7f26866e9 100644 --- a/launchers/chat/src/main/resources/META-INF/spring.factories +++ b/launchers/chat/src/main/resources/META-INF/spring.factories @@ -41,7 +41,10 @@ com.tencent.supersonic.chat.postprocessor.PostProcessor=\ com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\ - com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor + com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\ + com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\ + com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor + com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\ com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore \ No newline at end of file diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java index 7ca3f6d3f..4cf2a4ebe 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java @@ -17,13 +17,14 @@ import com.tencent.supersonic.chat.core.query.plugin.ParamOption; import com.tencent.supersonic.chat.core.query.plugin.WebBase; import com.tencent.supersonic.chat.server.service.AgentService; import com.tencent.supersonic.chat.server.service.ChatService; -import com.tencent.supersonic.chat.server.service.ConfigService; import com.tencent.supersonic.chat.server.service.PluginService; import com.tencent.supersonic.chat.server.service.QueryService; import com.tencent.supersonic.common.pojo.SysParameter; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.service.SysParameterService; import com.tencent.supersonic.common.util.JsonUtil; +import java.util.Arrays; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -32,9 +33,6 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import java.util.Arrays; -import java.util.List; - @Component @Slf4j @Order(3) @@ -47,8 +45,6 @@ public class ChatDemoLoader implements CommandLineRunner { @Autowired private ChatService chatService; @Autowired - private ConfigService configService; - @Autowired private PluginService pluginService; @Autowired private AgentService agentService; diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index c9f4240c3..da99d479d 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -43,7 +43,8 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\ com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\ com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\ - com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor + com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\ + com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\ com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore \ No newline at end of file