mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-12 20:51:48 +00:00
[improvement][Chat] MetricRatio of Chat has been migrated to the processor, Rule and LLM queries now support unified ratio queries. (#711)
This commit is contained in:
@@ -11,7 +11,6 @@ public class DimensionValueReq {
|
|||||||
@NotNull
|
@NotNull
|
||||||
private Long elementID;
|
private Long elementID;
|
||||||
|
|
||||||
@NotNull
|
|
||||||
private Long modelId;
|
private Long modelId;
|
||||||
|
|
||||||
private String bizName;
|
private String bizName;
|
||||||
|
|||||||
@@ -38,9 +38,7 @@ public class MetricFilterQuery extends MetricSemanticQuery {
|
|||||||
@Override
|
@Override
|
||||||
public QueryResult execute(User user) {
|
public QueryResult execute(User user) {
|
||||||
if (!isMultiStructQuery()) {
|
if (!isMultiStructQuery()) {
|
||||||
QueryResult queryResult = super.execute(user);
|
return super.execute(user);
|
||||||
fillAggregateInfo(user, queryResult);
|
|
||||||
return queryResult;
|
|
||||||
}
|
}
|
||||||
return super.multiStructExecute(user);
|
return super.multiStructExecute(user);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
package com.tencent.supersonic.chat.core.query.rule.metric;
|
package com.tencent.supersonic.chat.core.query.rule.metric;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL;
|
||||||
|
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL;
|
|
||||||
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST;
|
|
||||||
@Component
|
@Component
|
||||||
public class MetricModelQuery extends MetricSemanticQuery {
|
public class MetricModelQuery extends MetricSemanticQuery {
|
||||||
|
|
||||||
@@ -25,7 +25,6 @@ public class MetricModelQuery extends MetricSemanticQuery {
|
|||||||
@Override
|
@Override
|
||||||
public QueryResult execute(User user) {
|
public QueryResult execute(User user) {
|
||||||
QueryResult queryResult = super.execute(user);
|
QueryResult queryResult = super.execute(user);
|
||||||
fillAggregateInfo(user, queryResult);
|
|
||||||
return queryResult;
|
return queryResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,61 +1,21 @@
|
|||||||
package com.tencent.supersonic.chat.core.query.rule.metric;
|
package com.tencent.supersonic.chat.core.query.rule.metric;
|
||||||
|
|
||||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.MetricInfo;
|
|
||||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
|
||||||
import com.tencent.supersonic.chat.core.config.AggregatorConfig;
|
|
||||||
import com.tencent.supersonic.chat.core.pojo.ChatContext;
|
|
||||||
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
|
||||||
import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery;
|
|
||||||
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
|
|
||||||
import com.tencent.supersonic.common.pojo.DateConf;
|
|
||||||
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
|
|
||||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.RatioOverType;
|
|
||||||
import com.tencent.supersonic.common.pojo.enums.TimeMode;
|
|
||||||
import com.tencent.supersonic.common.util.ContextUtils;
|
|
||||||
import com.tencent.supersonic.common.util.DateUtils;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.text.DecimalFormat;
|
|
||||||
import java.time.DayOfWeek;
|
|
||||||
import java.time.LocalDate;
|
|
||||||
import java.time.LocalDateTime;
|
|
||||||
import java.time.YearMonth;
|
|
||||||
import java.time.format.DateTimeFormatter;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.concurrent.CompletableFuture;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
|
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
|
||||||
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
|
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
|
||||||
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
|
import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.DAY;
|
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
|
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT;
|
import com.tencent.supersonic.chat.api.pojo.ViewSchema;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
|
import com.tencent.supersonic.chat.core.pojo.ChatContext;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
|
import com.tencent.supersonic.chat.core.pojo.QueryContext;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT;
|
import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT;
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT;
|
import com.tencent.supersonic.common.pojo.enums.TimeMode;
|
||||||
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
|
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
public abstract class MetricSemanticQuery extends RuleSemanticQuery {
|
public abstract class MetricSemanticQuery extends RuleSemanticQuery {
|
||||||
@@ -99,163 +59,4 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
|
|||||||
parseInfo.setDateInfo(dateInfo);
|
parseInfo.setDateInfo(dateInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void fillAggregateInfo(User user, QueryResult queryResult) {
|
|
||||||
if (Objects.nonNull(queryResult)) {
|
|
||||||
SemanticQueryResp queryResp = new SemanticQueryResp();
|
|
||||||
queryResp.setColumns(queryResult.getQueryColumns());
|
|
||||||
queryResp.setResultList(queryResult.getQueryResults());
|
|
||||||
AggregateInfo aggregateInfo = getAggregateInfo(user, parseInfo, queryResp);
|
|
||||||
queryResult.setAggregateInfo(aggregateInfo);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo,
|
|
||||||
SemanticQueryResp result) {
|
|
||||||
AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class);
|
|
||||||
|
|
||||||
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio()) {
|
|
||||||
return new AggregateInfo();
|
|
||||||
}
|
|
||||||
List<String> resultMetricNames = result.getColumns().stream().map(c -> c.getNameEn())
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
Optional<SchemaElement> ratioMetric = semanticParseInfo.getMetrics().stream()
|
|
||||||
.filter(m -> resultMetricNames.contains(m.getBizName())).findFirst();
|
|
||||||
if (ratioMetric.isPresent()) {
|
|
||||||
AggregateInfo aggregateInfo = new AggregateInfo();
|
|
||||||
MetricInfo metricInfo = new MetricInfo();
|
|
||||||
metricInfo.setStatistics(new HashMap<>());
|
|
||||||
try {
|
|
||||||
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
|
|
||||||
|
|
||||||
Optional<String> lastDayOp = result.getResultList().stream().filter(r -> r.containsKey(dateField))
|
|
||||||
.map(r -> r.get(dateField).toString())
|
|
||||||
.sorted(Comparator.reverseOrder()).findFirst();
|
|
||||||
if (!lastDayOp.isPresent()) {
|
|
||||||
return new AggregateInfo();
|
|
||||||
}
|
|
||||||
Optional<Map<String, Object>> lastValue = result.getResultList().stream()
|
|
||||||
.filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst();
|
|
||||||
if (lastValue.isPresent() && lastValue.get().containsKey(ratioMetric.get().getBizName())) {
|
|
||||||
DecimalFormat df = new DecimalFormat("#.####");
|
|
||||||
metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName())));
|
|
||||||
}
|
|
||||||
metricInfo.setDate(lastValue.get().get(dateField).toString());
|
|
||||||
|
|
||||||
CompletableFuture<MetricInfo> metricInfoRoll = CompletableFuture
|
|
||||||
.supplyAsync(() -> {
|
|
||||||
return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL,
|
|
||||||
result);
|
|
||||||
});
|
|
||||||
CompletableFuture<MetricInfo> metricInfoOver = CompletableFuture
|
|
||||||
.supplyAsync(() -> {
|
|
||||||
return queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER,
|
|
||||||
result);
|
|
||||||
});
|
|
||||||
CompletableFuture.allOf(metricInfoRoll, metricInfoOver);
|
|
||||||
metricInfo.setName(metricInfoRoll.get().getName());
|
|
||||||
metricInfo.setValue(metricInfoRoll.get().getValue());
|
|
||||||
metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics());
|
|
||||||
metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics());
|
|
||||||
aggregateInfo.getMetricInfos().add(metricInfo);
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("queryRatio error {}", e);
|
|
||||||
}
|
|
||||||
return aggregateInfo;
|
|
||||||
}
|
|
||||||
return new AggregateInfo();
|
|
||||||
}
|
|
||||||
|
|
||||||
private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric,
|
|
||||||
AggOperatorEnum aggOperatorEnum, SemanticQueryResp results) {
|
|
||||||
MetricInfo metricInfo = new MetricInfo();
|
|
||||||
metricInfo.setStatistics(new HashMap<>());
|
|
||||||
QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum);
|
|
||||||
DateConf dateInfo = semanticParseInfo.getDateInfo();
|
|
||||||
String dateField = QueryReqBuilder.getDateField(dateInfo);
|
|
||||||
|
|
||||||
queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField)));
|
|
||||||
queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, results));
|
|
||||||
|
|
||||||
SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
|
|
||||||
|
|
||||||
if (Objects.nonNull(queryResp) && !CollectionUtils.isEmpty(queryResp.getResultList())) {
|
|
||||||
|
|
||||||
Map<String, Object> result = queryResp.getResultList().get(0);
|
|
||||||
Optional<QueryColumn> valueColumn = queryResp.getColumns().stream()
|
|
||||||
.filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst();
|
|
||||||
if (valueColumn.isPresent()) {
|
|
||||||
|
|
||||||
String valueField = String.format("%s_%s", valueColumn.get().getNameEn(),
|
|
||||||
aggOperatorEnum.getOperator());
|
|
||||||
if (result.containsKey(valueColumn.get().getNameEn())) {
|
|
||||||
DecimalFormat df = new DecimalFormat("#.####");
|
|
||||||
metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn())));
|
|
||||||
}
|
|
||||||
String ratio = "";
|
|
||||||
if (Objects.nonNull(result.get(valueField))) {
|
|
||||||
ratio = String.format("%.2f",
|
|
||||||
(Double.valueOf(result.get(valueField).toString()) * 100)) + "%";
|
|
||||||
}
|
|
||||||
String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName();
|
|
||||||
String statisticsOverName = RatioOverType.WEEK_ON_DAY.getShowName();
|
|
||||||
if (MONTH.equals(dateInfo.getPeriod())) {
|
|
||||||
statisticsRollName = RatioOverType.MONTH_ON_MONTH.getShowName();
|
|
||||||
statisticsOverName = RatioOverType.YEAR_ON_MONTH.getShowName();
|
|
||||||
}
|
|
||||||
if (WEEK.equals(dateInfo.getPeriod())) {
|
|
||||||
statisticsRollName = RatioOverType.WEEK_ON_WEEK.getShowName();
|
|
||||||
statisticsOverName = RatioOverType.MONTH_ON_WEEK.getShowName();
|
|
||||||
}
|
|
||||||
metricInfo.getStatistics().put(aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? statisticsRollName
|
|
||||||
: statisticsOverName,
|
|
||||||
ratio);
|
|
||||||
}
|
|
||||||
metricInfo.setName(metric.getName());
|
|
||||||
}
|
|
||||||
return metricInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo,
|
|
||||||
SemanticQueryResp results) {
|
|
||||||
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
|
|
||||||
Optional<String> lastDayOp = results.getResultList().stream()
|
|
||||||
.map(r -> r.get(dateField).toString())
|
|
||||||
.sorted(Comparator.reverseOrder()).findFirst();
|
|
||||||
if (lastDayOp.isPresent()) {
|
|
||||||
String lastDay = lastDayOp.get();
|
|
||||||
DateConf dateConf = new DateConf();
|
|
||||||
dateConf.setPeriod(semanticParseInfo.getDateInfo().getPeriod());
|
|
||||||
dateConf.setDateMode(DateMode.LIST);
|
|
||||||
List<String> dayList = new ArrayList<>();
|
|
||||||
dayList.add(lastDay);
|
|
||||||
String start = "";
|
|
||||||
if (DAY.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) {
|
|
||||||
DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay,
|
|
||||||
new String[]{DAY_FORMAT, DAY_FORMAT_INT});
|
|
||||||
LocalDate end = LocalDate.parse(lastDay, formatter);
|
|
||||||
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusDays(1).format(formatter)
|
|
||||||
: end.minusWeeks(1).format(formatter);
|
|
||||||
}
|
|
||||||
if (WEEK.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) {
|
|
||||||
DateTimeFormatter formatter = DateUtils.getTimeFormatter(lastDay,
|
|
||||||
new String[]{TIMES_FORMAT, DAY_FORMAT, TIME_FORMAT, DAY_FORMAT_INT});
|
|
||||||
LocalDateTime end = LocalDateTime.parse(lastDay, formatter);
|
|
||||||
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusWeeks(1).format(formatter)
|
|
||||||
: end.minusMonths(1).with(DayOfWeek.MONDAY).format(formatter);
|
|
||||||
}
|
|
||||||
if (MONTH.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) {
|
|
||||||
DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay,
|
|
||||||
new String[]{MONTH_FORMAT, MONTH_FORMAT_INT});
|
|
||||||
YearMonth end = YearMonth.parse(lastDay, formatter);
|
|
||||||
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusMonths(1).format(formatter)
|
|
||||||
: end.minusYears(1).format(formatter);
|
|
||||||
}
|
|
||||||
dayList.add(start);
|
|
||||||
dateConf.setDateList(dayList);
|
|
||||||
return dateConf;
|
|
||||||
|
|
||||||
}
|
|
||||||
return semanticParseInfo.getDateInfo();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,9 +40,7 @@ public class MetricTagQuery extends MetricSemanticQuery {
|
|||||||
@Override
|
@Override
|
||||||
public QueryResult execute(User user) {
|
public QueryResult execute(User user) {
|
||||||
if (!isMultiStructQuery()) {
|
if (!isMultiStructQuery()) {
|
||||||
QueryResult queryResult = super.execute(user);
|
return super.execute(user);
|
||||||
fillAggregateInfo(user, queryResult);
|
|
||||||
return queryResult;
|
|
||||||
}
|
}
|
||||||
return super.multiStructExecute(user);
|
return super.multiStructExecute(user);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
package com.tencent.supersonic.chat.server.processor.execute;
|
||||||
|
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.DAY;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT_INT;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT_INT;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.TIMES_FORMAT;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT;
|
||||||
|
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
|
||||||
|
|
||||||
|
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.response.MetricInfo;
|
||||||
|
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||||
|
import com.tencent.supersonic.chat.core.config.AggregatorConfig;
|
||||||
|
import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.ComponentFactory;
|
||||||
|
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
|
||||||
|
import com.tencent.supersonic.common.pojo.DateConf;
|
||||||
|
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
|
||||||
|
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||||
|
import com.tencent.supersonic.common.pojo.enums.RatioOverType;
|
||||||
|
import com.tencent.supersonic.common.util.ContextUtils;
|
||||||
|
import com.tencent.supersonic.common.util.DateUtils;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
|
||||||
|
import java.text.DecimalFormat;
|
||||||
|
import java.time.DayOfWeek;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.YearMonth;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add ratio queries for metric queries.
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
public class MetricRatioProcessor implements ExecuteResultProcessor {
|
||||||
|
|
||||||
|
private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void process(QueryResult queryResult, SemanticParseInfo semanticParseInfo, ExecuteQueryReq queryReq) {
|
||||||
|
|
||||||
|
AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class);
|
||||||
|
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics())
|
||||||
|
|| !aggregatorConfig.getEnableRatio()
|
||||||
|
|| !QueryType.METRIC.equals(semanticParseInfo.getQueryType())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
AggregateInfo aggregateInfo = getAggregateInfo(queryReq.getUser(), semanticParseInfo, queryResult);
|
||||||
|
queryResult.setAggregateInfo(aggregateInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, QueryResult queryResult) {
|
||||||
|
|
||||||
|
Set<String> resultMetricNames = queryResult.getQueryColumns()
|
||||||
|
.stream().map(c -> c.getNameEn()).collect(Collectors.toSet());
|
||||||
|
|
||||||
|
Optional<SchemaElement> ratioMetric = semanticParseInfo.getMetrics().stream()
|
||||||
|
.filter(m -> resultMetricNames.contains(m.getBizName())).findFirst();
|
||||||
|
|
||||||
|
AggregateInfo aggregateInfo = new AggregateInfo();
|
||||||
|
if (!ratioMetric.isPresent()) {
|
||||||
|
return aggregateInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
|
||||||
|
Optional<String> lastDayOp = queryResult.getQueryResults().stream()
|
||||||
|
.filter(r -> r.containsKey(dateField))
|
||||||
|
.map(r -> r.get(dateField).toString())
|
||||||
|
.sorted(Comparator.reverseOrder()).findFirst();
|
||||||
|
|
||||||
|
if (!lastDayOp.isPresent()) {
|
||||||
|
return new AggregateInfo();
|
||||||
|
}
|
||||||
|
Optional<Map<String, Object>> lastValue = queryResult.getQueryResults().stream()
|
||||||
|
.filter(r -> r.get(dateField).toString().equals(lastDayOp.get())).findFirst();
|
||||||
|
|
||||||
|
MetricInfo metricInfo = new MetricInfo();
|
||||||
|
metricInfo.setStatistics(new HashMap<>());
|
||||||
|
if (lastValue.isPresent() && lastValue.get().containsKey(ratioMetric.get().getBizName())) {
|
||||||
|
DecimalFormat df = new DecimalFormat("#.####");
|
||||||
|
metricInfo.setValue(df.format(lastValue.get().get(ratioMetric.get().getBizName())));
|
||||||
|
}
|
||||||
|
metricInfo.setDate(lastValue.get().get(dateField).toString());
|
||||||
|
|
||||||
|
CompletableFuture<MetricInfo> metricInfoRoll = CompletableFuture.supplyAsync(
|
||||||
|
() -> queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_ROLL,
|
||||||
|
queryResult));
|
||||||
|
CompletableFuture<MetricInfo> metricInfoOver = CompletableFuture.supplyAsync(
|
||||||
|
() -> queryRatio(user, semanticParseInfo, ratioMetric.get(), AggOperatorEnum.RATIO_OVER,
|
||||||
|
queryResult));
|
||||||
|
CompletableFuture.allOf(metricInfoRoll, metricInfoOver);
|
||||||
|
metricInfo.setName(metricInfoRoll.get().getName());
|
||||||
|
metricInfo.setValue(metricInfoRoll.get().getValue());
|
||||||
|
metricInfo.getStatistics().putAll(metricInfoRoll.get().getStatistics());
|
||||||
|
metricInfo.getStatistics().putAll(metricInfoOver.get().getStatistics());
|
||||||
|
aggregateInfo.getMetricInfos().add(metricInfo);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("queryRatio error {}", e);
|
||||||
|
}
|
||||||
|
return aggregateInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric,
|
||||||
|
AggOperatorEnum aggOperatorEnum, QueryResult queryResult) {
|
||||||
|
|
||||||
|
QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum);
|
||||||
|
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
|
||||||
|
queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField)));
|
||||||
|
queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, queryResult));
|
||||||
|
|
||||||
|
SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
|
||||||
|
MetricInfo metricInfo = new MetricInfo();
|
||||||
|
metricInfo.setStatistics(new HashMap<>());
|
||||||
|
if (Objects.isNull(queryResp) || CollectionUtils.isEmpty(queryResp.getResultList())) {
|
||||||
|
return metricInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, Object> result = queryResp.getResultList().get(0);
|
||||||
|
Optional<QueryColumn> valueColumn = queryResp.getColumns().stream()
|
||||||
|
.filter(c -> c.getNameEn().equals(metric.getBizName())).findFirst();
|
||||||
|
|
||||||
|
if (!valueColumn.isPresent()) {
|
||||||
|
return metricInfo;
|
||||||
|
}
|
||||||
|
String valueField = String.format("%s_%s", valueColumn.get().getNameEn(), aggOperatorEnum.getOperator());
|
||||||
|
if (result.containsKey(valueColumn.get().getNameEn())) {
|
||||||
|
DecimalFormat df = new DecimalFormat("#.####");
|
||||||
|
metricInfo.setValue(df.format(result.get(valueColumn.get().getNameEn())));
|
||||||
|
}
|
||||||
|
String ratio = "";
|
||||||
|
if (Objects.nonNull(result.get(valueField))) {
|
||||||
|
ratio = String.format("%.2f",
|
||||||
|
(Double.valueOf(result.get(valueField).toString()) * 100)) + "%";
|
||||||
|
}
|
||||||
|
String statisticsRollName = RatioOverType.DAY_ON_DAY.getShowName();
|
||||||
|
String statisticsOverName = RatioOverType.WEEK_ON_DAY.getShowName();
|
||||||
|
if (MONTH.equals(semanticParseInfo.getDateInfo().getPeriod())) {
|
||||||
|
statisticsRollName = RatioOverType.MONTH_ON_MONTH.getShowName();
|
||||||
|
statisticsOverName = RatioOverType.YEAR_ON_MONTH.getShowName();
|
||||||
|
}
|
||||||
|
if (WEEK.equals(semanticParseInfo.getDateInfo().getPeriod())) {
|
||||||
|
statisticsRollName = RatioOverType.WEEK_ON_WEEK.getShowName();
|
||||||
|
statisticsOverName = RatioOverType.MONTH_ON_WEEK.getShowName();
|
||||||
|
}
|
||||||
|
metricInfo.getStatistics().put(aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL)
|
||||||
|
? statisticsRollName : statisticsOverName, ratio);
|
||||||
|
metricInfo.setName(metric.getName());
|
||||||
|
return metricInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo,
|
||||||
|
QueryResult queryResult) {
|
||||||
|
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
|
||||||
|
|
||||||
|
Optional<String> lastDayOp = queryResult.getQueryResults()
|
||||||
|
.stream().map(r -> r.get(dateField).toString())
|
||||||
|
.sorted(Comparator.reverseOrder()).findFirst();
|
||||||
|
|
||||||
|
if (!lastDayOp.isPresent()) {
|
||||||
|
return semanticParseInfo.getDateInfo();
|
||||||
|
}
|
||||||
|
String lastDay = lastDayOp.get();
|
||||||
|
DateConf dateConf = new DateConf();
|
||||||
|
dateConf.setPeriod(semanticParseInfo.getDateInfo().getPeriod());
|
||||||
|
dateConf.setDateMode(DateMode.LIST);
|
||||||
|
List<String> dayList = new ArrayList<>();
|
||||||
|
dayList.add(lastDay);
|
||||||
|
String start = "";
|
||||||
|
if (DAY.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) {
|
||||||
|
DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay,
|
||||||
|
new String[]{DAY_FORMAT, DAY_FORMAT_INT});
|
||||||
|
LocalDate end = LocalDate.parse(lastDay, formatter);
|
||||||
|
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusDays(1).format(formatter)
|
||||||
|
: end.minusWeeks(1).format(formatter);
|
||||||
|
}
|
||||||
|
if (WEEK.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) {
|
||||||
|
DateTimeFormatter formatter = DateUtils.getTimeFormatter(lastDay,
|
||||||
|
new String[]{TIMES_FORMAT, DAY_FORMAT, TIME_FORMAT, DAY_FORMAT_INT});
|
||||||
|
LocalDateTime end = LocalDateTime.parse(lastDay, formatter);
|
||||||
|
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusWeeks(1).format(formatter)
|
||||||
|
: end.minusMonths(1).with(DayOfWeek.MONDAY).format(formatter);
|
||||||
|
}
|
||||||
|
if (MONTH.equalsIgnoreCase(semanticParseInfo.getDateInfo().getPeriod())) {
|
||||||
|
DateTimeFormatter formatter = DateUtils.getDateFormatter(lastDay,
|
||||||
|
new String[]{MONTH_FORMAT, MONTH_FORMAT_INT});
|
||||||
|
YearMonth end = YearMonth.parse(lastDay, formatter);
|
||||||
|
start = aggOperatorEnum.equals(AggOperatorEnum.RATIO_ROLL) ? end.minusMonths(1).format(formatter)
|
||||||
|
: end.minusYears(1).format(formatter);
|
||||||
|
}
|
||||||
|
dayList.add(start);
|
||||||
|
dateConf.setDateList(dayList);
|
||||||
|
return dateConf;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -41,7 +41,10 @@ com.tencent.supersonic.chat.postprocessor.PostProcessor=\
|
|||||||
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
|
com.tencent.supersonic.chat.postprocessor.RespBuildPostProcessor
|
||||||
|
|
||||||
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
|
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
|
||||||
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor
|
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
|
||||||
|
com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\
|
||||||
|
com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor
|
||||||
|
|
||||||
|
|
||||||
com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\
|
com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\
|
||||||
com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore
|
com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore
|
||||||
@@ -17,13 +17,14 @@ import com.tencent.supersonic.chat.core.query.plugin.ParamOption;
|
|||||||
import com.tencent.supersonic.chat.core.query.plugin.WebBase;
|
import com.tencent.supersonic.chat.core.query.plugin.WebBase;
|
||||||
import com.tencent.supersonic.chat.server.service.AgentService;
|
import com.tencent.supersonic.chat.server.service.AgentService;
|
||||||
import com.tencent.supersonic.chat.server.service.ChatService;
|
import com.tencent.supersonic.chat.server.service.ChatService;
|
||||||
import com.tencent.supersonic.chat.server.service.ConfigService;
|
|
||||||
import com.tencent.supersonic.chat.server.service.PluginService;
|
import com.tencent.supersonic.chat.server.service.PluginService;
|
||||||
import com.tencent.supersonic.chat.server.service.QueryService;
|
import com.tencent.supersonic.chat.server.service.QueryService;
|
||||||
import com.tencent.supersonic.common.pojo.SysParameter;
|
import com.tencent.supersonic.common.pojo.SysParameter;
|
||||||
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
import com.tencent.supersonic.common.pojo.enums.QueryType;
|
||||||
import com.tencent.supersonic.common.service.SysParameterService;
|
import com.tencent.supersonic.common.service.SysParameterService;
|
||||||
import com.tencent.supersonic.common.util.JsonUtil;
|
import com.tencent.supersonic.common.util.JsonUtil;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.beans.factory.annotation.Qualifier;
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
@@ -32,9 +33,6 @@ import org.springframework.boot.CommandLineRunner;
|
|||||||
import org.springframework.core.annotation.Order;
|
import org.springframework.core.annotation.Order;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Order(3)
|
@Order(3)
|
||||||
@@ -47,8 +45,6 @@ public class ChatDemoLoader implements CommandLineRunner {
|
|||||||
@Autowired
|
@Autowired
|
||||||
private ChatService chatService;
|
private ChatService chatService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private ConfigService configService;
|
|
||||||
@Autowired
|
|
||||||
private PluginService pluginService;
|
private PluginService pluginService;
|
||||||
@Autowired
|
@Autowired
|
||||||
private AgentService agentService;
|
private AgentService agentService;
|
||||||
|
|||||||
@@ -43,7 +43,8 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
|
|||||||
|
|
||||||
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
|
com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor=\
|
||||||
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
|
com.tencent.supersonic.chat.server.processor.execute.MetricRecommendProcessor,\
|
||||||
com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor
|
com.tencent.supersonic.chat.server.processor.execute.DimensionRecommendProcessor,\
|
||||||
|
com.tencent.supersonic.chat.server.processor.execute.MetricRatioProcessor
|
||||||
|
|
||||||
com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\
|
com.tencent.supersonic.common.util.embedding.S2EmbeddingStore=\
|
||||||
com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore
|
com.tencent.supersonic.common.util.embedding.InMemoryS2EmbeddingStore
|
||||||
Reference in New Issue
Block a user