From a200483b5cde874b5794bc5372e1e3e3658ffd74 Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Fri, 20 Sep 2024 18:35:25 +0800 Subject: [PATCH] Support limit (#1688) --- .../supersonic/common/pojo/Constants.java | 3 +++ .../api/pojo/MetricTypeDefaultConfig.java | 2 ++ .../headless/api/pojo/SemanticParseInfo.java | 25 +++++++++++++++++++ .../headless/api/pojo/SemanticSchema.java | 15 +++++++++++ .../api/pojo/TagTypeDefaultConfig.java | 2 ++ .../chat/parser/llm/LLMResponseService.java | 2 ++ .../chat/query/rule/RuleSemanticQuery.java | 1 + .../rule/detail/DetailSemanticQuery.java | 6 ++--- .../rule/metric/MetricSemanticQuery.java | 6 ++--- .../query/rule/metric/MetricTopNQuery.java | 2 -- .../service/impl/S2ChatLayerService.java | 2 +- 11 files changed, 57 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java index 9f551110d..7b6757d47 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java @@ -66,4 +66,7 @@ public class Constants { public static final Long DEFAULT_FREQUENCY = 100000L; public static final String TABLE_PREFIX = "t_"; + + public static final Long DEFAULT_DETAIL_LIMIT = 500L; + public static final Long DEFAULT_METRIC_LIMIT = 200L; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java index e5eac0cff..ed554ea09 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTypeDefaultConfig.java @@ -9,4 +9,6 @@ public class MetricTypeDefaultConfig { private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(7, Constants.DAY, TimeMode.RECENT); + + private Long limit; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java index db0edb029..51434b3e8 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java @@ -14,14 +14,19 @@ import lombok.Data; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import static com.tencent.supersonic.common.pojo.Constants.DEFAULT_DETAIL_LIMIT; +import static com.tencent.supersonic.common.pojo.Constants.DEFAULT_METRIC_LIMIT; + @Data public class SemanticParseInfo { private Integer id; private String queryMode = "PLAIN_TEXT"; private SchemaElement dataSet; + private QueryConfig queryConfig; private Set metrics = Sets.newTreeSet(new SchemaNameLengthComparator()); private Set dimensions = Sets.newTreeSet(new SchemaNameLengthComparator()); private SchemaElement entity; @@ -68,4 +73,24 @@ public class SemanticParseInfo { } return dataSet.getDataSetId(); } + + public Long getDetailLimit() { + Long limit = DEFAULT_DETAIL_LIMIT; + if (Objects.nonNull(queryConfig) + && Objects.nonNull(queryConfig.getTagTypeDefaultConfig()) + && Objects.nonNull(queryConfig.getTagTypeDefaultConfig().getLimit())) { + limit = queryConfig.getTagTypeDefaultConfig().getLimit(); + } + return limit; + } + + public Long getMetricLimit() { + Long limit = DEFAULT_METRIC_LIMIT; + if (Objects.nonNull(queryConfig) + && Objects.nonNull(queryConfig.getMetricTypeDefaultConfig()) + && Objects.nonNull(queryConfig.getMetricTypeDefaultConfig().getLimit())) { + limit = queryConfig.getMetricTypeDefaultConfig().getLimit(); + } + return limit; + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java index 610318a41..eb8cc5e2a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -151,6 +152,20 @@ public class SemanticSchema implements Serializable { return getElementsById(dataSetId, dataSets).orElse(null); } + public QueryConfig getQueryConfig(Long dataSetId) { + DataSetSchema first = + dataSetSchemaList.stream() + .filter( + dataSetSchema -> + dataSetId.equals(dataSetSchema.getDataSet().getDataSetId())) + .findFirst() + .orElse(null); + if (Objects.nonNull(first)) { + return first.getQueryConfig(); + } + return null; + } + public List getDataSets() { List dataSets = new ArrayList<>(); dataSetSchemaList.stream().forEach(d -> dataSets.add(d.getDataSet())); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java index fddc79f3a..eefb2f0ef 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagTypeDefaultConfig.java @@ -9,4 +9,6 @@ public class TagTypeDefaultConfig { // default time to filter tag selection results private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); + + private Long limit; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java index e69e73fab..e8a302158 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java @@ -31,6 +31,8 @@ public class LLMResponseService { LLMSemanticQuery semanticQuery = QueryManager.createLLMQuery(LLMSqlQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); parseInfo.setDataSet(queryCtx.getSemanticSchema().getDataSet(parseResult.getDataSetId())); + parseInfo.setQueryConfig( + queryCtx.getSemanticSchema().getQueryConfig(parseResult.getDataSetId())); parseInfo .getElementMatches() .addAll(queryCtx.getMapInfo().getMatchedElements(parseInfo.getDataSetId())); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java index 0b559876e..94fea6e2b 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java @@ -113,6 +113,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { .collect(Collectors.toSet()); Long dataSetId = dataSetIds.iterator().next(); parseInfo.setDataSet(semanticSchema.getDataSet(dataSetId)); + parseInfo.setQueryConfig(semanticSchema.getQueryConfig(dataSetId)); Map> dim2Values = new HashMap<>(); Map> id2Values = new HashMap<>(); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java index 84fec0787..35b086d54 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java @@ -18,8 +18,6 @@ import java.util.Objects; @Slf4j public abstract class DetailSemanticQuery extends RuleSemanticQuery { - private static final Long DETAIL_MAX_RESULTS = 500L; - public DetailSemanticQuery() { super(); } @@ -35,7 +33,9 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery { super.fillParseInfo(chatQueryContext); parseInfo.setQueryType(QueryType.DETAIL); - parseInfo.setLimit(DETAIL_MAX_RESULTS); + if (Objects.isNull(parseInfo.getLimit())) { + parseInfo.setLimit(parseInfo.getDetailLimit()); + } if (!needFillDateConf(chatQueryContext)) { return; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java index 0d82bd5a3..8f1f5db9c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java @@ -20,8 +20,6 @@ import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.R @Slf4j public abstract class MetricSemanticQuery extends RuleSemanticQuery { - private static final Long METRIC_MAX_RESULTS = 365L; - public MetricSemanticQuery() { super(); queryMatcher.addOption(METRIC, REQUIRED, AT_LEAST, 1); @@ -36,7 +34,9 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { @Override public void fillParseInfo(ChatQueryContext chatQueryContext) { super.fillParseInfo(chatQueryContext); - parseInfo.setLimit(METRIC_MAX_RESULTS); + if (Objects.isNull(parseInfo.getLimit())) { + parseInfo.setLimit(parseInfo.getMetricLimit()); + } fillDateInfo(chatQueryContext); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java index f2d92b773..e600a5a10 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricTopNQuery.java @@ -23,7 +23,6 @@ import static com.tencent.supersonic.headless.chat.query.rule.QueryMatchOption.R public class MetricTopNQuery extends MetricSemanticQuery { public static final String QUERY_MODE = "METRIC_ORDERBY"; - private static final Long ORDERBY_MAX_RESULTS = 3L; private static final Pattern INTENT_PATTERN = Pattern.compile("(.*)(最大|最高|最多)(.*)"); public MetricTopNQuery() { @@ -52,7 +51,6 @@ public class MetricTopNQuery extends MetricSemanticQuery { public void fillParseInfo(ChatQueryContext chatQueryContext) { super.fillParseInfo(chatQueryContext); - parseInfo.setLimit(ORDERBY_MAX_RESULTS); parseInfo.setScore(parseInfo.getScore() + 2.0); parseInfo.setAggType(AggregateTypeEnum.SUM); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java index 8ea16788b..a3682bdf8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java @@ -145,7 +145,7 @@ public class S2ChatLayerService implements ChatLayerService { } SchemaElement dataSet = semanticSchema.getDataSet(dataSetId); semanticParseInfo.setDataSet(dataSet); - + semanticParseInfo.setQueryConfig(semanticSchema.getQueryConfig(dataSetId)); ComponentFactory.getSemanticCorrectors() .forEach( corrector -> {