topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 13:07:32 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Set default query limits of DataSet.

Browse Source
This commit is contained in:
jerryjzhang
2024-09-27 16:07:25 +08:00
parent c8ac5a3ba2
commit 764d3d24aa
11 changed files with 26 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java
View File

@@ -37,6 +37,7 @@ public class Constants {
public static final String CONTEXT = "CONTEXT"; public static final String CONTEXT = "CONTEXT";
public static final Long DEFAULT_FREQUENCY = 100000L; public static final Long DEFAULT_FREQUENCY = 100000L;
public static final String TABLE_PREFIX = "t_"; public static final String TABLE_PREFIX = "t_";
public static final Long DEFAULT_DETAIL_LIMIT = 500L; public static final long DEFAULT_DETAIL_LIMIT = 500;
public static final Long DEFAULT_METRIC_LIMIT = 200L; public static final long DEFAULT_METRIC_LIMIT = 200;
public static final long DEFAULT_DOWNLOAD_LIMIT = 10000;
} }

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/AggregateTypeDefaultConfig.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.api.pojo; package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.common.pojo.enums.TimeMode;
import lombok.Data; import lombok.Data;
@@ -10,5 +11,5 @@ public class AggregateTypeDefaultConfig {
private TimeDefaultConfig timeDefaultConfig = private TimeDefaultConfig timeDefaultConfig =
new TimeDefaultConfig(7, DatePeriodEnum.DAY, TimeMode.RECENT); new TimeDefaultConfig(7, DatePeriodEnum.DAY, TimeMode.RECENT);
private Long limit; private long limit = Constants.DEFAULT_METRIC_LIMIT;
} }

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DetailTypeDefaultConfig.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.api.pojo; package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.common.pojo.Constants;
import lombok.Data; import lombok.Data;
@Data @Data
@@ -10,5 +11,5 @@ public class DetailTypeDefaultConfig {
// default time to filter tag selection results // default time to filter tag selection results
private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig(); private TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig();
private Long limit; private long limit = Constants.DEFAULT_DETAIL_LIMIT;
} }

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ItemValueConfig.java
View File

@@ -14,6 +14,6 @@ public class ItemValueConfig {
private List<String> blackList = new ArrayList<>(); private List<String> blackList = new ArrayList<>();
private List<String> whiteList = new ArrayList<>(); private List<String> whiteList = new ArrayList<>();
private List<String> ruleList = new ArrayList<>(); private List<String> ruleList = new ArrayList<>();
private Long limit; private int limit;
private DateConf dateConf; private DateConf dateConf;
} }

10
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java
View File

@@ -36,7 +36,7 @@ public class SemanticParseInfo {
private Set<QueryFilter> metricFilters = Sets.newHashSet(); private Set<QueryFilter> metricFilters = Sets.newHashSet();
private Set<Order> orders = Sets.newHashSet(); private Set<Order> orders = Sets.newHashSet();
private DateConf dateInfo; private DateConf dateInfo;
private Long limit; private long limit = DEFAULT_DETAIL_LIMIT;
private double score; private double score;
private List<SchemaElementMatch> elementMatches = Lists.newArrayList(); private List<SchemaElementMatch> elementMatches = Lists.newArrayList();
private SqlInfo sqlInfo = new SqlInfo(); private SqlInfo sqlInfo = new SqlInfo();
@@ -74,8 +74,8 @@ public class SemanticParseInfo {
return dataSet.getDataSetId(); return dataSet.getDataSetId();
} }
public Long getDetailLimit() { public long getDetailLimit() {
Long limit = DEFAULT_DETAIL_LIMIT; long limit = DEFAULT_DETAIL_LIMIT;
if (Objects.nonNull(queryConfig) if (Objects.nonNull(queryConfig)
&& Objects.nonNull(queryConfig.getDetailTypeDefaultConfig()) && Objects.nonNull(queryConfig.getDetailTypeDefaultConfig())
&& Objects.nonNull(queryConfig.getDetailTypeDefaultConfig().getLimit())) { && Objects.nonNull(queryConfig.getDetailTypeDefaultConfig().getLimit())) {
@@ -84,8 +84,8 @@ public class SemanticParseInfo {
return limit; return limit;
} }
public Long getMetricLimit() { public long getMetricLimit() {
Long limit = DEFAULT_METRIC_LIMIT; long limit = DEFAULT_METRIC_LIMIT;
if (Objects.nonNull(queryConfig) if (Objects.nonNull(queryConfig)
&& Objects.nonNull(queryConfig.getAggregateTypeDefaultConfig()) && Objects.nonNull(queryConfig.getAggregateTypeDefaultConfig())
&& Objects.nonNull(queryConfig.getAggregateTypeDefaultConfig().getLimit())) { && Objects.nonNull(queryConfig.getAggregateTypeDefaultConfig().getLimit())) {

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMetricReq.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.headless.api.pojo.request; package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Filter;
import lombok.Data; import lombok.Data;
@@ -26,7 +27,7 @@ public class QueryMetricReq {
private DateConf dateInfo = new DateConf(); private DateConf dateInfo = new DateConf();
private Long limit = 2000L; private long limit = Constants.DEFAULT_METRIC_LIMIT;
private boolean innerLayerNative = false; private boolean innerLayerNative = false;
} }

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -48,7 +48,7 @@ public class QueryStructReq extends SemanticQueryReq {
private List<Filter> dimensionFilters = new ArrayList<>(); private List<Filter> dimensionFilters = new ArrayList<>();
private List<Filter> metricFilters = new ArrayList<>(); private List<Filter> metricFilters = new ArrayList<>();
private DateConf dateInfo; private DateConf dateInfo;
private Long limit = 2000L; private long limit = Constants.DEFAULT_DETAIL_LIMIT;
private QueryType queryType = QueryType.ID; private QueryType queryType = QueryType.ID;
private boolean convertToSql = true; private boolean convertToSql = true;

3
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java
View File

@@ -33,9 +33,8 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery {
super.fillParseInfo(chatQueryContext); super.fillParseInfo(chatQueryContext);
parseInfo.setQueryType(QueryType.DETAIL); parseInfo.setQueryType(QueryType.DETAIL);
if (Objects.isNull(parseInfo.getLimit())) {
parseInfo.setLimit(parseInfo.getDetailLimit()); parseInfo.setLimit(parseInfo.getDetailLimit());
}
if (!needFillDateConf(chatQueryContext)) { if (!needFillDateConf(chatQueryContext)) {
return; return;
} }

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java
View File

@@ -34,9 +34,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
@Override @Override
public void fillParseInfo(ChatQueryContext chatQueryContext) { public void fillParseInfo(ChatQueryContext chatQueryContext) {
super.fillParseInfo(chatQueryContext); super.fillParseInfo(chatQueryContext);
if (Objects.isNull(parseInfo.getLimit())) {
parseInfo.setLimit(parseInfo.getMetricLimit()); parseInfo.setLimit(parseInfo.getMetricLimit());
}
fillDateInfo(chatQueryContext); fillDateInfo(chatQueryContext);
} }

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -9,6 +9,7 @@ import com.alibaba.excel.write.metadata.WriteSheet;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
@@ -58,7 +59,7 @@ public class DownloadServiceImpl implements DownloadService {
private static final String internMetricCol = "指标名称"; private static final String internMetricCol = "指标名称";
private static final long downloadSize = 10000; private static final long downloadLimit = Constants.DEFAULT_DOWNLOAD_LIMIT;
private static final String dateFormat = "yyyyMMddHHmmss"; private static final String dateFormat = "yyyyMMddHHmmss";
@@ -293,7 +294,7 @@ public class DownloadServiceImpl implements DownloadService {
queryStructReq.setAggregators(Lists.newArrayList(aggregator)); queryStructReq.setAggregators(Lists.newArrayList(aggregator));
queryStructReq.setDateInfo(dateConf); queryStructReq.setDateInfo(dateConf);
queryStructReq.setModelIds(modelIds); queryStructReq.setModelIds(modelIds);
queryStructReq.setLimit(downloadSize); queryStructReq.setLimit(downloadLimit);
return queryStructReq; return queryStructReq;
} }

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
View File

@@ -68,7 +68,7 @@ public class DictUtils {
private String dimMultiValueSplit; private String dimMultiValueSplit;
@Value("${s2.item.value.max.count:100000}") @Value("${s2.item.value.max.count:100000}")
private Long itemValueMaxCount; private int itemValueMaxCount;
@Value("${s2.item.value.white.frequency:999999}") @Value("${s2.item.value.white.frequency:999999}")
private Long itemValueWhiteFrequency; private Long itemValueWhiteFrequency;
@@ -285,7 +285,7 @@ public class DictUtils {
String whereStr = generateWhereStr(dictItemResp); String whereStr = generateWhereStr(dictItemResp);
String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr; String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr;
ItemValueConfig config = dictItemResp.getConfig(); ItemValueConfig config = dictItemResp.getConfig();
Long limit = int limit =
(Objects.isNull(config) || Objects.isNull(config.getLimit())) (Objects.isNull(config) || Objects.isNull(config.getLimit()))
? itemValueMaxCount ? itemValueMaxCount
: dictItemResp.getConfig().getLimit(); : dictItemResp.getConfig().getLimit();
@@ -331,7 +331,7 @@ public class DictUtils {
String whereStr = generateWhereStr(dictItemResp); String whereStr = generateWhereStr(dictItemResp);
String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr; String where = StringUtils.isEmpty(whereStr) ? "" : "WHERE" + whereStr;
ItemValueConfig config = dictItemResp.getConfig(); ItemValueConfig config = dictItemResp.getConfig();
Long limit = long limit =
(Objects.isNull(config) || Objects.isNull(config.getLimit())) (Objects.isNull(config) || Objects.isNull(config.getLimit()))
? itemValueMaxCount ? itemValueMaxCount
: dictItemResp.getConfig().getLimit(); : dictItemResp.getConfig().getLimit();
@@ -371,7 +371,7 @@ public class DictUtils {
fillStructDateInfo(queryStructReq, dictItemResp); fillStructDateInfo(queryStructReq, dictItemResp);
Long limit = int limit =
Objects.isNull(dictItemResp.getConfig().getLimit()) Objects.isNull(dictItemResp.getConfig().getLimit())
? itemValueMaxCount ? itemValueMaxCount
: dictItemResp.getConfig().getLimit(); : dictItemResp.getConfig().getLimit();