topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Enable partition dates in the demo and support partition dates at the dataset granularity. (#1513)

Browse Source
This commit is contained in:
lexluo09
2024-08-04 23:01:54 +08:00
committed by GitHub
parent e2e45a40ab
commit 0a4272c25e
11 changed files with 125 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java
View File

@@ -18,20 +18,20 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.chat.query.QueryManager;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
@Slf4j
public class QueryReqBuilder {
@@ -161,78 +161,86 @@ public class QueryReqBuilder {
}
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {
List<Aggregator> aggregators = new ArrayList<>();
if (metric != null) {
String agg = "";
if (Objects.isNull(aggregateType) || aggregateType.equals(AggregateTypeEnum.NONE)
|| AggOperatorEnum.COUNT_DISTINCT.name().equalsIgnoreCase(metric.getDefaultAgg())) {
if (StringUtils.isNotBlank(metric.getDefaultAgg())) {
agg = metric.getDefaultAgg();
}
} else {
agg = aggregateType.name();
}
aggregators.add(new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg)));
if (metric == null) {
return Collections.emptyList();
}
return aggregators;
String agg = determineAggregator(aggregateType, metric);
return Collections.singletonList(new Aggregator(metric.getBizName(), AggOperatorEnum.of(agg)));
}
private static String determineAggregator(AggregateTypeEnum aggregateType, SchemaElement metric) {
if (aggregateType == null || aggregateType.equals(AggregateTypeEnum.NONE)
|| AggOperatorEnum.COUNT_DISTINCT.name().equalsIgnoreCase(metric.getDefaultAgg())) {
return StringUtils.defaultIfBlank(metric.getDefaultAgg(), "");
}
return aggregateType.name();
}
private static void addDateDimension(SemanticParseInfo parseInfo) {
if (parseInfo != null) {
String queryMode = parseInfo.getQueryMode();
if (parseInfo.getDateInfo() == null) {
return;
}
if (parseInfo.getAggType() != null && (parseInfo.getAggType().equals(AggregateTypeEnum.MAX)
|| parseInfo.getAggType().equals(AggregateTypeEnum.MIN)) && !CollectionUtils.isEmpty(
parseInfo.getDimensions())) {
return;
}
DateConf dateInfo = parseInfo.getDateInfo();
String dateField = getDateField(dateInfo);
if (parseInfo == null || parseInfo.getDateInfo() == null) {
return;
}
for (SchemaElement dimension : parseInfo.getDimensions()) {
if (dimension.getBizName().equalsIgnoreCase(dateField)) {
return;
}
}
if (shouldSkipAddingDateDimension(parseInfo)) {
return;
}
if (Objects.nonNull(parseInfo.getAggType()) && !parseInfo.getAggType().equals(AggregateTypeEnum.NONE)) {
return;
}
String dateField = getDateField(parseInfo.getDateInfo());
if (isDateFieldAlreadyPresent(parseInfo, dateField)) {
return;
}
SchemaElement dimension = new SchemaElement();
dimension.setBizName(dateField);
SchemaElement dimension = new SchemaElement();
dimension.setBizName(dateField);
if (QueryManager.isMetricQuery(queryMode)) {
List<String> timeDimensions = Arrays.asList(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.MONTH.getName());
Set<SchemaElement> dimensions = parseInfo.getDimensions().stream()
.filter(d -> !timeDimensions.contains(d.getBizName().toLowerCase())).collect(
Collectors.toSet());
dimensions.add(dimension);
parseInfo.setDimensions(dimensions);
}
if (QueryManager.isMetricQuery(parseInfo.getQueryMode())) {
addDimension(parseInfo, dimension);
}
}
public static Set<Order> getOrder(Set<Order> parseOrder, AggregateTypeEnum aggregator, SchemaElement metric) {
if (!CollectionUtils.isEmpty(parseOrder)) {
return parseOrder;
}
Set<Order> orders = new LinkedHashSet();
if (metric == null) {
return orders;
private static boolean shouldSkipAddingDateDimension(SemanticParseInfo parseInfo) {
return parseInfo.getAggType() != null
&& (parseInfo.getAggType().equals(AggregateTypeEnum.MAX)
|| parseInfo.getAggType().equals(AggregateTypeEnum.MIN))
&& !CollectionUtils.isEmpty(parseInfo.getDimensions());
}
private static boolean isDateFieldAlreadyPresent(SemanticParseInfo parseInfo, String dateField) {
return parseInfo.getDimensions().stream()
.anyMatch(dimension -> dimension.getBizName().equalsIgnoreCase(dateField));
}
private static void addDimension(SemanticParseInfo parseInfo, SchemaElement dimension) {
List<String> timeDimensions = Arrays.asList(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.MONTH.getName());
Set<SchemaElement> dimensions = parseInfo.getDimensions().stream()
.filter(d -> !timeDimensions.contains(d.getBizName().toLowerCase()))
.collect(Collectors.toSet());
dimensions.add(dimension);
parseInfo.setDimensions(dimensions);
}
public static Set<Order> getOrder(Set<Order> existingOrders,
AggregateTypeEnum aggregator, SchemaElement metric) {
if (existingOrders != null && !existingOrders.isEmpty()) {
return existingOrders;
}
if ((AggregateTypeEnum.TOPN.equals(aggregator) || AggregateTypeEnum.MAX.equals(aggregator)
|| AggregateTypeEnum.MIN.equals(
aggregator))) {
if (metric == null) {
return Collections.emptySet();
}
Set<Order> orders = new LinkedHashSet<>();
if (aggregator == AggregateTypeEnum.TOPN
|| aggregator == AggregateTypeEnum.MAX
|| aggregator == AggregateTypeEnum.MIN) {
Order order = new Order();
order.setColumn(metric.getBizName());
order.setDirection("desc");
orders.add(order);
}
return orders;
}

10
launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java
View File

@@ -189,7 +189,7 @@ public class S2VisitsDemo extends S2BaseDemo {
}
public ModelResp addModel_1(DomainResp s2Domain, DatabaseResp s2Database,
TagObjectResp s2TagObject) throws Exception {
TagObjectResp s2TagObject) throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("用户部门");
modelReq.setBizName("user_department");
@@ -282,7 +282,7 @@ public class S2VisitsDemo extends S2BaseDemo {
modelDetail.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionType.time.name(), 0);
Dim dimension1 = new Dim("数据日期", "imp_date", DimensionType.partition_time.name(), 1);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
Dim dimension2 = new Dim("页面", "page", DimensionType.categorical.name(), 1);
@@ -352,7 +352,7 @@ public class S2VisitsDemo extends S2BaseDemo {
}
public void updateMetric(ModelResp stayTimeModel, DimensionResp departmentDimension,
DimensionResp userDimension) throws Exception {
DimensionResp userDimension) throws Exception {
MetricResp stayHoursMetric =
metricService.getMetric(stayTimeModel.getId(), "stay_hours");
MetricReq metricReq = new MetricReq();
@@ -378,7 +378,7 @@ public class S2VisitsDemo extends S2BaseDemo {
}
public void updateMetric_pv(ModelResp pvUvModel, DimensionResp departmentDimension,
DimensionResp userDimension, MetricResp metricPv) throws Exception {
DimensionResp userDimension, MetricResp metricPv) throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(pvUvModel.getId());
metricReq.setId(metricPv.getId());
@@ -420,7 +420,7 @@ public class S2VisitsDemo extends S2BaseDemo {
}
public MetricResp addMetric_pv_avg(MetricResp metricPv, MetricResp metricUv,
DimensionResp departmentDimension, ModelResp pvModel) throws Exception {
DimensionResp departmentDimension, ModelResp pvModel) throws Exception {
MetricReq metricReq = new MetricReq();
metricReq.setModelId(pvModel.getId());
metricReq.setName("人均访问次数");

22
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt
View File

@@ -5,14 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _4_7 100
陈奕迅 _4_7 100
林俊杰 _4_7 100
张碧晨 _4_7 100
程响 _4_7 100
Taylor#Swift _4_7 100
内地 _4_4 100
欧美 _4_4 100
港台 _4_4 100
流行 _4_6 100
国风 _4_6 100
周杰伦 _4_8 100
陈奕迅 _4_8 100
林俊杰 _4_8 100
张碧晨 _4_8 100
程响 _4_8 100
Taylor#Swift _4_8 100
内地 _4_5 100
欧美 _4_5 100
港台 _4_5 100
流行 _4_7 100
国风 _4_7 100

12
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt
View File

@@ -1,6 +1,6 @@
p1 _3_3 52
p2 _3_3 47
p3 _3_3 31
p4 _3_3 36
p5 _3_3 50
p6 _3_3 38
p1 _3_4 52
p2 _3_4 47
p3 _3_4 31
p4 _3_4 36
p5 _3_4 50
p6 _3_4 38

9
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_8.txt Normal file
View File

@@ -0,0 +1,9 @@
周杰伦 _4_8 9000
周深 _4_8 8000
周传雄 _4_8 7000
周华建 _4_8 6000
陈奕迅 _4_8 8000
林俊杰 _4_8 7000
张碧晨 _4_8 7000
程响 _4_8 7000
Taylor#Swift _4_8 7000

9
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt
View File

@@ -1,9 +0,0 @@
周杰伦 _4_7 9000
周深 _4_7 8000
周传雄 _4_7 7000
周华建 _4_7 6000
陈奕迅 _4_7 8000
林俊杰 _4_7 7000
张碧晨 _4_7 7000
程响 _4_7 7000
Taylor#Swift _4_7 7000

2
launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java
View File

@@ -33,7 +33,7 @@ public class TagTest extends BaseTest {
expectedParseInfo.setAggType(AggregateTypeEnum.NONE);
QueryFilter dimensionFilter = DataUtils.getFilter("genre", FilterOperatorEnum.EQUALS,
"流行", "风格", 6L);
"流行", "风格", 7L);
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
SchemaElement metric = SchemaElement.builder().name("播放量").build();

22
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt
View File

@@ -5,14 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _4_7 100
陈奕迅 _4_7 100
林俊杰 _4_7 100
张碧晨 _4_7 100
程响 _4_7 100
Taylor#Swift _4_7 100
内地 _4_4 100
欧美 _4_4 100
港台 _4_4 100
流行 _4_6 100
国风 _4_6 100
周杰伦 _4_8 100
陈奕迅 _4_8 100
林俊杰 _4_8 100
张碧晨 _4_8 100
程响 _4_8 100
Taylor#Swift _4_8 100
内地 _4_5 100
欧美 _4_5 100
港台 _4_5 100
流行 _4_7 100
国风 _4_7 100

12
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt
View File

@@ -1,6 +1,6 @@
p1 _3_3 52
p2 _3_3 47
p3 _3_3 31
p4 _3_3 36
p5 _3_3 50
p6 _3_3 38
p1 _3_4 52
p2 _3_4 47
p3 _3_4 31
p4 _3_4 36
p5 _3_4 50
p6 _3_4 38

9
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_8.txt Normal file
View File

@@ -0,0 +1,9 @@
周杰伦 _4_8 9000
周深 _4_8 8000
周传雄 _4_8 7000
周华建 _4_8 6000
陈奕迅 _4_8 8000
林俊杰 _4_8 7000
张碧晨 _4_8 7000
程响 _4_8 7000
Taylor#Swift _4_8 7000

9
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt
View File

@@ -1,9 +0,0 @@
周杰伦 _4_7 9000
周深 _4_7 8000
周传雄 _4_7 7000
周华建 _4_7 6000
陈奕迅 _4_7 8000
林俊杰 _4_7 7000
张碧晨 _4_7 7000
程响 _4_7 7000
Taylor#Swift _4_7 7000