topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 04:57:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) adapter for derived metrics (#646)

Browse Source
This commit is contained in:
jipeli
2024-01-18 11:11:56 +08:00
committed by GitHub
parent 71c491a80d
commit d4eecc1bf8
11 changed files with 537 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -1,5 +1,12 @@
package com.tencent.supersonic.headless.core.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -8,8 +15,26 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -19,22 +44,6 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
/**
* tools functions to analyze queryStructReq
*/
@@ -54,7 +63,7 @@ public class SqlGenerateUtils {
private String internalMetricNameSuffix;
public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils,
DateModeUtils dateModeUtils) {
DateModeUtils dateModeUtils) {
this.sqlFilterUtils = sqlFilterUtils;
this.dateModeUtils = dateModeUtils;
}
@@ -97,6 +106,13 @@ public class SqlGenerateUtils {
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
}
public String getSelect(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
String aggStr = queryStructCmd.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics))
.collect(Collectors.joining(","));
return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
: String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
}
public String getSelectField(final Aggregator agg) {
if (AggOperatorEnum.COUNT_DISTINCT.equals(agg.getFunc())) {
return "count(distinct " + agg.getColumn() + " ) AS " + agg.getColumn() + " ";
@@ -109,6 +125,13 @@ public class SqlGenerateUtils {
).collect(Collectors.joining(",")) + " ) AS " + agg.getColumn() + " ";
}
public String getSelectField(final Aggregator agg, Map<String, String> deriveMetrics) {
if (!deriveMetrics.containsKey(agg.getColumn())) {
return getSelectField(agg);
}
return deriveMetrics.get(agg.getColumn());
}
public String getGroupBy(QueryStructReq queryStructCmd) {
if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
return "";
@@ -125,6 +148,19 @@ public class SqlGenerateUtils {
.collect(Collectors.joining(","));
}
public String getOrderBy(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) {
return "";
}
if (!queryStructCmd.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
return getOrderBy(queryStructCmd);
}
return "order by " + queryStructCmd.getOrders().stream()
.map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn())
: order.getColumn()) + " " + order.getDirection() + " ")
.collect(Collectors.joining(","));
}
public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) {
String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter());
String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp);
@@ -132,7 +168,7 @@ public class SqlGenerateUtils {
}
private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
String whereFromDate) {
String whereFromDate) {
if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
} else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
@@ -233,4 +269,83 @@ public class SqlGenerateUtils {
return modelBizName + UNDERLINE + internalMetricNameSuffix;
}
public String generateDerivedMetric(final List<MetricResp> metricResps, final Set<String> allFields,
final Map<String, Measure> allMeasures, final List<DimensionResp> dimensionResps,
final String expression, final MetricDefineType metricDefineType, Set<String> visitedMetric,
Set<String> measures,
Set<String> dimensions) {
Set<String> fields = SqlParserSelectHelper.getColumnFromExpr(expression);
if (!CollectionUtils.isEmpty(fields)) {
Map<String, String> replace = new HashMap<>();
for (String field : fields) {
switch (metricDefineType) {
case METRIC:
Optional<MetricResp> metricItem = metricResps.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
if (metricItem.isPresent()) {
if (MetricType.isDerived(metricItem.get().getMetricDefineType(),
metricItem.get().getTypeParams())) {
if (visitedMetric.contains(field)) {
break;
}
replace.put(field,
generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps,
getExpr(metricItem.get()), metricItem.get().getMetricDefineType(),
visitedMetric, measures, dimensions));
visitedMetric.add(field);
} else {
replace.put(field, getExpr(metricItem.get()));
}
}
break;
case MEASURE:
if (allMeasures.containsKey(field)) {
measures.add(field);
replace.put(field, getExpr(allMeasures.get(field)));
}
break;
case FIELD:
if (allFields.contains(field)) {
Optional<DimensionResp> dimensionItem = dimensionResps.stream()
.filter(d -> d.getBizName().equals(field)).findFirst();
if (dimensionItem.isPresent()) {
dimensions.add(field);
} else {
measures.add(field);
}
}
break;
default:
break;
}
}
if (!CollectionUtils.isEmpty(replace)) {
return SqlParserReplaceHelper.replaceExpression(expression, replace);
}
}
return expression;
}
public String getExpr(Measure measure) {
if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) {
return AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " " + measure.getBizName()
+ " ) ";
}
return measure.getAgg() + " ( " + measure.getBizName() + " ) ";
}
public String getExpr(MetricResp metricResp) {
if (Objects.isNull(metricResp.getMetricDefineType())) {
return metricResp.getTypeParams().getExpr();
}
if (metricResp.getMetricDefineType().equals(MetricDefineType.METRIC)) {
return metricResp.getMetricDefineByMetricParams().getExpr();
}
if (metricResp.getMetricDefineType().equals(MetricDefineType.FIELD)) {
return metricResp.getMetricDefineByFieldParams().getExpr();
}
// measure add agg function
return metricResp.getTypeParams().getExpr();
}
}