topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Remove fixed TimeEnums fields.

Browse Source
This commit is contained in:
jerryjzhang
2024-12-11 21:33:43 +08:00
parent f97ac1da83
commit 8c44c9f42f
36 changed files with 142 additions and 722 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java
View File

@@ -41,7 +41,6 @@ public class DefaultDimValueConverter implements QueryConverter {
}
String sql = queryStatement.getSqlQueryParam().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql).stream()
.filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;

17
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
@@ -350,15 +349,8 @@ public class MetricRatioConverter implements QueryConverter {
return CollectionUtils.isEmpty(groups) ? aggStr : String.join(",", groups) + "," + aggStr;
}
private String getGroupDimWithOutTime(StructQueryParam structQueryParam) {
String timeDim = getTimeDim(structQueryParam);
return structQueryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
private static String getTimeDim(StructQueryParam structQueryParam) {
DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
return dateModeUtils.getSysDateCol(structQueryParam.getDateInfo());
return structQueryParam.getDateInfo().getDateField();
}
private static String getLimit(StructQueryParam structQueryParam) {
@@ -385,13 +377,6 @@ public class MetricRatioConverter implements QueryConverter {
return sqlGenerateUtils.getSelectField(agg);
}
private String getGroupBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", structQueryParam.getGroups());
}
private static String getOrderBy(StructQueryParam structQueryParam) {
return "order by " + getTimeDim(structQueryParam) + " desc";
}

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
@@ -115,8 +114,8 @@ public class SqlQueryConverter implements QueryConverter {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(),
SchemaItem::getBizName, (k1, k2) -> k1));
dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.DAY.getName());
// dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
// TimeDimensionEnum.DAY.getName());
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase()))
@@ -167,8 +166,6 @@ public class SqlQueryConverter implements QueryConverter {
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
dimensionResults.putAll(metricResults);
return dimensionResults;
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java
View File

@@ -2,8 +2,6 @@ package com.tencent.supersonic.headless.core.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
@@ -53,7 +51,7 @@ public class DataTransformUtils {
private static String getRowKey(Map<String, Object> originalRow, List<String> groups) {
List<Object> values = Lists.newArrayList();
for (String key : originalRow.keySet()) {
if (groups.contains(key) && !TimeDimensionEnum.getNameList().contains(key)) {
if (groups.contains(key)) {
values.add(originalRow.get(key));
}
}
@@ -61,12 +59,6 @@ public class DataTransformUtils {
}
private static String getTimeDimension(DateConf dateConf) {
if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
return dateConf.getDateField();
}
}

22
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -7,7 +7,6 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
@@ -29,19 +28,11 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
/** tools functions to analyze queryStructReq */
@Component
@@ -68,12 +59,7 @@ public class SqlGenerateUtils {
if (group.contains(JOIN_UNDERLINE)) {
group = group.split(JOIN_UNDERLINE)[1];
}
if (!TimeDimensionEnum.getNameList().contains(group)) {
locate++;
sb.append(group).append(" as ").append("name").append(locate).append(",");
} else {
sb.append(group).append(",");
}
sb.append(group).append(",");
}
locate = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
@@ -271,10 +257,6 @@ public class SqlGenerateUtils {
return true;
}
public String generateInternalMetricName(String modelBizName) {
return modelBizName + UNDERLINE + executorConfig.getInternalMetricNameSuffix();
}
public String generateDerivedMetric(final List<MetricSchemaResp> metricResps,
final Set<String> allFields, final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps, final String expression,

102
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java
View File

@@ -1,102 +0,0 @@
package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** generate system time dimension tools */
@Slf4j
public class SysTimeDimensionBuilder {
// Defines the regular expression pattern for the time keyword
private static final Pattern TIME_KEYWORD_PATTERN =
Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b",
Pattern.CASE_INSENSITIVE);
public static void addSysTimeDimension(List<Dimension> dims, DbAdaptor engineAdaptor) {
log.debug("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor);
Dimension timeDim = getTimeDim(dims);
if (timeDim == null) {
timeDim = Dimension.getDefault();
// todo not find the time dimension
return;
}
dims.add(generateSysDayDimension(timeDim, engineAdaptor));
dims.add(generateSysWeekDimension(timeDim, engineAdaptor));
dims.add(generateSysMonthDimension(timeDim, engineAdaptor));
log.debug("addSysTimeDimension after:{}, engineAdaptor:{}", dims, engineAdaptor);
}
private static Dimension generateSysDayDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.DAY.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.DAY.name().toLowerCase());
typeParams.setIsPrimary("true");
dim.setTypeParams(typeParams);
return dim;
}
private static Dimension generateSysWeekDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.WEEK.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.WEEK.name().toLowerCase());
typeParams.setIsPrimary("false");
dim.setTypeParams(typeParams);
return dim;
}
private static Dimension generateSysMonthDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.MONTH.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.MONTH.name().toLowerCase());
typeParams.setIsPrimary("false");
dim.setTypeParams(typeParams);
return dim;
}
private static boolean containsTimeKeyword(String fieldName) {
Matcher matcher = TIME_KEYWORD_PATTERN.matcher(fieldName);
return matcher.find();
}
// Check whether the time field contains keywords,Generation time expression
private static String generateTimeExpr(Dimension timeDim, String dateType,
DbAdaptor engineAdaptor) {
String bizName = timeDim.getBizName();
String dateFormat = timeDim.getDateFormat();
if (containsTimeKeyword(bizName)) {
String bizNameWithBackticks = String.format("`%s`", bizName);
return engineAdaptor.getDateFormat(dateType, dateFormat, bizNameWithBackticks);
} else {
return engineAdaptor.getDateFormat(dateType, dateFormat, bizName);
}
}
private static Dimension getTimeDim(List<Dimension> timeDims) {
for (Dimension dim : timeDims) {
if (dim.getType().equals(DimensionType.partition_time)) {
return dim;
}
}
return null;
}
}