topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 05:43:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(project) support for modifying filter conditions and fix group by pushdown and add windows scipt (#49)

Browse Source 
Co-authored-by: lexluo <lexluo@tencent.com>
This commit is contained in:
lexluo09
2023-09-03 23:51:47 +08:00
committed by GitHub
parent 8440f1f30e
commit 559ef974b0
317 changed files with 7449 additions and 9413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java
View File

@@ -1,8 +1,10 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import java.util.List;
import javax.validation.constraints.NotBlank;
import lombok.Data;
@Data

1
common/src/main/java/com/tencent/supersonic/common/pojo/Order.java
View File

@@ -4,6 +4,7 @@ import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER;
import com.google.common.base.Objects;
import javax.validation.constraints.NotBlank;
import lombok.Data;
@Data

1
common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.common.pojo;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

1
common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java
View File

@@ -7,7 +7,6 @@ import lombok.Data;
*/
@Data
public class ResultData<T> {
private int code;
private String msg;
private T data;

19
common/src/main/java/com/tencent/supersonic/common/util/DatePeriodEnum.java
View File

@@ -1,10 +1,19 @@
package com.tencent.supersonic.common.util;
public enum DatePeriodEnum {
DAY,
WEEK,
MONTH,
YEAR;
DAY("日"),
WEEK("周"),
MONTH("月"),
YEAR("年");
private String chName;
DatePeriodEnum(String chName) {
this.chName = chName;
}
public String getChName() {
return chName;
}
public static DatePeriodEnum get(String period) {
for (DatePeriodEnum value : values()) {
@@ -14,4 +23,6 @@ public enum DatePeriodEnum {
}
return null;
}
}

3
common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java
View File

@@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.InvalidParameterException;
@@ -20,6 +19,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;

2
common/src/main/java/com/tencent/supersonic/common/util/ThreadContext.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.common.util;
import java.util.Map;
import lombok.Builder;
import lombok.Data;
import lombok.ToString;
import java.util.Map;
@Builder
@ToString

1
common/src/main/java/com/tencent/supersonic/common/util/calcite/SqlParseUtils.java
View File

@@ -40,6 +40,7 @@ public class SqlParseUtils {
handlerSQL(sqlNode, sqlParserInfo);
sqlParserInfo.setAllFields(sqlParserInfo.getAllFields().stream().distinct().collect(Collectors.toList()));
sqlParserInfo.setSelectFields(

61
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/DateFunctionHelper.java Normal file
View File

@@ -0,0 +1,61 @@
package com.tencent.supersonic.common.util.jsqlparser;
import com.tencent.supersonic.common.util.DatePeriodEnum;
import com.tencent.supersonic.common.util.DateUtils;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
@Slf4j
public class DateFunctionHelper {
public static final String DATE_FUNCTION = "datediff";
public static final double HALF_YEAR = 0.5d;
public static final int SIX_MONTH = 6;
public static final String EQUAL = "=";
public static String getStartDateStr(ComparisonOperator minorThanEquals, List<Expression> expressions) {
String unitValue = getUnit(expressions);
String dateValue = getEndDateValue(expressions);
String dateStr = "";
Expression rightExpression = minorThanEquals.getRightExpression();
DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(unitValue);
if (rightExpression instanceof DoubleValue) {
DoubleValue value = (DoubleValue) rightExpression;
double doubleValue = value.getValue();
if (DatePeriodEnum.YEAR.equals(datePeriodEnum) && doubleValue == HALF_YEAR) {
datePeriodEnum = DatePeriodEnum.MONTH;
dateStr = DateUtils.getBeforeDate(dateValue, SIX_MONTH, datePeriodEnum);
}
} else if (rightExpression instanceof LongValue) {
LongValue value = (LongValue) rightExpression;
long doubleValue = value.getValue();
dateStr = DateUtils.getBeforeDate(dateValue, (int) doubleValue, datePeriodEnum);
}
return dateStr;
}
public static String getEndDateOperator(ComparisonOperator comparisonOperator) {
String operator = comparisonOperator.getStringExpression();
if (EQUAL.equalsIgnoreCase(operator)) {
operator = "<=";
}
return operator;
}
public static String getEndDateValue(List<Expression> leftExpressions) {
StringValue date = (StringValue) leftExpressions.get(2);
return date.getValue();
}
private static String getUnit(List<Expression> expressions) {
StringValue unit = (StringValue) expressions.get(0);
return unit.getValue();
}
}

52
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java
View File

@@ -1,9 +1,15 @@
package com.tencent.supersonic.common.util.jsqlparser;
import com.tencent.supersonic.common.util.DatePeriodEnum;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
@@ -13,6 +19,7 @@ import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter {
@@ -66,19 +73,42 @@ public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter {
columnName = column.getColumnName();
filterExpression.setFieldName(columnName);
}
if (rightExpression instanceof StringValue) {
StringValue stringValue = (StringValue) rightExpression;
filterExpression.setFieldValue(stringValue.getValue());
}
if (rightExpression instanceof DoubleValue) {
DoubleValue doubleValue = (DoubleValue) rightExpression;
filterExpression.setFieldValue(doubleValue.getValue());
}
if (rightExpression instanceof LongValue) {
LongValue longValue = (LongValue) rightExpression;
filterExpression.setFieldValue(longValue.getValue());
if (leftExpression instanceof Function) {
Function leftExpressionFunction = (Function) leftExpression;
String dateFunction = leftExpressionFunction.getName().toUpperCase();
List<DatePeriodEnum> collect = Arrays.stream(DatePeriodEnum.values()).collect(Collectors.toList());
DatePeriodEnum periodEnum = DatePeriodEnum.get(dateFunction);
if (Objects.nonNull(periodEnum) && collect.contains(periodEnum)) {
List<Expression> leftExpressions = leftExpressionFunction.getParameters().getExpressions();
if (CollectionUtils.isEmpty(leftExpressions) || leftExpressions.size() < 1) {
return filterExpression;
}
Column field = (Column) leftExpressions.get(0);
filterExpression.setFieldName(field.getColumnName());
filterExpression.setFieldValue(getFieldValue(rightExpression) + periodEnum.getChName());
filterExpression.setOperator(expr.getStringExpression());
return filterExpression;
}
}
filterExpression.setFieldValue(getFieldValue(rightExpression));
filterExpression.setOperator(expr.getStringExpression());
return filterExpression;
}
private Object getFieldValue(Expression rightExpression) {
if (rightExpression instanceof StringValue) {
StringValue stringValue = (StringValue) rightExpression;
return stringValue.getValue();
}
if (rightExpression instanceof DoubleValue) {
DoubleValue doubleValue = (DoubleValue) rightExpression;
return doubleValue.getValue();
}
if (rightExpression instanceof LongValue) {
LongValue longValue = (LongValue) rightExpression;
return longValue.getValue();
}
return null;
}
}

54
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldlValueReplaceVisitor.java Normal file
View File

@@ -0,0 +1,54 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Map;
import java.util.Objects;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
public class FieldlValueReplaceVisitor extends ExpressionVisitorAdapter {
private Map<String, Map<String, String>> filedNameToValueMap;
public FieldlValueReplaceVisitor(Map<String, Map<String, String>> filedNameToValueMap) {
this.filedNameToValueMap = filedNameToValueMap;
}
@Override
public void visit(EqualsTo expr) {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
if (!(rightExpression instanceof StringValue)) {
return;
}
if (!(leftExpression instanceof Column)) {
return;
}
if (CollectionUtils.isEmpty(filedNameToValueMap)) {
return;
}
if (Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) {
return;
}
Column leftColumnName = (Column) leftExpression;
StringValue rightStringValue = (StringValue) rightExpression;
String columnName = leftColumnName.getColumnName();
if (StringUtils.isEmpty(columnName)) {
return;
}
Map<String, String> valueMap = filedNameToValueMap.get(columnName);
if (Objects.isNull(valueMap) || valueMap.isEmpty()) {
return;
}
String replaceValue = valueMap.get(rightStringValue.getValue());
if (StringUtils.isNotEmpty(replaceValue)) {
rightStringValue.setValue(replaceValue);
}
}
}

4
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FiledValueReplaceVisitor.java → common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FiledNameReplaceVisitor.java
View File

@@ -10,11 +10,11 @@ import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.schema.Column;
import org.springframework.util.CollectionUtils;
public class FiledValueReplaceVisitor extends ExpressionVisitorAdapter {
public class FiledNameReplaceVisitor extends ExpressionVisitorAdapter {
private Map<String, Set<String>> fieldValueToFieldNames;
public FiledValueReplaceVisitor(Map<String, Set<String>> fieldValueToFieldNames) {
public FiledNameReplaceVisitor(Map<String, Set<String>> fieldValueToFieldNames) {
this.fieldValueToFieldNames = fieldValueToFieldNames;
}

59
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionReplaceVisitor.java
View File

@@ -1,19 +1,14 @@
package com.tencent.supersonic.common.util.jsqlparser;
import com.tencent.supersonic.common.util.DatePeriodEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.StringUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
@@ -27,10 +22,6 @@ import org.apache.commons.collections.CollectionUtils;
@Slf4j
public class FunctionReplaceVisitor extends ExpressionVisitorAdapter {
public static final String DATE_FUNCTION = "datediff";
public static final double HALF_YEAR = 0.5d;
public static final int SIX_MONTH = 6;
public static final String EQUAL = "=";
private List<Expression> waitingForAdds = new ArrayList<>();
@Override
@@ -86,7 +77,7 @@ public class FunctionReplaceVisitor extends ExpressionVisitorAdapter {
return result;
}
Function leftExpressionFunction = (Function) leftExpression;
if (!leftExpressionFunction.toString().contains(DATE_FUNCTION)) {
if (!leftExpressionFunction.toString().contains(DateFunctionHelper.DATE_FUNCTION)) {
return result;
}
List<Expression> leftExpressions = leftExpressionFunction.getParameters().getExpressions();
@@ -96,18 +87,18 @@ public class FunctionReplaceVisitor extends ExpressionVisitorAdapter {
Column field = (Column) leftExpressions.get(1);
String columnName = field.getColumnName();
try {
String startDateValue = getStartDateStr(comparisonOperator, leftExpressions);
String endDateValue = getEndDateValue(leftExpressions);
String startDateValue = DateFunctionHelper.getStartDateStr(comparisonOperator, leftExpressions);
String endDateValue = DateFunctionHelper.getEndDateValue(leftExpressions);
String endDateOperator = comparisonOperator.getStringExpression();
String condExpr =
columnName + StringUtil.getSpaceWrap(getEndDateOperator(comparisonOperator))
columnName + StringUtil.getSpaceWrap(DateFunctionHelper.getEndDateOperator(comparisonOperator))
+ StringUtil.getCommaWrap(endDateValue);
ComparisonOperator expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
String startDataCondExpr =
columnName + StringUtil.getSpaceWrap(startDateOperator) + StringUtil.getCommaWrap(startDateValue);
if (EQUAL.equalsIgnoreCase(endDateOperator)) {
if (DateFunctionHelper.EQUAL.equalsIgnoreCase(endDateOperator)) {
result.add(CCJSqlParserUtil.parseCondExpression(condExpr));
expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(" 1 = 1 ");
}
@@ -128,44 +119,4 @@ public class FunctionReplaceVisitor extends ExpressionVisitorAdapter {
return null;
}
private String getStartDateStr(ComparisonOperator minorThanEquals, List<Expression> expressions) {
String unitValue = getUnit(expressions);
String dateValue = getEndDateValue(expressions);
String dateStr = "";
Expression rightExpression = minorThanEquals.getRightExpression();
DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(unitValue);
if (rightExpression instanceof DoubleValue) {
DoubleValue value = (DoubleValue) rightExpression;
double doubleValue = value.getValue();
if (DatePeriodEnum.YEAR.equals(datePeriodEnum) && doubleValue == HALF_YEAR) {
datePeriodEnum = DatePeriodEnum.MONTH;
dateStr = DateUtils.getBeforeDate(dateValue, SIX_MONTH, datePeriodEnum);
}
} else if (rightExpression instanceof LongValue) {
LongValue value = (LongValue) rightExpression;
long doubleValue = value.getValue();
dateStr = DateUtils.getBeforeDate(dateValue, (int) doubleValue, datePeriodEnum);
}
return dateStr;
}
private String getEndDateOperator(ComparisonOperator comparisonOperator) {
String operator = comparisonOperator.getStringExpression();
if (EQUAL.equalsIgnoreCase(operator)) {
operator = "<=";
}
return operator;
}
private String getEndDateValue(List<Expression> leftExpressions) {
StringValue date = (StringValue) leftExpressions.get(2);
return date.getValue();
}
private String getUnit(List<Expression> expressions) {
StringValue unit = (StringValue) expressions.get(0);
return unit.getValue();
}
}

20
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java
View File

@@ -28,16 +28,30 @@ import org.springframework.util.CollectionUtils;
@Slf4j
public class SqlParserUpdateHelper {
public static String replaceValueFields(String sql, Map<String, Set<String>> fieldValueToFieldNames) {
public static String replaceValue(String sql, Map<String, Map<String, String>> filedNameToValueMap) {
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
//1. replace where fields
Expression where = plainSelect.getWhere();
FiledValueReplaceVisitor visitor = new FiledValueReplaceVisitor(fieldValueToFieldNames);
FieldlValueReplaceVisitor visitor = new FieldlValueReplaceVisitor(filedNameToValueMap);
if (Objects.nonNull(where)) {
where.accept(visitor);
}
return selectStatement.toString();
}
public static String replaceFieldNameByValue(String sql, Map<String, Set<String>> fieldValueToFieldNames) {
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
Expression where = plainSelect.getWhere();
FiledNameReplaceVisitor visitor = new FiledNameReplaceVisitor(fieldValueToFieldNames);
if (Objects.nonNull(where)) {
where.accept(visitor);
}

23
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java
View File

@@ -19,6 +19,29 @@ class SqlParserSelectHelperTest {
void getWhereFilterExpression() {
List<FilterExpression> filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE "
+ "sys_imp_date = '2023-08-08' AND YEAR(publish_date) = 2023 "
+ " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' "
+ " AND YEAR(publish_date) = 2023 "
+ " AND MONTH(publish_date) = 8"
+ " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'"
+ " AND YEAR(publish_date) = 2023 "
+ " AND MONTH(publish_date) = 8 AND DAY(publish_date) =20 "
+ " AND user_id = 'alice' ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' "
+ " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");