topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.7.2 version backend update (#28)

Browse Source 
Co-authored-by: jipengli <jipengli@tencent.com>
This commit is contained in:
jipeli
2023-08-15 08:56:18 +08:00
committed by GitHub
parent 27283001a8
commit b1952d64ab
461 changed files with 18548 additions and 11939 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
common/pom.xml
View File

@@ -133,6 +133,11 @@
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency>
<groupId>com.github.jsqlparser</groupId>
<artifactId>jsqlparser</artifactId>
<version>${jsqlparser.version}</version>
</dependency>
</dependencies>

2
common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java
View File

@@ -1,10 +1,8 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import java.util.List;
import javax.validation.constraints.NotBlank;
import lombok.Data;
@Data

15
common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java
View File

@@ -5,7 +5,6 @@ import static java.time.LocalDate.now;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.Data;
@Data
@@ -41,12 +40,18 @@ public class DateConf {
/**
* the text parse from , example "last 7 days" , "last mouth"
*/
private String text;
private String detectWord;
private boolean isInherited;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
DateConf dateConf = (DateConf) o;
return dateMode == dateConf.dateMode &&
Objects.equals(startDate, dateConf.startDate) &&
@@ -87,7 +92,7 @@ public class DateConf {
sb.append(",\"period\":\"")
.append(period).append('\"');
sb.append(",\"text\":\"")
.append(text).append('\"');
.append(detectWord).append('\"');
sb.append('}');
return sb.toString();
}

1
common/src/main/java/com/tencent/supersonic/common/pojo/Order.java
View File

@@ -4,7 +4,6 @@ import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER;
import com.google.common.base.Objects;
import javax.validation.constraints.NotBlank;
import lombok.Data;
@Data

1
common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.common.pojo;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

7
common/src/main/java/com/tencent/supersonic/common/pojo/QueryColumn.java
View File

@@ -22,6 +22,13 @@ public class QueryColumn {
this.nameEn = nameEn;
}
public QueryColumn(String name, String type, String nameEn) {
this.name = name;
this.type = type;
this.nameEn = nameEn;
this.showType = "CATEGORY";
}
public void setType(String type) {
this.type = type == null ? null : type;
}

1
common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java
View File

@@ -7,6 +7,7 @@ import lombok.Data;
*/
@Data
public class ResultData<T> {
private int code;
private String msg;
private T data;

8
common/src/main/java/com/tencent/supersonic/common/pojo/enums/AuthType.java Normal file
View File

@@ -0,0 +1,8 @@
package com.tencent.supersonic.common.pojo.enums;
public enum AuthType {
VISIBLE,
ADMIN
}

17
common/src/main/java/com/tencent/supersonic/common/util/DatePeriodEnum.java Normal file
View File

@@ -0,0 +1,17 @@
package com.tencent.supersonic.common.util;
public enum DatePeriodEnum {
DAY,
WEEK,
MONTH,
YEAR;
public static DatePeriodEnum get(String period) {
for (DatePeriodEnum value : values()) {
if (value.name().equalsIgnoreCase(period)) {
return value;
}
}
return null;
}
}

52
common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
View File

@@ -1,17 +1,23 @@
package com.tencent.supersonic.common.util;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.Date;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DateUtils {
public static Integer currentYear(){
public static final String DATE_FORMAT_DOT = "yyyy-MM-dd";
public static Integer currentYear() {
Date date = new Date();
SimpleDateFormat dateFormat= new SimpleDateFormat("yyyy-MM-dd");
String time=dateFormat.format(date).replaceAll("-","");
int year = Integer.parseInt(time.substring(0,4));
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT);
String time = dateFormat.format(date).replaceAll("-", "");
int year = Integer.parseInt(time.substring(0, 4));
return year;
}
@@ -33,11 +39,47 @@ public class DateUtils {
String format = formats[i];
try {
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(format);
LocalDateTime.parse(date,dateTimeFormatter);
LocalDateTime.parse(date, dateTimeFormatter);
return dateTimeFormatter;
} catch (Exception e) {
}
}
return DateTimeFormatter.ofPattern(formats[0]);
}
public static String getBeforeDate(int intervalDay) {
Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date());
calendar.add(Calendar.DAY_OF_MONTH, -intervalDay);
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT);
return dateFormat.format(calendar.getTime());
}
public static String getBeforeDate(String date, int intervalDay, DatePeriodEnum datePeriodEnum) {
Calendar calendar = Calendar.getInstance();
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT);
try {
calendar.setTime(dateFormat.parse(date));
} catch (ParseException e) {
log.error("parse error");
}
switch (datePeriodEnum) {
case DAY:
calendar.set(Calendar.DATE, calendar.get(Calendar.DATE) - intervalDay);
break;
case WEEK:
calendar.set(Calendar.DATE, calendar.get(Calendar.DATE) - intervalDay * 7);
break;
case MONTH:
calendar.set(Calendar.MONTH, calendar.get(Calendar.MONTH) - intervalDay);
break;
case YEAR:
calendar.set(Calendar.YEAR, calendar.get(Calendar.YEAR) - intervalDay);
break;
default:
}
return dateFormat.format(calendar.getTime());
}
}

3
common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java
View File

@@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.InvalidParameterException;
@@ -19,8 +20,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;

21
common/src/main/java/com/tencent/supersonic/common/util/StringUtil.java Normal file
View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.common.util;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class StringUtil {
public static final String COMMA_WRAPPER = "'%s'";
public static final String SPACE_WRAPPER = " %s ";
public static String getCommaWrap(String value) {
return String.format(COMMA_WRAPPER, value);
}
public static String getSpaceWrap(String value) {
return String.format(SPACE_WRAPPER, value);
}
}

2
common/src/main/java/com/tencent/supersonic/common/util/ThreadContext.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.common.util;
import java.util.Map;
import lombok.Builder;
import lombok.Data;
import lombok.ToString;
import java.util.Map;
@Builder
@ToString

6
common/src/main/java/com/tencent/supersonic/common/util/calcite/SqlParseUtils.java
View File

@@ -80,10 +80,15 @@ public class SqlParseUtils {
*/
private static void handlerOrderBy(SqlNode node, SqlParserInfo sqlParserInfo) {
SqlOrderBy sqlOrderBy = (SqlOrderBy) node;
SqlNode query = sqlOrderBy.query;
handlerSQL(query, sqlParserInfo);
SqlNodeList orderList = sqlOrderBy.orderList;
Set<String> orderFields = handlerField(orderList);
sqlParserInfo.getAllFields().addAll(orderFields);
}
@@ -107,6 +112,7 @@ public class SqlParseUtils {
Set<String> selectFields = handlerSelectField(sqlSelect);
allFields.addAll(selectFields);
}
private static Set<String> handlerSelectField(SqlSelect sqlSelect) {

19
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/AggregateFunctionVisitor.java Normal file
View File

@@ -0,0 +1,19 @@
package com.tencent.supersonic.common.util.jsqlparser;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
public class AggregateFunctionVisitor extends ExpressionVisitorAdapter {
private boolean hasAggregateFunction = false;
public boolean hasAggregateFunction() {
return hasAggregateFunction;
}
@Override
public void visit(Function function) {
super.visit(function);
hasAggregateFunction = true;
}
}

290
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/CCJSqlParserUtils.java Normal file
View File

@@ -0,0 +1,290 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SelectBody;
import net.sf.jsqlparser.statement.select.SelectItem;
import net.sf.jsqlparser.util.SelectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
/**
* CC JSql ParserUtils
*/
@Slf4j
public class CCJSqlParserUtils {
public static List<String> getWhereFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<String> result = new HashSet<>();
Expression where = plainSelect.getWhere();
where.accept(new FieldAcquireVisitor(result));
return new ArrayList<>(result);
}
public static List<String> getSelectFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
return new ArrayList<>(getSelectFields(plainSelect));
}
private static Set<String> getSelectFields(PlainSelect plainSelect) {
List<SelectItem> selectItems = plainSelect.getSelectItems();
Set<String> result = new HashSet<>();
for (SelectItem selectItem : selectItems) {
selectItem.accept(new FieldAcquireVisitor(result));
}
return result;
}
private static PlainSelect getPlainSelect(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return null;
}
return (PlainSelect) selectBody;
}
private static Select getSelect(String sql) {
Statement statement = null;
try {
statement = CCJSqlParserUtil.parse(sql);
} catch (JSQLParserException e) {
log.error("parse error", e);
return null;
}
if (!(statement instanceof Select)) {
return null;
}
return (Select) statement;
}
public static List<String> getAllFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<String> result = getSelectFields(plainSelect);
GroupByElement groupBy = plainSelect.getGroupBy();
if (groupBy != null) {
List<Expression> groupByExpressions = groupBy.getGroupByExpressions();
for (Expression expression : groupByExpressions) {
if (expression instanceof Column) {
Column column = (Column) expression;
result.add(column.getColumnName());
}
}
}
List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
if (orderByElements != null) {
for (OrderByElement orderByElement : orderByElements) {
Expression expression = orderByElement.getExpression();
if (expression instanceof Column) {
Column column = (Column) expression;
result.add(column.getColumnName());
}
}
}
Expression where = plainSelect.getWhere();
if (where != null) {
where.accept(new ExpressionVisitorAdapter() {
@Override
public void visit(Column column) {
result.add(column.getColumnName());
}
});
}
return new ArrayList<>(result);
}
public static String replaceFields(String sql, Map<String, String> fieldToBizName) {
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
//1. replace where fields
Expression where = plainSelect.getWhere();
FieldReplaceVisitor visitor = new FieldReplaceVisitor(fieldToBizName);
if (Objects.nonNull(where)) {
where.accept(visitor);
}
//2. replace select fields
for (SelectItem selectItem : plainSelect.getSelectItems()) {
selectItem.accept(visitor);
}
//3. replace oder by fields
List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
if (!CollectionUtils.isEmpty(orderByElements)) {
for (OrderByElement orderByElement : orderByElements) {
orderByElement.accept(new OrderByReplaceVisitor(fieldToBizName));
}
}
//4. replace group by fields
GroupByElement groupByElement = plainSelect.getGroupBy();
if (Objects.nonNull(groupByElement)) {
groupByElement.accept(new GroupByReplaceVisitor(fieldToBizName));
}
//5. add Waiting Expression
List<Expression> waitingForAdds = visitor.getWaitingForAdds();
addWaitingExpression(plainSelect, where, waitingForAdds);
return selectStatement.toString();
}
private static void addWaitingExpression(PlainSelect plainSelect, Expression where,
List<Expression> waitingForAdds) {
if (CollectionUtils.isEmpty(waitingForAdds)) {
return;
}
for (Expression expression : waitingForAdds) {
if (where == null) {
plainSelect.setWhere(expression);
} else {
plainSelect.setWhere(new AndExpression(where, expression));
}
}
}
public static String addFieldsToSelect(String sql, List<String> fields) {
Select selectStatement = getSelect(sql);
// add fields to select
for (String field : fields) {
SelectUtils.addExpression(selectStatement, new Column(field));
}
return selectStatement.toString();
}
public static String getTableName(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
SelectBody selectBody = selectStatement.getSelectBody();
PlainSelect plainSelect = (PlainSelect) selectBody;
Table table = (Table) plainSelect.getFromItem();
return table.getName();
}
public static String replaceTable(String sql, String tableName) {
if (StringUtils.isEmpty(tableName)) {
return sql;
}
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
PlainSelect plainSelect = (PlainSelect) selectBody;
// replace table name
Table table = (Table) plainSelect.getFromItem();
table.setName(tableName);
return selectStatement.toString();
}
public static String addWhere(String sql, String column, Object value) {
if (StringUtils.isEmpty(column) || Objects.isNull(value)) {
return sql;
}
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
Expression where = plainSelect.getWhere();
Expression right = new StringValue(value.toString());
if (value instanceof Integer || value instanceof Long) {
right = new LongValue(value.toString());
}
if (where == null) {
plainSelect.setWhere(new EqualsTo(new Column(column), right));
} else {
plainSelect.setWhere(new AndExpression(where, new EqualsTo(new Column(column), right)));
}
return selectStatement.toString();
}
public static String addWhere(String sql, Expression expression) {
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
Expression where = plainSelect.getWhere();
if (where == null) {
plainSelect.setWhere(expression);
} else {
plainSelect.setWhere(new AndExpression(where, expression));
}
return selectStatement.toString();
}
public static boolean hasAggregateFunction(String sql) {
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return false;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
List<SelectItem> selectItems = plainSelect.getSelectItems();
AggregateFunctionVisitor visitor = new AggregateFunctionVisitor();
for (SelectItem selectItem : selectItems) {
selectItem.accept(visitor);
}
return visitor.hasAggregateFunction();
}
}

20
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAcquireVisitor.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Set;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.schema.Column;
public class FieldAcquireVisitor extends ExpressionVisitorAdapter {
private Set<String> fields;
public FieldAcquireVisitor(Set<String> fields) {
this.fields = fields;
}
@Override
public void visit(Column column) {
String columnName = column.getColumnName();
fields.add(columnName);
}
}

70
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldReplaceVisitor.java Normal file
View File

@@ -0,0 +1,70 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
@Slf4j
public class FieldReplaceVisitor extends ExpressionVisitorAdapter {
ParseVisitorHelper parseVisitorHelper = new ParseVisitorHelper();
private Map<String, String> fieldToBizName;
private List<Expression> waitingForAdds = new ArrayList<>();
public FieldReplaceVisitor(Map<String, String> fieldToBizName) {
this.fieldToBizName = fieldToBizName;
}
@Override
public void visit(Column column) {
parseVisitorHelper.replaceColumn(column, fieldToBizName);
}
@Override
public void visit(MinorThan expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, ">");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
@Override
public void visit(MinorThanEquals expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, ">=");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
@Override
public void visit(GreaterThan expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, "<");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
@Override
public void visit(GreaterThanEquals expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, "<=");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
public List<Expression> getWaitingForAdds() {
return waitingForAdds;
}
}

36
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByReplaceVisitor.java Normal file
View File

@@ -0,0 +1,36 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import java.util.Map;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.GroupByVisitor;
import org.apache.commons.lang3.StringUtils;
public class GroupByReplaceVisitor implements GroupByVisitor {
ParseVisitorHelper parseVisitorHelper = new ParseVisitorHelper();
private Map<String, String> fieldToBizName;
public GroupByReplaceVisitor(Map<String, String> fieldToBizName) {
this.fieldToBizName = fieldToBizName;
}
public void visit(GroupByElement groupByElement) {
groupByElement.getGroupByExpressionList();
ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList();
List<Expression> groupByExpressions = groupByExpressionList.getExpressions();
for (int i = 0; i < groupByExpressions.size(); i++) {
Expression expression = groupByExpressions.get(i);
String replaceColumn = parseVisitorHelper.getReplaceColumn(expression.toString(), fieldToBizName);
if (StringUtils.isNotEmpty(replaceColumn)) {
groupByExpressions.set(i, new Column(replaceColumn));
}
}
}
}

26
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByReplaceVisitor.java Normal file
View File

@@ -0,0 +1,26 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Map;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter;
public class OrderByReplaceVisitor extends OrderByVisitorAdapter {
ParseVisitorHelper parseVisitorHelper = new ParseVisitorHelper();
private Map<String, String> fieldToBizName;
public OrderByReplaceVisitor(Map<String, String> fieldToBizName) {
this.fieldToBizName = fieldToBizName;
}
@Override
public void visit(OrderByElement orderBy) {
Expression expression = orderBy.getExpression();
if (expression instanceof Column) {
parseVisitorHelper.replaceColumn((Column) expression, fieldToBizName);
}
super.visit(orderBy);
}
}

168
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/ParseVisitorHelper.java Normal file
View File

@@ -0,0 +1,168 @@
package com.tencent.supersonic.common.util.jsqlparser;
import com.tencent.supersonic.common.util.DatePeriodEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.StringUtil;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class ParseVisitorHelper {
public static final double HALF_YEAR = 0.5d;
public static final int SIX_MONTH = 6;
public static final String DATE_FUNCTION = "datediff";
public void replaceColumn(Column column, Map<String, String> fieldToBizName) {
String columnName = column.getColumnName();
column.setColumnName(getReplaceColumn(columnName, fieldToBizName));
}
public String getReplaceColumn(String columnName, Map<String, String> fieldToBizName) {
String fieldBizName = fieldToBizName.get(columnName);
if (StringUtils.isNotEmpty(fieldBizName)) {
return fieldBizName;
} else {
Optional<Entry<String, String>> first = fieldToBizName.entrySet().stream().sorted((k1, k2) -> {
String k1FieldNameDb = k1.getKey();
String k2FieldNameDb = k2.getKey();
Double k1Similarity = getSimilarity(columnName, k1FieldNameDb);
Double k2Similarity = getSimilarity(columnName, k2FieldNameDb);
return k2Similarity.compareTo(k1Similarity);
}).collect(Collectors.toList()).stream().findFirst();
if (first.isPresent()) {
return first.get().getValue();
}
}
return columnName;
}
public Expression reparseDate(ComparisonOperator comparisonOperator, Map<String, String> fieldToBizName,
String startDateOperator) {
Expression leftExpression = comparisonOperator.getLeftExpression();
if (leftExpression instanceof Column) {
Column leftExpressionColumn = (Column) leftExpression;
replaceColumn(leftExpressionColumn, fieldToBizName);
return null;
}
if (!(leftExpression instanceof Function)) {
return null;
}
Function leftExpressionFunction = (Function) leftExpression;
if (!leftExpressionFunction.toString().contains(DATE_FUNCTION)) {
return null;
}
List<Expression> leftExpressions = leftExpressionFunction.getParameters().getExpressions();
if (CollectionUtils.isEmpty(leftExpressions) || leftExpressions.size() < 3) {
return null;
}
Column field = (Column) leftExpressions.get(1);
String columnName = field.getColumnName();
String startDateValue = getStartDateStr(comparisonOperator, leftExpressions);
String fieldBizName = fieldToBizName.get(columnName);
try {
String endDateValue = getEndDateValue(leftExpressions);
String stringExpression = comparisonOperator.getStringExpression();
String condExpr =
fieldBizName + StringUtil.getSpaceWrap(stringExpression) + StringUtil.getCommaWrap(endDateValue);
ComparisonOperator expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
comparisonOperator.setLeftExpression(null);
comparisonOperator.setRightExpression(null);
comparisonOperator.setASTNode(null);
comparisonOperator.setLeftExpression(expression.getLeftExpression());
comparisonOperator.setRightExpression(expression.getRightExpression());
comparisonOperator.setASTNode(expression.getASTNode());
String startDataCondExpr =
fieldBizName + StringUtil.getSpaceWrap(startDateOperator) + StringUtil.getCommaWrap(startDateValue);
return CCJSqlParserUtil.parseCondExpression(startDataCondExpr);
} catch (JSQLParserException e) {
log.error("JSQLParserException", e);
}
return null;
}
private String getEndDateValue(List<Expression> leftExpressions) {
StringValue date = (StringValue) leftExpressions.get(2);
return date.getValue();
}
private String getStartDateStr(ComparisonOperator minorThanEquals, List<Expression> expressions) {
String unitValue = getUnit(expressions);
String dateValue = getEndDateValue(expressions);
String dateStr = "";
Expression rightExpression = minorThanEquals.getRightExpression();
DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(unitValue);
if (rightExpression instanceof DoubleValue) {
DoubleValue value = (DoubleValue) rightExpression;
double doubleValue = value.getValue();
if (DatePeriodEnum.YEAR.equals(datePeriodEnum) && doubleValue == HALF_YEAR) {
datePeriodEnum = DatePeriodEnum.MONTH;
dateStr = DateUtils.getBeforeDate(dateValue, SIX_MONTH, datePeriodEnum);
}
} else if (rightExpression instanceof LongValue) {
LongValue value = (LongValue) rightExpression;
long doubleValue = value.getValue();
dateStr = DateUtils.getBeforeDate(dateValue, (int) doubleValue, datePeriodEnum);
}
return dateStr;
}
private String getUnit(List<Expression> expressions) {
StringValue unit = (StringValue) expressions.get(0);
return unit.getValue();
}
public static int editDistance(String word1, String word2) {
final int m = word1.length();
final int n = word2.length();
int[][] dp = new int[m + 1][n + 1];
for (int j = 0; j <= n; ++j) {
dp[0][j] = j;
}
for (int i = 0; i <= m; ++i) {
dp[i][0] = i;
}
for (int i = 1; i <= m; ++i) {
char ci = word1.charAt(i - 1);
for (int j = 1; j <= n; ++j) {
char cj = word2.charAt(j - 1);
if (ci == cj) {
dp[i][j] = dp[i - 1][j - 1];
} else if (i > 1 && j > 1 && ci == word2.charAt(j - 2) && cj == word1.charAt(i - 2)) {
dp[i][j] = 1 + Math.min(dp[i - 2][j - 2], Math.min(dp[i][j - 1], dp[i - 1][j]));
} else {
dp[i][j] = Math.min(dp[i - 1][j - 1] + 1, Math.min(dp[i][j - 1] + 1, dp[i - 1][j] + 1));
}
}
}
return dp[m][n];
}
public double getSimilarity(String word1, String word2) {
return 1 - (double) editDistance(word1, word2) / Math.max(word2.length(), word1.length());
}
}

26
common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java Normal file
View File

@@ -0,0 +1,26 @@
package com.tencent.supersonic.common.util;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
class DateUtilsTest {
@Test
void getBeforeDate() {
String dateStr = DateUtils.getBeforeDate("2023-08-10", 1, DatePeriodEnum.DAY);
Assert.assertEquals(dateStr, "2023-08-09");
dateStr = DateUtils.getBeforeDate("2023-08-10", 8, DatePeriodEnum.DAY);
Assert.assertEquals(dateStr, "2023-08-02");
dateStr = DateUtils.getBeforeDate("2023-08-10", 1, DatePeriodEnum.WEEK);
Assert.assertEquals(dateStr, "2023-08-03");
dateStr = DateUtils.getBeforeDate("2023-08-01", 1, DatePeriodEnum.MONTH);
Assert.assertEquals(dateStr, "2023-07-01");
dateStr = DateUtils.getBeforeDate("2023-08-01", 1, DatePeriodEnum.YEAR);
Assert.assertEquals(dateStr, "2022-08-01");
}
}

2
common/src/test/java/com/tencent/supersonic/common/util/calcite/SqlParseUtilsTest.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.common.util.calcite;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.sql.parser.SqlParseException;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
@@ -12,6 +13,7 @@ import org.junit.jupiter.api.Test;
*
* @date 2023/7/12 12:00
*/
@Slf4j
class SqlParseUtilsTest {
@Test

177
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/CCJSqlParserUtilsTest.java Normal file
View File

@@ -0,0 +1,177 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
/**
* CCJSqlParserUtils Test
*/
class CCJSqlParserUtilsTest {
@Test
void replaceFields() {
Map<String, String> fieldToBizName = initParams();
String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01' order by 播放量 desc limit 11";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') <= 0.5 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') >= 0.5 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' order by 访问次数 desc limit 1",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceTable(replaceSql, "s2");
replaceSql = CCJSqlParserUtils.addFieldsToSelect(replaceSql, Collections.singletonList("field_a"));
replaceSql = CCJSqlParserUtils.replaceFields(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1",
fieldToBizName);
Assert.assertEquals(replaceSql,
"SELECT department, sum(pv) FROM 超音数 WHERE sys_imp_date = '2023-08-08' AND user_id = user_id AND publish_date = '11' GROUP BY department LIMIT 1");
replaceSql = "select sum(访问次数) from 超音数 where 数据日期 >= '2023-08-06' and 数据日期 <= '2023-08-06' and 部门 = 'hr'";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
System.out.println(replaceSql);
}
@Test
void getAllFields() {
List<String> allFields = CCJSqlParserUtils.getAllFields(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
Assert.assertEquals(allFields.size(), 6);
allFields = CCJSqlParserUtils.getAllFields(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date >= '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
Assert.assertEquals(allFields.size(), 6);
allFields = CCJSqlParserUtils.getAllFields(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1");
Assert.assertEquals(allFields.size(), 5);
}
@Test
void replaceTable() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
String replaceSql = CCJSqlParserUtils.replaceTable(sql, "s2");
Assert.assertEquals(
"SELECT 部门, sum(访问次数) FROM s2 WHERE 数据日期 = '2023-08-08' AND 用户 = alice AND 发布日期 = '11' GROUP BY 部门 LIMIT 1",
replaceSql);
}
@Test
void getSelectFields() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
List<String> selectFields = CCJSqlParserUtils.getSelectFields(sql);
Assert.assertEquals(selectFields.contains("访问次数"), true);
Assert.assertEquals(selectFields.contains("部门"), true);
}
@Test
void getWhereFields() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1";
List<String> selectFields = CCJSqlParserUtils.getWhereFields(sql);
Assert.assertEquals(selectFields.contains("发布日期"), true);
Assert.assertEquals(selectFields.contains("数据日期"), true);
Assert.assertEquals(selectFields.contains("用户"), true);
}
@Test
void addWhere() throws JSQLParserException {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
sql = CCJSqlParserUtils.addWhere(sql, "column_a", 123444555);
List<String> selectFields = CCJSqlParserUtils.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_a"), true);
sql = CCJSqlParserUtils.addWhere(sql, "column_b", "123456666");
selectFields = CCJSqlParserUtils.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_b"), true);
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( column_c = 111 or column_d = 1111)");
sql = CCJSqlParserUtils.addWhere(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1",
expression);
Assert.assertEquals(sql.contains("column_c = 111"), true);
}
@Test
void hasAggregateFunction() throws JSQLParserException {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
boolean hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "SELECT count(1) FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, false);
sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11'";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, false);
}
private Map<String, String> initParams() {
Map<String, String> fieldToBizName = new HashMap<>();
fieldToBizName.put("部门", "department");
fieldToBizName.put("用户", "user_id");
fieldToBizName.put("数据日期", "sys_imp_date");
fieldToBizName.put("发布日期", "publish_date");
fieldToBizName.put("访问次数", "pv");
fieldToBizName.put("歌曲名", "song_name");
fieldToBizName.put("歌手名", "singer_name");
fieldToBizName.put("播放", "play_count");
fieldToBizName.put("歌曲发布时间", "song_publis_date");
fieldToBizName.put("歌曲发布年份", "song_publis_year");
fieldToBizName.put("转3.0前后30天结算份额衰减", "fdafdfdsa_fdas");
return fieldToBizName;
}
}