topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[release][project] supersonic 0.7.3 version backend update (#40)

Browse Source 
* [improvement] add some features

* [improvement] revise CHANGELOG

---------

Co-authored-by: zuopengge <hwzuopengge@tencent.com>
This commit is contained in:
mainmain
2023-08-29 20:06:34 +08:00
committed by GitHub
parent 6fe9ab79ed
commit e1911bc81b
260 changed files with 6466 additions and 7108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java
View File

@@ -53,11 +53,11 @@ public class DateConf {
return false;
}
DateConf dateConf = (DateConf) o;
return dateMode == dateConf.dateMode &&
Objects.equals(startDate, dateConf.startDate) &&
Objects.equals(endDate, dateConf.endDate) &&
Objects.equals(unit, dateConf.unit) &&
Objects.equals(period, dateConf.period);
return dateMode == dateConf.dateMode
&& Objects.equals(startDate, dateConf.startDate)
&& Objects.equals(endDate, dateConf.endDate)
&& Objects.equals(unit, dateConf.unit)
&& Objects.equals(period, dateConf.period);
}
@Override

4
common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java
View File

@@ -16,4 +16,8 @@ public class QueryAuthorization {
private List<String> dimensionFilters;
private List<String> dimensionFiltersDesc;
private String message;
public QueryAuthorization(String message) {
this.message = message;
}
}

107
common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java
View File

@@ -33,9 +33,9 @@ public class ChatGptHelper {
private Integer proxyPort;
public ChatGPT getChatGPT(){
public ChatGPT getChatGPT() {
Proxy proxy = null;
if (!"default".equals(proxyIp)){
if (!"default".equals(proxyIp)) {
proxy = Proxys.http(proxyIp, proxyPort);
}
return ChatGPT.builder()
@@ -47,8 +47,8 @@ public class ChatGptHelper {
.init();
}
public Message getChatCompletion(Message system,Message message){
ChatCompletion chatCompletion = ChatCompletion.builder()
public Message getChatCompletion(Message system, Message message) {
ChatCompletion chatCompletion = ChatCompletion.builder()
.model(ChatCompletion.Model.GPT_3_5_TURBO_16K.getName())
.messages(Arrays.asList(system, message))
.maxTokens(10000)
@@ -58,41 +58,61 @@ public class ChatGptHelper {
return response.getChoices().get(0).getMessage();
}
public String inferredTime(String queryText){
public String inferredTime(String queryText) {
long nowTime = System.currentTimeMillis();
Date date = new Date(nowTime);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String formattedDate = sdf.format(date);
Message system = Message.ofSystem("现在时间 "+formattedDate+"，你是一个专业的数据分析师，你的任务是基于数据，专业的解答用户的问题。" +
"你需要遵守以下规则：\n" +
"1.返回规范的数据格式，json，如： 输入：近 10 天的日活跃数，输出：{\"start\":\"2023-07-21\",\"end\":\"2023-07-31\"}" +
"2.你对时间数据要求规范，能从近 10 天，国庆节，端午节，获取到相应的时间，填写到 json 中。\n"+
"3.你的数据时间，只有当前及之前时间即可,超过则回复去年\n" +
"4.只需要解析出时间，时间可以是时间月和年或日、日历采用公历\n"+
"5.时间给出要是绝对正确，不能瞎编\n"
Message system = Message.ofSystem("现在时间 " + formattedDate + "，你是一个专业的数据分析师，你的任务是基于数据，专业的解答用户的问题。"
+ "你需要遵守以下规则：\n"
+ "1.返回规范的数据格式，json，如： 输入：近 10 天的日活跃数，输出：{\"start\":\"2023-07-21\",\"end\":\"2023-07-31\"}"
+ "2.你对时间数据要求规范，能从近 10 天，国庆节，端午节，获取到相应的时间，填写到 json 中。\n"
+ "3.你的数据时间，只有当前及之前时间即可,超过则回复去年\n"
+ "4.只需要解析出时间，时间可以是时间月和年或日、日历采用公历\n"
+ "5.时间给出要是绝对正确，不能瞎编\n"
);
Message message = Message.of("输入："+queryText+"，输出：");
Message message = Message.of("输入：" + queryText + "，输出：");
Message res = getChatCompletion(system, message);
return res.getContent();
}
public String mockAlias(String mockType,String name,String bizName,String table,String desc,Boolean isPercentage){
String msg = "Assuming you are a professional data analyst specializing in indicators, you have a vast amount of data analysis indicator content. You are familiar with the basic format of the content,Now, Construct your answer Based on the following json-schema.\n" +
"{\n" +
"\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n" +
"\"type\": \"array\",\n" +
"\"minItems\": 2,\n" +
"\"maxItems\": 4,\n" +
"\"items\": {\n" +
"\"type\": \"string\",\n" +
"\"description\": \"Assuming you are a data analyst and give a defined "+mockType+" name: " +name+","+
"this "+mockType+" is from database and table: "+table+ ",This "+mockType+" calculates the field source: "+bizName+", The description of this indicator is: "+desc+", provide some aliases for this，please take chinese or english,but more chinese and Not repeating，\"\n" +
"},\n" +
"\"additionalProperties\":false}\n" +
"Please double-check whether the answer conforms to the format described in the JSON-schema.\n" +
"ANSWER JSON:";
log.info("msg:{}",msg);
public String mockAlias(String mockType,
String name,
String bizName,
String table,
String desc,
Boolean isPercentage) {
String msg = "Assuming you are a professional data analyst specializing in indicators, "
+ "you have a vast amount of data analysis indicator content. You are familiar with the basic"
+ " format of the content,Now, Construct your answer Based on the following json-schema.\n"
+ "{\n"
+ "\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n"
+ "\"type\": \"array\",\n"
+ "\"minItems\": 2,\n"
+ "\"maxItems\": 4,\n"
+ "\"items\": {\n"
+ "\"type\": \"string\",\n"
+ "\"description\": \"Assuming you are a data analyst and give a defined "
+ mockType
+ " name: "
+ name + ","
+ "this "
+ mockType
+ " is from database and table: "
+ table + ",This "
+ mockType
+ " calculates the field source: "
+ bizName
+ ", The description of this indicator is: "
+ desc
+ ", provide some aliases for this，please take chinese or english,"
+ "but more chinese and Not repeating，\"\n"
+ "},\n"
+ "\"additionalProperties\":false}\n"
+ "Please double-check whether the answer conforms to the format described in the JSON-schema.\n"
+ "ANSWER JSON:";
log.info("msg:{}", msg);
Message system = Message.ofSystem("");
Message message = Message.of(msg);
Message res = getChatCompletion(system, message);
@@ -100,17 +120,19 @@ public class ChatGptHelper {
}
public String mockDimensionValueAlias(String json){
String msg = "Assuming you are a professional data analyst specializing in indicators,for you a json list，" +
"the required content to follow is as follows: " +
"1. The format of JSON," +
"2. Only return in JSON format," +
"3. the array item > 1 and < 5,more alias," +
"for example：input:[\"qq_music\",\"kugou_music\"],out:{\"tran\":[\"qq音乐\",\"酷狗音乐\"],\"alias\":{\"qq_music\":[\"q音\",\"qq音乐\"],\"kugou_music\":[\"kugou\",\"酷狗\"]}}," +
"now input: " + json + ","+
"answer json:";
log.info("msg:{}",msg);
public String mockDimensionValueAlias(String json) {
String msg = "Assuming you are a professional data analyst specializing in indicators,for you a json list，"
+ "the required content to follow is as follows: "
+ "1. The format of JSON,"
+ "2. Only return in JSON format,"
+ "3. the array item > 1 and < 5,more alias,"
+ "for example：input:[\"qq_music\",\"kugou_music\"],"
+ "out:{\"tran\":[\"qq音乐\",\"酷狗音乐\"],\"alias\":{\"qq_music\":[\"q音\",\"qq音乐\"],"
+ "\"kugou_music\":[\"kugou\",\"酷狗\"]}},"
+ "now input: "
+ json + ","
+ "answer json:";
log.info("msg:{}", msg);
Message system = Message.ofSystem("");
Message message = Message.of(msg);
Message res = getChatCompletion(system, message);
@@ -118,12 +140,9 @@ public class ChatGptHelper {
}
public static void main(String[] args) {
ChatGptHelper chatGptHelper = new ChatGptHelper();
System.out.println(chatGptHelper.mockAlias("","","","","",false));
System.out.println(chatGptHelper.mockAlias("", "", "", "", "", false));
}

2
common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
View File

@@ -31,6 +31,7 @@ public class DateUtils {
dateFormat.parse(date);
return DateTimeFormatter.ofPattern(format);
} catch (Exception e) {
log.info("date parse has a exception:{}", e.toString());
}
}
return DateTimeFormatter.ofPattern(formats[0]);
@@ -44,6 +45,7 @@ public class DateUtils {
LocalDateTime.parse(date, dateTimeFormatter);
return dateTimeFormatter;
} catch (Exception e) {
log.info("date parse has a exception:{}", e.toString());
}
}
return DateTimeFormatter.ofPattern(formats[0]);

84
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java Normal file
View File

@@ -0,0 +1,84 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Set;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter {
private Set<FilterExpression> filterExpressions;
public FieldAndValueAcquireVisitor(Set<FilterExpression> filterExpressions) {
this.filterExpressions = filterExpressions;
}
@Override
public void visit(MinorThan expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
}
@Override
public void visit(EqualsTo expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
}
@Override
public void visit(MinorThanEquals expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
}
@Override
public void visit(GreaterThan expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
}
@Override
public void visit(GreaterThanEquals expr) {
FilterExpression filterExpression = getFilterExpression(expr);
filterExpressions.add(filterExpression);
}
private FilterExpression getFilterExpression(ComparisonOperator expr) {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
FilterExpression filterExpression = new FilterExpression();
String columnName = null;
if (leftExpression instanceof Column) {
Column column = (Column) leftExpression;
columnName = column.getColumnName();
filterExpression.setFieldName(columnName);
}
if (rightExpression instanceof StringValue) {
StringValue stringValue = (StringValue) rightExpression;
filterExpression.setFieldValue(stringValue.getValue());
}
if (rightExpression instanceof DoubleValue) {
DoubleValue doubleValue = (DoubleValue) rightExpression;
filterExpression.setFieldValue(doubleValue.getValue());
}
if (rightExpression instanceof LongValue) {
LongValue longValue = (LongValue) rightExpression;
filterExpression.setFieldValue(longValue.getValue());
}
filterExpression.setOperator(expr.getStringExpression());
return filterExpression;
}
}

46
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FiledValueReplaceVisitor.java Normal file
View File

@@ -0,0 +1,46 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.schema.Column;
import org.springframework.util.CollectionUtils;
public class FiledValueReplaceVisitor extends ExpressionVisitorAdapter {
private Map<String, Set<String>> fieldValueToFieldNames;
public FiledValueReplaceVisitor(Map<String, Set<String>> fieldValueToFieldNames) {
this.fieldValueToFieldNames = fieldValueToFieldNames;
}
@Override
public void visit(EqualsTo expr) {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
if (!(rightExpression instanceof StringValue)) {
return;
}
if (!(leftExpression instanceof Column)) {
return;
}
if (CollectionUtils.isEmpty(fieldValueToFieldNames)) {
return;
}
if (Objects.isNull(rightExpression) || Objects.isNull(leftExpression)) {
return;
}
Column leftColumnName = (Column) leftExpression;
StringValue rightStringValue = (StringValue) rightExpression;
Set<String> fieldNames = fieldValueToFieldNames.get(rightStringValue.getValue());
if (!CollectionUtils.isEmpty(fieldNames) && !fieldNames.contains(leftColumnName.getColumnName())) {
leftColumnName.setColumnName(fieldNames.stream().findFirst().get());
}
}
}

14
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FilterExpression.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.common.util.jsqlparser;
import lombok.Data;
@Data
public class FilterExpression {
private String operator;
private String fieldName;
private Object fieldValue;
}

25
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByAcquireVisitor.java Normal file
View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Set;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter;
public class OrderByAcquireVisitor extends OrderByVisitorAdapter {
private Set<String> fields;
public OrderByAcquireVisitor(Set<String> fields) {
this.fields = fields;
}
@Override
public void visit(OrderByElement orderBy) {
Expression expression = orderBy.getExpression();
if (expression instanceof Column) {
fields.add(((Column) expression).getColumnName());
}
super.visit(orderBy);
}
}

188
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java Normal file
View File

@@ -0,0 +1,188 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SelectBody;
import net.sf.jsqlparser.statement.select.SelectItem;
import org.springframework.util.CollectionUtils;
/**
* Sql Parser Select Helper
*/
@Slf4j
public class SqlParserSelectHelper {
public static List<FilterExpression> getFilterExpression(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<FilterExpression> result = new HashSet<>();
Expression where = plainSelect.getWhere();
if (Objects.nonNull(where)) {
where.accept(new FieldAndValueAcquireVisitor(result));
}
return new ArrayList<>(result);
}
public static List<String> getWhereFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<String> result = new HashSet<>();
Expression where = plainSelect.getWhere();
if (Objects.nonNull(where)) {
where.accept(new FieldAcquireVisitor(result));
}
return new ArrayList<>(result);
}
public static List<String> getOrderByFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<String> result = new HashSet<>();
List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
if (!CollectionUtils.isEmpty(orderByElements)) {
for (OrderByElement orderByElement : orderByElements) {
orderByElement.accept(new OrderByAcquireVisitor(result));
}
}
return new ArrayList<>(result);
}
public static List<String> getSelectFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
return new ArrayList<>(getSelectFields(plainSelect));
}
public static Set<String> getSelectFields(PlainSelect plainSelect) {
List<SelectItem> selectItems = plainSelect.getSelectItems();
Set<String> result = new HashSet<>();
for (SelectItem selectItem : selectItems) {
selectItem.accept(new FieldAcquireVisitor(result));
}
return result;
}
public static PlainSelect getPlainSelect(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return null;
}
return (PlainSelect) selectBody;
}
public static Select getSelect(String sql) {
Statement statement = null;
try {
statement = CCJSqlParserUtil.parse(sql);
} catch (JSQLParserException e) {
log.error("parse error", e);
return null;
}
if (!(statement instanceof Select)) {
return null;
}
return (Select) statement;
}
public static List<String> getAllFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<String> result = getSelectFields(plainSelect);
GroupByElement groupBy = plainSelect.getGroupBy();
if (groupBy != null) {
List<Expression> groupByExpressions = groupBy.getGroupByExpressions();
for (Expression expression : groupByExpressions) {
if (expression instanceof Column) {
Column column = (Column) expression;
result.add(column.getColumnName());
}
}
}
List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
if (orderByElements != null) {
for (OrderByElement orderByElement : orderByElements) {
Expression expression = orderByElement.getExpression();
if (expression instanceof Column) {
Column column = (Column) expression;
result.add(column.getColumnName());
}
}
}
Expression where = plainSelect.getWhere();
if (where != null) {
where.accept(new ExpressionVisitorAdapter() {
@Override
public void visit(Column column) {
result.add(column.getColumnName());
}
});
}
return new ArrayList<>(result);
}
public static String getTableName(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
SelectBody selectBody = selectStatement.getSelectBody();
PlainSelect plainSelect = (PlainSelect) selectBody;
Table table = (Table) plainSelect.getFromItem();
return table.getName();
}
public static boolean hasAggregateFunction(String sql) {
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return false;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
List<SelectItem> selectItems = plainSelect.getSelectItems();
AggregateFunctionVisitor visitor = new AggregateFunctionVisitor();
for (SelectItem selectItem : selectItems) {
selectItem.accept(visitor);
}
return visitor.hasAggregateFunction();
}
}

156
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/CCJSqlParserUtils.java → common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java
View File

@@ -1,23 +1,17 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.PlainSelect;
@@ -29,115 +23,30 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
/**
* CC JSql ParserUtils
* Sql Parser Update Helper
*/
@Slf4j
public class CCJSqlParserUtils {
public class SqlParserUpdateHelper {
public static List<String> getWhereFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<String> result = new HashSet<>();
Expression where = plainSelect.getWhere();
if (Objects.nonNull(where)) {
where.accept(new FieldAcquireVisitor(result));
}
return new ArrayList<>(result);
}
public static List<String> getSelectFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
return new ArrayList<>(getSelectFields(plainSelect));
}
private static Set<String> getSelectFields(PlainSelect plainSelect) {
List<SelectItem> selectItems = plainSelect.getSelectItems();
Set<String> result = new HashSet<>();
for (SelectItem selectItem : selectItems) {
selectItem.accept(new FieldAcquireVisitor(result));
}
return result;
}
private static PlainSelect getPlainSelect(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
public static String replaceValueFields(String sql, Map<String, Set<String>> fieldValueToFieldNames) {
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return null;
}
return (PlainSelect) selectBody;
}
private static Select getSelect(String sql) {
Statement statement = null;
try {
statement = CCJSqlParserUtil.parse(sql);
} catch (JSQLParserException e) {
log.error("parse error", e);
return null;
}
if (!(statement instanceof Select)) {
return null;
}
return (Select) statement;
}
public static List<String> getAllFields(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
return new ArrayList<>();
}
Set<String> result = getSelectFields(plainSelect);
GroupByElement groupBy = plainSelect.getGroupBy();
if (groupBy != null) {
List<Expression> groupByExpressions = groupBy.getGroupByExpressions();
for (Expression expression : groupByExpressions) {
if (expression instanceof Column) {
Column column = (Column) expression;
result.add(column.getColumnName());
}
}
}
List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
if (orderByElements != null) {
for (OrderByElement orderByElement : orderByElements) {
Expression expression = orderByElement.getExpression();
if (expression instanceof Column) {
Column column = (Column) expression;
result.add(column.getColumnName());
}
}
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
//1. replace where fields
Expression where = plainSelect.getWhere();
if (where != null) {
where.accept(new ExpressionVisitorAdapter() {
@Override
public void visit(Column column) {
result.add(column.getColumnName());
}
});
FiledValueReplaceVisitor visitor = new FiledValueReplaceVisitor(fieldValueToFieldNames);
if (Objects.nonNull(where)) {
where.accept(visitor);
}
return new ArrayList<>(result);
return selectStatement.toString();
}
public static String replaceFields(String sql, Map<String, String> fieldToBizName) {
Select selectStatement = getSelect(sql);
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
@@ -173,7 +82,7 @@ public class CCJSqlParserUtils {
public static String replaceFunction(String sql) {
Select selectStatement = getSelect(sql);
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
@@ -208,7 +117,7 @@ public class CCJSqlParserUtils {
public static String addFieldsToSelect(String sql, List<String> fields) {
Select selectStatement = getSelect(sql);
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
// add fields to select
for (String field : fields) {
SelectUtils.addExpression(selectStatement, new Column(field));
@@ -216,24 +125,11 @@ public class CCJSqlParserUtils {
return selectStatement.toString();
}
public static String getTableName(String sql) {
Select selectStatement = getSelect(sql);
if (selectStatement == null) {
return null;
}
SelectBody selectBody = selectStatement.getSelectBody();
PlainSelect plainSelect = (PlainSelect) selectBody;
Table table = (Table) plainSelect.getFromItem();
return table.getName();
}
public static String replaceTable(String sql, String tableName) {
if (StringUtils.isEmpty(tableName)) {
return sql;
}
Select selectStatement = getSelect(sql);
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
PlainSelect plainSelect = (PlainSelect) selectBody;
// replace table name
@@ -247,7 +143,7 @@ public class CCJSqlParserUtils {
if (StringUtils.isEmpty(column) || Objects.isNull(value)) {
return sql;
}
Select selectStatement = getSelect(sql);
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
@@ -271,7 +167,7 @@ public class CCJSqlParserUtils {
public static String addWhere(String sql, Expression expression) {
Select selectStatement = getSelect(sql);
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
@@ -288,23 +184,5 @@ public class CCJSqlParserUtils {
return selectStatement.toString();
}
public static boolean hasAggregateFunction(String sql) {
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return false;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
List<SelectItem> selectItems = plainSelect.getSelectItems();
AggregateFunctionVisitor visitor = new AggregateFunctionVisitor();
for (SelectItem selectItem : selectItems) {
selectItem.accept(visitor);
}
return visitor.hasAggregateFunction();
}
}

5
common/src/test/java/com/tencent/supersonic/common/util/calcite/SqlParseUtilsTest.java
View File

@@ -121,7 +121,8 @@ class SqlParseUtilsTest {
SqlParserInfo sqlParserInfo = SqlParseUtils.getSqlParseInfo(
"select uv from "
+ " ( "
+ " select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' and user_id = 22 "
+ " select * from t_1 where sys_imp_date >= '2023-07-07' and "
+ "sys_imp_date <= '2023-07-07' and user_id = 22 "
+ " ) as t_sub_1 "
+ " where user_name_元 = 'zhangsan' order by play_count desc limit 10"
);
@@ -129,4 +130,4 @@ class SqlParseUtilsTest {
.collect(Collectors.toList());
Assert.assertTrue(collect.contains("user_id"));
}
}
}

244
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/CCJSqlParserUtilsTest.java
View File

@@ -1,244 +0,0 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
/**
* CCJSqlParserUtils Test
*/
class CCJSqlParserUtilsTest {
@Test
void replaceFields() {
Map<String, String> fieldToBizName = initParams();
String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01' order by 播放量 desc limit 11";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND song_publis_date = '2023-08-01' AND publish_date >= '2023-08-08' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' group by YEAR(发行日期)";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 WHERE YEAR(publish_date) IN (2022, 2023) AND sys_imp_date = '2023-08-14' GROUP BY YEAR(publish_date)",
replaceSql);
replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' group by 发行日期";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 WHERE YEAR(publish_date) IN (2022, 2023) AND sys_imp_date = '2023-08-14' GROUP BY publish_date",
replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-11') <= 1 and 结算播放量 > 1000000 and datediff('day', 数据日期, '2023-08-11') <= 30",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-11' AND play_count > 1000000 AND sys_imp_date <= '2023-08-11' AND publish_date >= '2022-08-11' AND sys_imp_date >= '2023-07-12'"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date >= '2023-08-08' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') = 0 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE 1 = 1 AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date <= '2023-08-09' AND publish_date >= '2023-01-01' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') <= 0.5 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date >= '2023-02-09' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') >= 0.5 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date >= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date <= '2023-02-09' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice' and 发布日期 ='11' order by 访问次数 desc limit 1",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT department, user_id FROM 超音数 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceTable(replaceSql, "s2");
replaceSql = CCJSqlParserUtils.addFieldsToSelect(replaceSql, Collections.singletonList("field_a"));
replaceSql = CCJSqlParserUtils.replaceFields(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice' and 发布日期 ='11' group by 部门 limit 1",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT department, sum(pv) FROM 超音数 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' GROUP BY department LIMIT 1",
replaceSql);
replaceSql = "select sum(访问次数) from 超音数 where 数据日期 >= '2023-08-06' and 数据日期 <= '2023-08-06' and 部门 = 'hr'";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT sum(pv) FROM 超音数 WHERE sys_imp_date >= '2023-08-06' AND sys_imp_date <= '2023-08-06' AND department = 'hr'",
replaceSql);
}
@Test
void getAllFields() {
List<String> allFields = CCJSqlParserUtils.getAllFields(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
Assert.assertEquals(allFields.size(), 6);
allFields = CCJSqlParserUtils.getAllFields(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date >= '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
Assert.assertEquals(allFields.size(), 6);
allFields = CCJSqlParserUtils.getAllFields(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1");
Assert.assertEquals(allFields.size(), 5);
}
@Test
void replaceTable() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
String replaceSql = CCJSqlParserUtils.replaceTable(sql, "s2");
Assert.assertEquals(
"SELECT 部门, sum(访问次数) FROM s2 WHERE 数据日期 = '2023-08-08' AND 用户 = alice AND 发布日期 = '11' GROUP BY 部门 LIMIT 1",
replaceSql);
}
@Test
void getSelectFields() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
List<String> selectFields = CCJSqlParserUtils.getSelectFields(sql);
Assert.assertEquals(selectFields.contains("访问次数"), true);
Assert.assertEquals(selectFields.contains("部门"), true);
}
@Test
void getWhereFields() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1";
List<String> selectFields = CCJSqlParserUtils.getWhereFields(sql);
Assert.assertEquals(selectFields.contains("发布日期"), true);
Assert.assertEquals(selectFields.contains("数据日期"), true);
Assert.assertEquals(selectFields.contains("用户"), true);
}
@Test
void addWhere() throws JSQLParserException {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
sql = CCJSqlParserUtils.addWhere(sql, "column_a", 123444555);
List<String> selectFields = CCJSqlParserUtils.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_a"), true);
sql = CCJSqlParserUtils.addWhere(sql, "column_b", "123456666");
selectFields = CCJSqlParserUtils.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_b"), true);
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( column_c = 111 or column_d = 1111)");
sql = CCJSqlParserUtils.addWhere(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1",
expression);
Assert.assertEquals(sql.contains("column_c = 111"), true);
}
@Test
void hasAggregateFunction() throws JSQLParserException {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
boolean hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "SELECT count(1) FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, false);
sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11'";
hasAggregateFunction = CCJSqlParserUtils.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, false);
}
private Map<String, String> initParams() {
Map<String, String> fieldToBizName = new HashMap<>();
fieldToBizName.put("部门", "department");
fieldToBizName.put("用户", "user_id");
fieldToBizName.put("数据日期", "sys_imp_date");
fieldToBizName.put("发布日期", "publish_date");
fieldToBizName.put("访问次数", "pv");
fieldToBizName.put("歌曲名", "song_name");
fieldToBizName.put("歌手名", "singer_name");
fieldToBizName.put("播放", "play_count");
fieldToBizName.put("歌曲发布时间", "song_publis_date");
fieldToBizName.put("歌曲发布年份", "song_publis_year");
fieldToBizName.put("转3.0前后30天结算份额衰减", "fdafdfdsa_fdas");
return fieldToBizName;
}
}

190
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java Normal file
View File

@@ -0,0 +1,190 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
/**
* SqlParserSelectHelper Test
*/
class SqlParserSelectHelperTest {
@Test
void getWhereFilterExpression() {
List<FilterExpression> filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' "
+ " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' "
+ "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' "
+ "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE "
+ "user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
filterExpression = SqlParserSelectHelper.getFilterExpression(
"SELECT department, user_id, field_a FROM s2 WHERE "
+ "user_id = 'alice' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1");
System.out.println(filterExpression);
}
@Test
void getAllFields() {
List<String> allFields = SqlParserSelectHelper.getAllFields(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'"
+ " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
Assert.assertEquals(allFields.size(), 6);
allFields = SqlParserSelectHelper.getAllFields(
"SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date >= '2023-08-08'"
+ " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1");
Assert.assertEquals(allFields.size(), 6);
allFields = SqlParserSelectHelper.getAllFields(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 = 'alice'"
+ " and 发布日期 ='11' group by 部门 limit 1");
Assert.assertEquals(allFields.size(), 5);
}
@Test
void getSelectFields() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sql);
Assert.assertEquals(selectFields.contains("访问次数"), true);
Assert.assertEquals(selectFields.contains("部门"), true);
}
@Test
void getWhereFields() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08'"
+ " and 用户 = 'alice' and 发布日期 ='11' group by 部门 limit 1";
List<String> selectFields = SqlParserSelectHelper.getWhereFields(sql);
Assert.assertEquals(selectFields.contains("发布日期"), true);
Assert.assertEquals(selectFields.contains("数据日期"), true);
Assert.assertEquals(selectFields.contains("用户"), true);
sql = "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'"
+ " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1";
selectFields = SqlParserSelectHelper.getWhereFields(sql);
Assert.assertEquals(selectFields.contains("发布日期"), true);
Assert.assertEquals(selectFields.contains("数据日期"), true);
Assert.assertEquals(selectFields.contains("用户"), true);
}
@Test
void getOrderByFields() {
String sql = "select 部门,用户 from 超音数 where 数据日期 = '2023-08-08'"
+ " and 用户 = 'alice' and 发布日期 ='11' order by 访问次数 limit 1";
List<String> selectFields = SqlParserSelectHelper.getOrderByFields(sql);
Assert.assertEquals(selectFields.contains("访问次数"), true);
}
@Test
void addWhere() throws JSQLParserException {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
sql = SqlParserUpdateHelper.addWhere(sql, "column_a", 123444555);
List<String> selectFields = SqlParserSelectHelper.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_a"), true);
sql = SqlParserUpdateHelper.addWhere(sql, "column_b", "123456666");
selectFields = SqlParserSelectHelper.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_b"), true);
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( column_c = 111 or column_d = 1111)");
sql = SqlParserUpdateHelper.addWhere(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1",
expression);
Assert.assertEquals(sql.contains("column_c = 111"), true);
}
@Test
void hasAggregateFunction() throws JSQLParserException {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
boolean hasAggregateFunction = SqlParserSelectHelper.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "select 部门,count (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
hasAggregateFunction = SqlParserSelectHelper.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "SELECT count(1) FROM s2 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice'"
+ " AND publish_date = '11' ORDER BY pv DESC LIMIT 1";
hasAggregateFunction = SqlParserSelectHelper.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, true);
sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08' "
+ "AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1";
hasAggregateFunction = SqlParserSelectHelper.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, false);
sql = "SELECT department, user_id, field_a FROM s2 WHERE sys_imp_date = '2023-08-08'"
+ " AND user_id = 'alice' AND publish_date = '11'";
hasAggregateFunction = SqlParserSelectHelper.hasAggregateFunction(sql);
Assert.assertEquals(hasAggregateFunction, false);
}
private Map<String, String> initParams() {
Map<String, String> fieldToBizName = new HashMap<>();
fieldToBizName.put("部门", "department");
fieldToBizName.put("用户", "user_id");
fieldToBizName.put("数据日期", "sys_imp_date");
fieldToBizName.put("发布日期", "publish_date");
fieldToBizName.put("访问次数", "pv");
fieldToBizName.put("歌曲名", "song_name");
fieldToBizName.put("歌手名", "singer_name");
fieldToBizName.put("播放", "play_count");
fieldToBizName.put("歌曲发布时间", "song_publis_date");
fieldToBizName.put("歌曲发布年份", "song_publis_year");
fieldToBizName.put("转3.0前后30天结算份额衰减", "fdafdfdsa_fdas");
return fieldToBizName;
}
}

198
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelperTest.java Normal file
View File

@@ -0,0 +1,198 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
/**
* SqlParserUpdateHelper Test
*/
class SqlParserUpdateHelperTest {
@Test
void replaceFields() {
Map<String, String> fieldToBizName = initParams();
String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 "
+ "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01'"
+ " order by 播放量 desc limit 11";
replaceSql = SqlParserUpdateHelper.replaceFields(replaceSql, fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' "
+ "AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND "
+ "song_publis_date = '2023-08-01' AND publish_date >= '2023-08-08' "
+ "ORDER BY play_count DESC LIMIT 11", replaceSql);
replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) "
+ "in (2022, 2023) and 数据日期 = '2023-08-14' group by YEAR(发行日期)";
replaceSql = SqlParserUpdateHelper.replaceFields(replaceSql, fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 "
+ "WHERE YEAR(publish_date) IN (2022, 2023) AND sys_imp_date = '2023-08-14' "
+ "GROUP BY YEAR(publish_date)",
replaceSql);
replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 "
+ "where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' "
+ "group by 发行日期";
replaceSql = SqlParserUpdateHelper.replaceFields(replaceSql, fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 "
+ "WHERE YEAR(publish_date) IN (2022, 2023) AND sys_imp_date = '2023-08-14'"
+ " GROUP BY publish_date",
replaceSql);
replaceSql = SqlParserUpdateHelper.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-11') <= 1 "
+ "and 结算播放量 > 1000000 and datediff('day', 数据日期, '2023-08-11') <= 30",
fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-11' "
+ "AND play_count > 1000000 AND sys_imp_date <= '2023-08-11' AND "
+ "publish_date >= '2022-08-11' AND sys_imp_date >= '2023-07-12'", replaceSql);
replaceSql = SqlParserUpdateHelper.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 "
+ "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' "
+ "AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' "
+ "AND publish_date >= '2023-08-08' ORDER BY play_count DESC LIMIT 11", replaceSql);
replaceSql = SqlParserUpdateHelper.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') = 0 "
+ "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE 1 = 1 AND singer_name = '邓紫棋'"
+ " AND sys_imp_date = '2023-08-09' AND publish_date <= '2023-08-09' "
+ "AND publish_date >= '2023-01-01' ORDER BY play_count DESC LIMIT 11", replaceSql);
replaceSql = SqlParserUpdateHelper.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') <= 0.5 "
+ "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' "
+ "AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' "
+ "AND publish_date >= '2023-02-09' ORDER BY play_count DESC LIMIT 11", replaceSql);
replaceSql = SqlParserUpdateHelper.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') >= 0.5 "
+ "and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11", fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date >= '2023-08-09' "
+ "AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' "
+ "AND publish_date <= '2023-02-09' ORDER BY play_count DESC LIMIT 11", replaceSql);
replaceSql = SqlParserUpdateHelper.replaceFields(
"select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice'"
+ " and 发布日期 ='11' order by 访问次数 desc limit 1", fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT department, user_id FROM 超音数 WHERE sys_imp_date = '2023-08-08'"
+ " AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1", replaceSql);
replaceSql = SqlParserUpdateHelper.replaceTable(replaceSql, "s2");
replaceSql = SqlParserUpdateHelper.addFieldsToSelect(replaceSql, Collections.singletonList("field_a"));
replaceSql = SqlParserUpdateHelper.replaceFields(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 ='alice' and 发布日期 ='11' group by 部门 limit 1", fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT department, sum(pv) FROM 超音数 WHERE sys_imp_date = '2023-08-08'"
+ " AND user_id = 'alice' AND publish_date = '11' GROUP BY department LIMIT 1", replaceSql);
replaceSql = "select sum(访问次数) from 超音数 where 数据日期 >= '2023-08-06' "
+ "and 数据日期 <= '2023-08-06' and 部门 = 'hr'";
replaceSql = SqlParserUpdateHelper.replaceFields(replaceSql, fieldToBizName);
replaceSql = SqlParserUpdateHelper.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT sum(pv) FROM 超音数 WHERE sys_imp_date >= '2023-08-06' "
+ "AND sys_imp_date <= '2023-08-06' AND department = 'hr'", replaceSql);
}
@Test
void replaceTable() {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
String replaceSql = SqlParserUpdateHelper.replaceTable(sql, "s2");
Assert.assertEquals(
"SELECT 部门, sum(访问次数) FROM s2 WHERE 数据日期 = '2023-08-08' "
+ "AND 用户 = alice AND 发布日期 = '11' GROUP BY 部门 LIMIT 1", replaceSql);
}
@Test
void addWhere() throws JSQLParserException {
String sql = "select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1";
sql = SqlParserUpdateHelper.addWhere(sql, "column_a", 123444555);
List<String> selectFields = SqlParserSelectHelper.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_a"), true);
sql = SqlParserUpdateHelper.addWhere(sql, "column_b", "123456666");
selectFields = SqlParserSelectHelper.getAllFields(sql);
Assert.assertEquals(selectFields.contains("column_b"), true);
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( column_c = 111 or column_d = 1111)");
sql = SqlParserUpdateHelper.addWhere(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' "
+ "and 用户 =alice and 发布日期 ='11' group by 部门 limit 1",
expression);
Assert.assertEquals(sql.contains("column_c = 111"), true);
}
private Map<String, String> initParams() {
Map<String, String> fieldToBizName = new HashMap<>();
fieldToBizName.put("部门", "department");
fieldToBizName.put("用户", "user_id");
fieldToBizName.put("数据日期", "sys_imp_date");
fieldToBizName.put("发布日期", "publish_date");
fieldToBizName.put("访问次数", "pv");
fieldToBizName.put("歌曲名", "song_name");
fieldToBizName.put("歌手名", "singer_name");
fieldToBizName.put("播放", "play_count");
fieldToBizName.put("歌曲发布时间", "song_publis_date");
fieldToBizName.put("歌曲发布年份", "song_publis_year");
fieldToBizName.put("转3.0前后30天结算份额衰减", "fdafdfdsa_fdas");
return fieldToBizName;
}
}