topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 20:25:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](supersonic) based on version 0.7.2 (#34)

Browse Source 
Co-authored-by: zuopengge <hwzuopengge@tencent.com>
This commit is contained in:
mainmain
2023-08-20 17:30:35 +08:00
committed by GitHub
parent c93e60ced7
commit cf1b5336c3
122 changed files with 4045 additions and 1075 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
common/pom.xml
View File

@@ -118,6 +118,12 @@
</dependency>
<dependency>
<groupId>com.github.plexpt</groupId>
<artifactId>chatgpt</artifactId>
<version>4.1.2</version>
</dependency>
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper</artifactId>

130
common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java Normal file
View File

@@ -0,0 +1,130 @@
package com.tencent.supersonic.common.util;
import com.plexpt.chatgpt.ChatGPT;
import com.plexpt.chatgpt.entity.chat.ChatCompletion;
import com.plexpt.chatgpt.entity.chat.ChatCompletionResponse;
import com.plexpt.chatgpt.entity.chat.Message;
import com.plexpt.chatgpt.util.Proxys;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.net.Proxy;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
@Component
@Slf4j
public class ChatGptHelper {
@Value("${llm.chatgpt.apikey:}")
private String apiKey;
@Value("${llm.chatgpt.apiHost:}")
private String apiHost;
@Value("${llm.chatgpt.proxyIp:}")
private String proxyIp;
@Value("${llm.chatgpt.proxyPort:}")
private Integer proxyPort;
public ChatGPT getChatGPT(){
Proxy proxy = null;
if (!"default".equals(proxyIp)){
proxy = Proxys.http(proxyIp, proxyPort);
}
return ChatGPT.builder()
.apiKey(apiKey)
.proxy(proxy)
.timeout(900)
.apiHost(apiHost) //反向代理地址
.build()
.init();
}
public Message getChatCompletion(Message system,Message message){
ChatCompletion chatCompletion = ChatCompletion.builder()
.model(ChatCompletion.Model.GPT_3_5_TURBO_16K.getName())
.messages(Arrays.asList(system, message))
.maxTokens(10000)
.temperature(0.9)
.build();
ChatCompletionResponse response = getChatGPT().chatCompletion(chatCompletion);
return response.getChoices().get(0).getMessage();
}
public String inferredTime(String queryText){
long nowTime = System.currentTimeMillis();
Date date = new Date(nowTime);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String formattedDate = sdf.format(date);
Message system = Message.ofSystem("现在时间 "+formattedDate+"，你是一个专业的数据分析师，你的任务是基于数据，专业的解答用户的问题。" +
"你需要遵守以下规则：\n" +
"1.返回规范的数据格式，json，如： 输入：近 10 天的日活跃数，输出：{\"start\":\"2023-07-21\",\"end\":\"2023-07-31\"}" +
"2.你对时间数据要求规范，能从近 10 天，国庆节，端午节，获取到相应的时间，填写到 json 中。\n"+
"3.你的数据时间，只有当前及之前时间即可,超过则回复去年\n" +
"4.只需要解析出时间，时间可以是时间月和年或日、日历采用公历\n"+
"5.时间给出要是绝对正确，不能瞎编\n"
);
Message message = Message.of("输入："+queryText+"，输出：");
Message res = getChatCompletion(system, message);
return res.getContent();
}
public String mockAlias(String mockType,String name,String bizName,String table,String desc,Boolean isPercentage){
String msg = "Assuming you are a professional data analyst specializing in indicators, you have a vast amount of data analysis indicator content. You are familiar with the basic format of the content,Now, Construct your answer Based on the following json-schema.\n" +
"{\n" +
"\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n" +
"\"type\": \"array\",\n" +
"\"minItems\": 2,\n" +
"\"maxItems\": 4,\n" +
"\"items\": {\n" +
"\"type\": \"string\",\n" +
"\"description\": \"Assuming you are a data analyst and give a defined "+mockType+" name: " +name+","+
"this "+mockType+" is from database and table: "+table+ ",This "+mockType+" calculates the field source: "+bizName+", The description of this indicator is: "+desc+", provide some aliases for this，please take chinese or english,but more chinese and Not repeating，\"\n" +
"},\n" +
"\"additionalProperties\":false}\n" +
"Please double-check whether the answer conforms to the format described in the JSON-schema.\n" +
"ANSWER JSON:";
log.info("msg:{}",msg);
Message system = Message.ofSystem("");
Message message = Message.of(msg);
Message res = getChatCompletion(system, message);
return res.getContent();
}
public String mockDimensionValueAlias(String json){
String msg = "Assuming you are a professional data analyst specializing in indicators,for you a json list，" +
"the required content to follow is as follows: " +
"1. The format of JSON," +
"2. Only return in JSON format," +
"3. the array item > 1 and < 5,more alias," +
"for example：input:[\"qq_music\",\"kugou_music\"],out:{\"tran\":[\"qq音乐\",\"酷狗音乐\"],\"alias\":{\"qq_music\":[\"q音\",\"qq音乐\"],\"kugou_music\":[\"kugou\",\"酷狗\"]}}," +
"now input: " + json + ","+
"answer json:";
log.info("msg:{}",msg);
Message system = Message.ofSystem("");
Message message = Message.of(msg);
Message res = getChatCompletion(system, message);
return res.getContent();
}
public static void main(String[] args) {
ChatGptHelper chatGptHelper = new ChatGptHelper();
System.out.println(chatGptHelper.mockAlias("","","","","",false));
}
}

37
common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
View File

@@ -1,11 +1,13 @@
package com.tencent.supersonic.common.util;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAdjusters;
import java.util.Calendar;
import java.util.Date;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -56,30 +58,37 @@ public class DateUtils {
return dateFormat.format(calendar.getTime());
}
public static String getBeforeDate(String date, int intervalDay, DatePeriodEnum datePeriodEnum) {
Calendar calendar = Calendar.getInstance();
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT);
try {
calendar.setTime(dateFormat.parse(date));
} catch (ParseException e) {
log.error("parse error");
}
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_FORMAT_DOT);
LocalDate currentDate = LocalDate.parse(date, dateTimeFormatter);
LocalDate result = null;
switch (datePeriodEnum) {
case DAY:
calendar.set(Calendar.DATE, calendar.get(Calendar.DATE) - intervalDay);
result = currentDate.minusDays(intervalDay);
break;
case WEEK:
calendar.set(Calendar.DATE, calendar.get(Calendar.DATE) - intervalDay * 7);
result = currentDate.minusWeeks(intervalDay);
if (intervalDay == 0) {
result = result.with(TemporalAdjusters.previousOrSame(java.time.DayOfWeek.MONDAY));
}
break;
case MONTH:
calendar.set(Calendar.MONTH, calendar.get(Calendar.MONTH) - intervalDay);
result = currentDate.minusMonths(intervalDay);
if (intervalDay == 0) {
result = result.with(TemporalAdjusters.firstDayOfMonth());
}
break;
case YEAR:
calendar.set(Calendar.YEAR, calendar.get(Calendar.YEAR) - intervalDay);
result = currentDate.minusYears(intervalDay);
if (intervalDay == 0) {
result = result.with(TemporalAdjusters.firstDayOfYear());
}
break;
default:
}
return dateFormat.format(calendar.getTime());
if (Objects.nonNull(result)) {
return result.format(DateTimeFormatter.ofPattern(DATE_FORMAT_DOT));
}
return null;
}
}

1
common/src/main/java/com/tencent/supersonic/common/util/StringUtil.java
View File

@@ -9,7 +9,6 @@ public class StringUtil {
public static final String COMMA_WRAPPER = "'%s'";
public static final String SPACE_WRAPPER = " %s ";
public static String getCommaWrap(String value) {
return String.format(COMMA_WRAPPER, value);
}

26
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/CCJSqlParserUtils.java
View File

@@ -41,7 +41,9 @@ public class CCJSqlParserUtils {
}
Set<String> result = new HashSet<>();
Expression where = plainSelect.getWhere();
where.accept(new FieldAcquireVisitor(result));
if (Objects.nonNull(where)) {
where.accept(new FieldAcquireVisitor(result));
}
return new ArrayList<>(result);
}
@@ -166,7 +168,24 @@ public class CCJSqlParserUtils {
if (Objects.nonNull(groupByElement)) {
groupByElement.accept(new GroupByReplaceVisitor(fieldToBizName));
}
//5. add Waiting Expression
return selectStatement.toString();
}
public static String replaceFunction(String sql) {
Select selectStatement = getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
//1. replace where dataDiff function
Expression where = plainSelect.getWhere();
FunctionReplaceVisitor visitor = new FunctionReplaceVisitor();
if (Objects.nonNull(where)) {
where.accept(visitor);
}
//2. add Waiting Expression
List<Expression> waitingForAdds = visitor.getWaitingForAdds();
addWaitingExpression(plainSelect, where, waitingForAdds);
return selectStatement.toString();
@@ -181,9 +200,10 @@ public class CCJSqlParserUtils {
if (where == null) {
plainSelect.setWhere(expression);
} else {
plainSelect.setWhere(new AndExpression(where, expression));
where = new AndExpression(where, expression);
}
}
plainSelect.setWhere(where);
}

48
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldReplaceVisitor.java
View File

@@ -1,25 +1,15 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
@Slf4j
public class FieldReplaceVisitor extends ExpressionVisitorAdapter {
ParseVisitorHelper parseVisitorHelper = new ParseVisitorHelper();
private Map<String, String> fieldToBizName;
private List<Expression> waitingForAdds = new ArrayList<>();
public FieldReplaceVisitor(Map<String, String> fieldToBizName) {
this.fieldToBizName = fieldToBizName;
@@ -29,42 +19,4 @@ public class FieldReplaceVisitor extends ExpressionVisitorAdapter {
public void visit(Column column) {
parseVisitorHelper.replaceColumn(column, fieldToBizName);
}
@Override
public void visit(MinorThan expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, ">");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
@Override
public void visit(MinorThanEquals expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, ">=");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
@Override
public void visit(GreaterThan expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, "<");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
@Override
public void visit(GreaterThanEquals expr) {
Expression expression = parseVisitorHelper.reparseDate(expr, fieldToBizName, "<=");
if (Objects.nonNull(expression)) {
waitingForAdds.add(expression);
}
}
public List<Expression> getWaitingForAdds() {
return waitingForAdds;
}
}

171
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionReplaceVisitor.java Normal file
View File

@@ -0,0 +1,171 @@
package com.tencent.supersonic.common.util.jsqlparser;
import com.tencent.supersonic.common.util.DatePeriodEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.StringUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
@Slf4j
public class FunctionReplaceVisitor extends ExpressionVisitorAdapter {
public static final String DATE_FUNCTION = "datediff";
public static final double HALF_YEAR = 0.5d;
public static final int SIX_MONTH = 6;
public static final String EQUAL = "=";
private List<Expression> waitingForAdds = new ArrayList<>();
@Override
public void visit(MinorThan expr) {
List<Expression> expressions = reparseDate(expr, ">");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(EqualsTo expr) {
List<Expression> expressions = reparseDate(expr, ">=");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(MinorThanEquals expr) {
List<Expression> expressions = reparseDate(expr, ">=");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(GreaterThan expr) {
List<Expression> expressions = reparseDate(expr, "<");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
@Override
public void visit(GreaterThanEquals expr) {
List<Expression> expressions = reparseDate(expr, "<=");
if (Objects.nonNull(expressions)) {
waitingForAdds.addAll(expressions);
}
}
public List<Expression> getWaitingForAdds() {
return waitingForAdds;
}
public List<Expression> reparseDate(ComparisonOperator comparisonOperator, String startDateOperator) {
List<Expression> result = new ArrayList<>();
Expression leftExpression = comparisonOperator.getLeftExpression();
if (!(leftExpression instanceof Function)) {
return result;
}
Function leftExpressionFunction = (Function) leftExpression;
if (!leftExpressionFunction.toString().contains(DATE_FUNCTION)) {
return result;
}
List<Expression> leftExpressions = leftExpressionFunction.getParameters().getExpressions();
if (CollectionUtils.isEmpty(leftExpressions) || leftExpressions.size() < 3) {
return result;
}
Column field = (Column) leftExpressions.get(1);
String columnName = field.getColumnName();
try {
String startDateValue = getStartDateStr(comparisonOperator, leftExpressions);
String endDateValue = getEndDateValue(leftExpressions);
String endDateOperator = comparisonOperator.getStringExpression();
String condExpr =
columnName + StringUtil.getSpaceWrap(getEndDateOperator(comparisonOperator))
+ StringUtil.getCommaWrap(endDateValue);
ComparisonOperator expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
String startDataCondExpr =
columnName + StringUtil.getSpaceWrap(startDateOperator) + StringUtil.getCommaWrap(startDateValue);
if (EQUAL.equalsIgnoreCase(endDateOperator)) {
result.add(CCJSqlParserUtil.parseCondExpression(condExpr));
expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(" 1 = 1 ");
}
comparisonOperator.setLeftExpression(null);
comparisonOperator.setRightExpression(null);
comparisonOperator.setASTNode(null);
comparisonOperator.setLeftExpression(expression.getLeftExpression());
comparisonOperator.setRightExpression(expression.getRightExpression());
comparisonOperator.setASTNode(expression.getASTNode());
result.add(CCJSqlParserUtil.parseCondExpression(startDataCondExpr));
return result;
} catch (JSQLParserException e) {
log.error("JSQLParserException", e);
}
return null;
}
private String getStartDateStr(ComparisonOperator minorThanEquals, List<Expression> expressions) {
String unitValue = getUnit(expressions);
String dateValue = getEndDateValue(expressions);
String dateStr = "";
Expression rightExpression = minorThanEquals.getRightExpression();
DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(unitValue);
if (rightExpression instanceof DoubleValue) {
DoubleValue value = (DoubleValue) rightExpression;
double doubleValue = value.getValue();
if (DatePeriodEnum.YEAR.equals(datePeriodEnum) && doubleValue == HALF_YEAR) {
datePeriodEnum = DatePeriodEnum.MONTH;
dateStr = DateUtils.getBeforeDate(dateValue, SIX_MONTH, datePeriodEnum);
}
} else if (rightExpression instanceof LongValue) {
LongValue value = (LongValue) rightExpression;
long doubleValue = value.getValue();
dateStr = DateUtils.getBeforeDate(dateValue, (int) doubleValue, datePeriodEnum);
}
return dateStr;
}
private String getEndDateOperator(ComparisonOperator comparisonOperator) {
String operator = comparisonOperator.getStringExpression();
if (EQUAL.equalsIgnoreCase(operator)) {
operator = "<=";
}
return operator;
}
private String getEndDateValue(List<Expression> leftExpressions) {
StringValue date = (StringValue) leftExpressions.get(2);
return date.getValue();
}
private String getUnit(List<Expression> expressions) {
StringValue unit = (StringValue) expressions.get(0);
return unit.getValue();
}
}

17
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByReplaceVisitor.java
View File

@@ -2,13 +2,18 @@ package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.GroupByVisitor;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class GroupByReplaceVisitor implements GroupByVisitor {
ParseVisitorHelper parseVisitorHelper = new ParseVisitorHelper();
@@ -29,7 +34,17 @@ public class GroupByReplaceVisitor implements GroupByVisitor {
String replaceColumn = parseVisitorHelper.getReplaceColumn(expression.toString(), fieldToBizName);
if (StringUtils.isNotEmpty(replaceColumn)) {
groupByExpressions.set(i, new Column(replaceColumn));
if (expression instanceof Column) {
groupByExpressions.set(i, new Column(replaceColumn));
}
if (expression instanceof Function) {
try {
Expression element = CCJSqlParserUtil.parseExpression(replaceColumn);
((Function) expression).getParameters().getExpressions().set(0, element);
} catch (JSQLParserException e) {
log.error("e", e);
}
}
}
}
}

99
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/ParseVisitorHelper.java
View File

@@ -1,33 +1,16 @@
package com.tencent.supersonic.common.util.jsqlparser;
import com.tencent.supersonic.common.util.DatePeriodEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.StringUtil;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class ParseVisitorHelper {
public static final double HALF_YEAR = 0.5d;
public static final int SIX_MONTH = 6;
public static final String DATE_FUNCTION = "datediff";
public void replaceColumn(Column column, Map<String, String> fieldToBizName) {
String columnName = column.getColumnName();
column.setColumnName(getReplaceColumn(columnName, fieldToBizName));
@@ -53,88 +36,6 @@ public class ParseVisitorHelper {
return columnName;
}
public Expression reparseDate(ComparisonOperator comparisonOperator, Map<String, String> fieldToBizName,
String startDateOperator) {
Expression leftExpression = comparisonOperator.getLeftExpression();
if (leftExpression instanceof Column) {
Column leftExpressionColumn = (Column) leftExpression;
replaceColumn(leftExpressionColumn, fieldToBizName);
return null;
}
if (!(leftExpression instanceof Function)) {
return null;
}
Function leftExpressionFunction = (Function) leftExpression;
if (!leftExpressionFunction.toString().contains(DATE_FUNCTION)) {
return null;
}
List<Expression> leftExpressions = leftExpressionFunction.getParameters().getExpressions();
if (CollectionUtils.isEmpty(leftExpressions) || leftExpressions.size() < 3) {
return null;
}
Column field = (Column) leftExpressions.get(1);
String columnName = field.getColumnName();
String startDateValue = getStartDateStr(comparisonOperator, leftExpressions);
String fieldBizName = fieldToBizName.get(columnName);
try {
String endDateValue = getEndDateValue(leftExpressions);
String stringExpression = comparisonOperator.getStringExpression();
String condExpr =
fieldBizName + StringUtil.getSpaceWrap(stringExpression) + StringUtil.getCommaWrap(endDateValue);
ComparisonOperator expression = (ComparisonOperator) CCJSqlParserUtil.parseCondExpression(condExpr);
comparisonOperator.setLeftExpression(null);
comparisonOperator.setRightExpression(null);
comparisonOperator.setASTNode(null);
comparisonOperator.setLeftExpression(expression.getLeftExpression());
comparisonOperator.setRightExpression(expression.getRightExpression());
comparisonOperator.setASTNode(expression.getASTNode());
String startDataCondExpr =
fieldBizName + StringUtil.getSpaceWrap(startDateOperator) + StringUtil.getCommaWrap(startDateValue);
return CCJSqlParserUtil.parseCondExpression(startDataCondExpr);
} catch (JSQLParserException e) {
log.error("JSQLParserException", e);
}
return null;
}
private String getEndDateValue(List<Expression> leftExpressions) {
StringValue date = (StringValue) leftExpressions.get(2);
return date.getValue();
}
private String getStartDateStr(ComparisonOperator minorThanEquals, List<Expression> expressions) {
String unitValue = getUnit(expressions);
String dateValue = getEndDateValue(expressions);
String dateStr = "";
Expression rightExpression = minorThanEquals.getRightExpression();
DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(unitValue);
if (rightExpression instanceof DoubleValue) {
DoubleValue value = (DoubleValue) rightExpression;
double doubleValue = value.getValue();
if (DatePeriodEnum.YEAR.equals(datePeriodEnum) && doubleValue == HALF_YEAR) {
datePeriodEnum = DatePeriodEnum.MONTH;
dateStr = DateUtils.getBeforeDate(dateValue, SIX_MONTH, datePeriodEnum);
}
} else if (rightExpression instanceof LongValue) {
LongValue value = (LongValue) rightExpression;
long doubleValue = value.getValue();
dateStr = DateUtils.getBeforeDate(dateValue, (int) doubleValue, datePeriodEnum);
}
return dateStr;
}
private String getUnit(List<Expression> expressions) {
StringValue unit = (StringValue) expressions.get(0);
return unit.getValue();
}
public static int editDistance(String word1, String word2) {
final int m = word1.length();
final int n = word2.length();

12
common/src/test/java/com/tencent/supersonic/common/util/DateUtilsTest.java
View File

@@ -14,13 +14,25 @@ class DateUtilsTest {
dateStr = DateUtils.getBeforeDate("2023-08-10", 8, DatePeriodEnum.DAY);
Assert.assertEquals(dateStr, "2023-08-02");
dateStr = DateUtils.getBeforeDate("2023-08-10", 0, DatePeriodEnum.DAY);
Assert.assertEquals(dateStr, "2023-08-10");
dateStr = DateUtils.getBeforeDate("2023-08-10", 1, DatePeriodEnum.WEEK);
Assert.assertEquals(dateStr, "2023-08-03");
dateStr = DateUtils.getBeforeDate("2023-08-10", 0, DatePeriodEnum.WEEK);
Assert.assertEquals(dateStr, "2023-08-07");
dateStr = DateUtils.getBeforeDate("2023-08-01", 1, DatePeriodEnum.MONTH);
Assert.assertEquals(dateStr, "2023-07-01");
dateStr = DateUtils.getBeforeDate("2023-08-10", 0, DatePeriodEnum.MONTH);
Assert.assertEquals(dateStr, "2023-08-01");
dateStr = DateUtils.getBeforeDate("2023-08-01", 1, DatePeriodEnum.YEAR);
Assert.assertEquals(dateStr, "2022-08-01");
dateStr = DateUtils.getBeforeDate("2023-08-10", 0, DatePeriodEnum.YEAR);
Assert.assertEquals(dateStr, "2023-01-01");
}
}

77
common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/CCJSqlParserUtilsTest.java
View File

@@ -22,37 +22,104 @@ class CCJSqlParserUtilsTest {
String replaceSql = "select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时 = '2023-08-01' order by 播放量 desc limit 11";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND song_publis_date = '2023-08-01' AND publish_date >= '2023-08-08' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' group by YEAR(发行日期)";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 WHERE YEAR(publish_date) IN (2022, 2023) AND sys_imp_date = '2023-08-14' GROUP BY YEAR(publish_date)",
replaceSql);
replaceSql = "select YEAR(发行日期), count(歌曲名) from 歌曲库 where YEAR(发行日期) in (2022, 2023) and 数据日期 = '2023-08-14' group by 发行日期";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT YEAR(publish_date), count(song_name) FROM 歌曲库 WHERE YEAR(publish_date) IN (2022, 2023) AND sys_imp_date = '2023-08-14' GROUP BY publish_date",
replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-11') <= 1 and 结算播放量 > 1000000 and datediff('day', 数据日期, '2023-08-11') <= 30",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-11' AND play_count > 1000000 AND sys_imp_date <= '2023-08-11' AND publish_date >= '2022-08-11' AND sys_imp_date >= '2023-07-12'"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('day', 发布日期, '2023-08-09') <= 1 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date >= '2023-08-08' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') = 0 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE 1 = 1 AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date <= '2023-08-09' AND publish_date >= '2023-01-01' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') <= 0.5 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date <= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date >= '2023-02-09' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 歌曲名 from 歌曲库 where datediff('year', 发布日期, '2023-08-09') >= 0.5 and 歌手名 = '邓紫棋' and 数据日期 = '2023-08-09' order by 播放量 desc limit 11",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT song_name FROM 歌曲库 WHERE publish_date >= '2023-08-09' AND singer_name = '邓紫棋' AND sys_imp_date = '2023-08-09' AND publish_date <= '2023-02-09' ORDER BY play_count DESC LIMIT 11"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceFields(
"select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' order by 访问次数 desc limit 1",
"select 部门,用户 from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice' and 发布日期 ='11' order by 访问次数 desc limit 1",
fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT department, user_id FROM 超音数 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' ORDER BY pv DESC LIMIT 1"
, replaceSql);
replaceSql = CCJSqlParserUtils.replaceTable(replaceSql, "s2");
replaceSql = CCJSqlParserUtils.addFieldsToSelect(replaceSql, Collections.singletonList("field_a"));
replaceSql = CCJSqlParserUtils.replaceFields(
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 =alice and 发布日期 ='11' group by 部门 limit 1",
"select 部门,sum (访问次数) from 超音数 where 数据日期 = '2023-08-08' and 用户 ='alice' and 发布日期 ='11' group by 部门 limit 1",
fieldToBizName);
Assert.assertEquals(replaceSql,
"SELECT department, sum(pv) FROM 超音数 WHERE sys_imp_date = '2023-08-08' AND user_id = user_id AND publish_date = '11' GROUP BY department LIMIT 1");
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
Assert.assertEquals(
"SELECT department, sum(pv) FROM 超音数 WHERE sys_imp_date = '2023-08-08' AND user_id = 'alice' AND publish_date = '11' GROUP BY department LIMIT 1",
replaceSql);
replaceSql = "select sum(访问次数) from 超音数 where 数据日期 >= '2023-08-06' and 数据日期 <= '2023-08-06' and 部门 = 'hr'";
replaceSql = CCJSqlParserUtils.replaceFields(replaceSql, fieldToBizName);
replaceSql = CCJSqlParserUtils.replaceFunction(replaceSql);
System.out.println(replaceSql);
Assert.assertEquals(
"SELECT sum(pv) FROM 超音数 WHERE sys_imp_date >= '2023-08-06' AND sys_imp_date <= '2023-08-06' AND department = 'hr'",
replaceSql);
}