topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-15 14:36:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) Forcefully delete the data date generated by the large model. (#1607)

Browse Source
This commit is contained in:
lexluo09
2024-08-27 12:36:39 +08:00
committed by GitHub
parent 5606633481
commit b9ae0a4c92
6 changed files with 113 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelper.java
View File

@@ -25,11 +25,13 @@ import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SelectItem;
import net.sf.jsqlparser.statement.select.SelectItemVisitorAdapter;
import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@@ -42,10 +44,7 @@ public class SqlRemoveHelper {
public static String removeAsteriskAndAddFields(String sql, Set<String> needAddDefaultFields) {
Select selectStatement = SqlSelectHelper.getSelect(sql);
if (Objects.isNull(selectStatement)) {
return sql;
}
if (!(selectStatement instanceof PlainSelect)) {
if (isInvalidSelect(selectStatement)) {
return sql;
}
List<SelectItem<?>> selectItems = ((PlainSelect) selectStatement).getSelectItems();
@@ -63,10 +62,7 @@ public class SqlRemoveHelper {
public static String removeSameFieldFromSelect(String sql) {
Select selectStatement = SqlSelectHelper.getSelect(sql);
if (selectStatement == null) {
return sql;
}
if (!(selectStatement instanceof PlainSelect)) {
if (isInvalidSelect(selectStatement)) {
return sql;
}
List<SelectItem<?>> selectItems = ((PlainSelect) selectStatement).getSelectItems();
@@ -106,10 +102,7 @@ public class SqlRemoveHelper {
public static String removeNumberFilter(String sql) {
Select selectStatement = SqlSelectHelper.getSelect(sql);
if (selectStatement == null) {
return sql;
}
if (!(selectStatement instanceof PlainSelect)) {
if (isInvalidSelect(selectStatement)) {
return sql;
}
Expression where = ((PlainSelect) selectStatement).getWhere();
@@ -226,10 +219,7 @@ public class SqlRemoveHelper {
public static String removeGroupBy(String sql, Set<String> fields) {
Select selectStatement = SqlSelectHelper.getSelect(sql);
if (selectStatement == null) {
return sql;
}
if (!(selectStatement instanceof PlainSelect)) {
if (isInvalidSelect(selectStatement)) {
return sql;
}
GroupByElement groupByElement = ((PlainSelect) selectStatement).getGroupBy();
@@ -250,6 +240,30 @@ public class SqlRemoveHelper {
return selectStatement.toString();
}
public static String removeSelect(String sql, Set<String> fields) {
Select selectStatement = SqlSelectHelper.getSelect(sql);
if (isInvalidSelect(selectStatement)) {
return sql;
}
List<SelectItem<?>> selectItems = ((PlainSelect) selectStatement).getSelectItems();
Iterator<SelectItem<?>> iterator = selectItems.iterator();
while (iterator.hasNext()) {
SelectItem selectItem = iterator.next();
selectItem.accept(new SelectItemVisitorAdapter() {
@Override
public void visit(SelectItem item) {
if (fields.contains(item.getExpression().toString())) {
iterator.remove();
}
}
});
}
if (selectItems.isEmpty()) {
selectItems.add(new SelectItem(new AllColumns()));
}
return selectStatement.toString();
}
public static Expression filteredExpression(Expression where, SqlEditEnum sqlEditEnum) throws Exception {
if (Objects.isNull(where)) {
return null;
@@ -339,5 +353,9 @@ public class SqlRemoveHelper {
}
}
private static boolean isInvalidSelect(Select selectStatement) {
return Objects.isNull(selectStatement) || !(selectStatement instanceof PlainSelect);
}
}

39
common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlRemoveHelperTest.java
View File

@@ -1,10 +1,11 @@
package com.tencent.supersonic.common.jsqlparser;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
/**
* SqlParser Remove Helper Test
@@ -128,4 +129,38 @@ class SqlRemoveHelperTest {
replaceSql);
}
@Test
void testRemoveSelect() {
String sql = "select 数据日期,歌曲名 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时间 = '2023-08-01'";
Set<String> removeFieldNames = new HashSet<>();
removeFieldNames.add("数据日期");
String replaceSql = SqlRemoveHelper.removeSelect(sql, removeFieldNames);
Assert.assertEquals(
"SELECT 歌曲名 FROM 歌曲库 WHERE 歌曲名 = '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01'",
replaceSql);
sql = "select 数据日期 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and 歌曲发布时间 = '2023-08-01'";
replaceSql = SqlRemoveHelper.removeSelect(sql, removeFieldNames);
Assert.assertEquals(
"SELECT * FROM 歌曲库 WHERE 歌曲名 = '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01'",
replaceSql);
}
@Test
void testRemoveGroupBy() {
String sql = "select 数据日期 from 歌曲库 where 歌曲名 = '邓紫棋' and 数据日期 = '2023-08-09' and "
+ "歌曲发布时间 = '2023-08-01' group by 数据日期";
Set<String> removeFieldNames = new HashSet<>();
removeFieldNames.add("数据日期");
String replaceSql = SqlRemoveHelper.removeGroupBy(sql, removeFieldNames);
Assert.assertEquals(
"SELECT 数据日期 FROM 歌曲库 WHERE 歌曲名 = '邓紫棋' AND 数据日期 = '2023-08-09' AND 歌曲发布时间 = '2023-08-01'",
replaceSql);
}
}