topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[release](project)update version 0.7.4 backend (#66)

Browse Source
This commit is contained in:
daikon
2023-09-10 21:26:46 +08:00
committed by GitHub
parent 02068f58c7
commit a8add4c013
172 changed files with 2180 additions and 1082 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java
View File

@@ -62,4 +62,6 @@ public class Constants {
public static final String BRACKETS_START = "[";
public static final String BRACKETS_END = "]";
public static final Long DEFAULT_FREQUENCY = 100000L;
}

51
common/src/main/java/com/tencent/supersonic/common/pojo/DataAddEvent.java Normal file
View File

@@ -0,0 +1,51 @@
package com.tencent.supersonic.common.pojo;
import org.springframework.context.ApplicationEvent;
public class DataAddEvent extends ApplicationEvent {
private static final long serialVersionUID = 1L;
private String name;
private Long modelId;
private Long id;
private String type;
public DataAddEvent(Object source, String name, Long modelId, Long id, String type) {
super(source);
this.name = name;
this.modelId = modelId;
this.id = id;
this.type = type;
}
public void setName(String name) {
this.name = name;
}
public String getName() {
return name;
}
public void setId(Long id) {
this.id = id;
}
public Long getId() {
return id;
}
public Long getModelId() {
return modelId;
}
public void setModelId(Long modelId) {
this.modelId = modelId;
}
public void setType(String type) {
this.type = type;
}
public String getType() {
return type;
}
}

51
common/src/main/java/com/tencent/supersonic/common/pojo/DataDeleteEvent.java Normal file
View File

@@ -0,0 +1,51 @@
package com.tencent.supersonic.common.pojo;
import org.springframework.context.ApplicationEvent;
public class DataDeleteEvent extends ApplicationEvent {
private static final long serialVersionUID = 1L;
private String name;
private Long modelId;
private Long id;
private String type;
public DataDeleteEvent(Object source, String name, Long modelId, Long id, String type) {
super(source);
this.name = name;
this.modelId = modelId;
this.id = id;
this.type = type;
}
public void setName(String name) {
this.name = name;
}
public String getName() {
return name;
}
public void setId(Long id) {
this.id = id;
}
public Long getId() {
return id;
}
public Long getModelId() {
return modelId;
}
public void setModelId(Long modelId) {
this.modelId = modelId;
}
public void setType(String type) {
this.type = type;
}
public String getType() {
return type;
}
}

61
common/src/main/java/com/tencent/supersonic/common/pojo/DataUpdateEvent.java Normal file
View File

@@ -0,0 +1,61 @@
package com.tencent.supersonic.common.pojo;
import org.springframework.context.ApplicationEvent;
public class DataUpdateEvent extends ApplicationEvent {
private static final long serialVersionUID = 1L;
private String name;
private String newName;
private Long modelId;
private Long id;
private String type;
public DataUpdateEvent(Object source, String name, String newName, Long modelId, Long id, String type) {
super(source);
this.name = name;
this.newName = newName;
this.modelId = modelId;
this.id = id;
this.type = type;
}
public void setName(String name) {
this.name = name;
}
public String getName() {
return name;
}
public String getNewName() {
return newName;
}
public void setNewName(String newName) {
this.newName = newName;
}
public void setId(Long id) {
this.id = id;
}
public Long getId() {
return id;
}
public Long getModelId() {
return modelId;
}
public void setModelId(Long modelId) {
this.modelId = modelId;
}
public void setType(String type) {
this.type = type;
}
public String getType() {
return type;
}
}

55
common/src/main/java/com/tencent/supersonic/common/pojo/enums/DictWordType.java Normal file
View File

@@ -0,0 +1,55 @@
package com.tencent.supersonic.common.pojo.enums;
import org.apache.commons.lang3.StringUtils;
/***
* nature type
* such as : metric、dimension etc.
*/
public enum DictWordType {
METRIC("metric"),
DIMENSION("dimension"),
VALUE("value"),
DOMAIN("dm"),
MODEL("model"),
ENTITY("entity"),
NUMBER("m"),
SUFFIX("suffix");
public static final String NATURE_SPILT = "_";
public static final String SPACE = " ";
private String type;
DictWordType(String type) {
this.type = type;
}
public String getType() {
return NATURE_SPILT + type;
}
public static DictWordType getNatureType(String nature) {
if (StringUtils.isEmpty(nature) || !nature.startsWith(NATURE_SPILT)) {
return null;
}
for (DictWordType dictWordType : values()) {
if (nature.endsWith(dictWordType.getType())) {
return dictWordType;
}
}
//domain
String[] natures = nature.split(DictWordType.NATURE_SPILT);
if (natures.length == 2 && StringUtils.isNumeric(natures[1])) {
return DOMAIN;
}
//dimension value
if (natures.length == 3 && StringUtils.isNumeric(natures[1]) && StringUtils.isNumeric(natures[2])) {
return VALUE;
}
return null;
}
}

15
common/src/main/java/com/tencent/supersonic/common/util/ChatGptHelper.java
View File

@@ -83,8 +83,8 @@ public class ChatGptHelper {
String table,
String desc,
Boolean isPercentage) {
String msg = "Assuming you are a professional data analyst specializing in indicators, "
+ "you have a vast amount of data analysis indicator content. You are familiar with the basic"
String msg = "Assuming you are a professional data analyst specializing in metrics and dimensions, "
+ "you have a vast amount of data analysis metrics content. You are familiar with the basic"
+ " format of the content,Now, Construct your answer Based on the following json-schema.\n"
+ "{\n"
+ "\"$schema\": \"http://json-schema.org/draft-07/schema#\",\n"
@@ -104,10 +104,15 @@ public class ChatGptHelper {
+ mockType
+ " calculates the field source: "
+ bizName
+ ", The description of this indicator is: "
+ ", The description of this metrics is: "
+ desc
+ ", provide some aliases for this，please take chinese or english,"
+ "but more chinese and Not repeating，\"\n"
+ ", provide some aliases for this, please take chinese or english,"
+ "You must adhere to the following rules:\n"
+ "1. Please do not generate aliases like xxx1, xxx2, xxx3.\n"
+ "2. Please do not generate aliases that are the same as the original names of metrics/dimensions.\n"
+ "3. Please pay attention to the quality of the generated aliases and "
+ " avoid creating aliases that look like test data.\n"
+ "4. Please generate more Chinese aliases."
+ "},\n"
+ "\"additionalProperties\":false}\n"
+ "Please double-check whether the answer conforms to the format described in the JSON-schema.\n"

38
common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.common.util;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -13,11 +14,13 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DateUtils {
public static final String DATE_FORMAT_DOT = "yyyy-MM-dd";
public static final String DATE_FORMAT = "yyyy-MM-dd";
public static final String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
public static Integer currentYear() {
Date date = new Date();
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT);
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
String time = dateFormat.format(date).replaceAll("-", "");
int year = Integer.parseInt(time.substring(0, 4));
return year;
@@ -56,12 +59,19 @@ public class DateUtils {
Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date());
calendar.add(Calendar.DAY_OF_MONTH, -intervalDay);
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_DOT);
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
return dateFormat.format(calendar.getTime());
}
public static String getBeforeDate(int intervalDay, DatePeriodEnum datePeriodEnum) {
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
String currentDate = dateFormat.format(new Date());
return getBeforeDate(currentDate, intervalDay, datePeriodEnum);
}
public static String getBeforeDate(String date, int intervalDay, DatePeriodEnum datePeriodEnum) {
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_FORMAT_DOT);
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_FORMAT);
LocalDate currentDate = LocalDate.parse(date, dateTimeFormatter);
LocalDate result = null;
switch (datePeriodEnum) {
@@ -89,8 +99,26 @@ public class DateUtils {
default:
}
if (Objects.nonNull(result)) {
return result.format(DateTimeFormatter.ofPattern(DATE_FORMAT_DOT));
return result.format(DateTimeFormatter.ofPattern(DATE_FORMAT));
}
return null;
}
public static String format(Date date) {
DateFormat dateFormat;
if (containsTime(date)) {
dateFormat = new SimpleDateFormat(DateUtils.TIME_FORMAT);
} else {
dateFormat = new SimpleDateFormat(DateUtils.DATE_FORMAT);
}
return dateFormat.format(date);
}
private static boolean containsTime(Date date) {
DateFormat timeFormat = new SimpleDateFormat("HH:mm:ss");
String timeString = timeFormat.format(date);
return !timeString.equals("00:00:00");
}
}

15
common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java vendored
View File

@@ -20,7 +20,7 @@ public class CaffeineCacheConfig {
@Value("${caffeine.max.size:5000}")
private Integer caffeineMaximumSize;
@Bean
@Bean(name = "caffeineCache")
public Cache<String, Object> caffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES)
@@ -30,4 +30,15 @@ public class CaffeineCacheConfig {
.maximumSize(caffeineMaximumSize)
.build();
}
}
@Bean(name = "searchCaffeineCache")
public Cache<Long, Object> searchCaffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(10000, TimeUnit.MINUTES)
// 初始的缓存空间大小
.initialCapacity(caffeineInitialCapacity)
// 缓存的最大条数
.maximumSize(caffeineMaximumSize)
.build();
}
}

5
common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java vendored
View File

@@ -6,6 +6,7 @@ import com.google.common.base.Joiner;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
@Component
@@ -15,7 +16,9 @@ public class CaffeineCacheImpl implements CacheUtils {
@Autowired
private CacheCommonConfig cacheCommonConfig;
@Autowired
@Qualifier("caffeineCache")
private Cache<String, Object> caffeineCache;
@Override
@@ -46,4 +49,4 @@ public class CaffeineCacheImpl implements CacheUtils {
caffeineCache.asMap().remove(key);
return true;
}
}
}

16
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FieldAndValueAcquireVisitor.java
View File

@@ -16,6 +16,7 @@ import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.expression.operators.relational.GreaterThan;
import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
import net.sf.jsqlparser.expression.operators.relational.LikeExpression;
import net.sf.jsqlparser.expression.operators.relational.MinorThan;
import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
import net.sf.jsqlparser.schema.Column;
@@ -29,6 +30,21 @@ public class FieldAndValueAcquireVisitor extends ExpressionVisitorAdapter {
this.filterExpressions = filterExpressions;
}
public void visit(LikeExpression expr) {
Expression leftExpression = expr.getLeftExpression();
Expression rightExpression = expr.getRightExpression();
FilterExpression filterExpression = new FilterExpression();
String columnName = null;
if (leftExpression instanceof Column) {
Column column = (Column) leftExpression;
columnName = column.getColumnName();
filterExpression.setFieldName(columnName);
}
filterExpression.setFieldValue(getFieldValue(rightExpression));
filterExpression.setOperator(expr.getStringExpression());
filterExpressions.add(filterExpression);
}
@Override
public void visit(MinorThan expr) {

25
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/FunctionNameReplaceVisitor.java Normal file
View File

@@ -0,0 +1,25 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class FunctionNameReplaceVisitor extends ExpressionVisitorAdapter {
private Map<String, String> functionMap;
public FunctionNameReplaceVisitor(Map<String, String> functionMap) {
this.functionMap = functionMap;
}
public void visit(Function function) {
String replaceFunctionName = functionMap.get(function.getName().toLowerCase());
if (StringUtils.isNotBlank(replaceFunctionName)) {
function.setName(replaceFunctionName);
}
}
}

38
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByFunctionReplaceVisitor.java Normal file
View File

@@ -0,0 +1,38 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.GroupByVisitor;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class GroupByFunctionReplaceVisitor implements GroupByVisitor {
private Map<String, String> functionMap;
public GroupByFunctionReplaceVisitor(Map<String, String> functionMap) {
this.functionMap = functionMap;
}
public void visit(GroupByElement groupByElement) {
groupByElement.getGroupByExpressionList();
ExpressionList groupByExpressionList = groupByElement.getGroupByExpressionList();
List<Expression> groupByExpressions = groupByExpressionList.getExpressions();
for (int i = 0; i < groupByExpressions.size(); i++) {
Expression expression = groupByExpressions.get(i);
if (expression instanceof Function) {
Function function = (Function) expression;
String replaceName = functionMap.get(function.getName().toLowerCase());
if (StringUtils.isNotBlank(replaceName)) {
function.setName(replaceName);
}
}
}
}
}

18
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/GroupByVisitor.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.common.util.jsqlparser;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.statement.select.GroupByElement;
@Slf4j
public class GroupByVisitor implements net.sf.jsqlparser.statement.select.GroupByVisitor {
private boolean hasAggregateFunction = false;
public void visit(GroupByElement groupByElement) {
this.hasAggregateFunction = true;
}
public boolean isHasAggregateFunction() {
return hasAggregateFunction;
}
}

11
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByAcquireVisitor.java
View File

@@ -1,7 +1,9 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import java.util.Set;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter;
@@ -20,6 +22,15 @@ public class OrderByAcquireVisitor extends OrderByVisitorAdapter {
if (expression instanceof Column) {
fields.add(((Column) expression).getColumnName());
}
if (expression instanceof Function) {
Function function = (Function) expression;
List<Expression> expressions = function.getParameters().getExpressions();
for (Expression column : expressions) {
if (column instanceof Column) {
fields.add(((Column) column).getColumnName());
}
}
}
super.visit(orderBy);
}
}

11
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/OrderByReplaceVisitor.java
View File

@@ -1,7 +1,9 @@
package com.tencent.supersonic.common.util.jsqlparser;
import java.util.List;
import java.util.Map;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.OrderByVisitorAdapter;
@@ -21,6 +23,15 @@ public class OrderByReplaceVisitor extends OrderByVisitorAdapter {
if (expression instanceof Column) {
parseVisitorHelper.replaceColumn((Column) expression, fieldToBizName);
}
if (expression instanceof Function) {
Function function = (Function) expression;
List<Expression> expressions = function.getParameters().getExpressions();
for (Expression column : expressions) {
if (column instanceof Column) {
parseVisitorHelper.replaceColumn((Column) column, fieldToBizName);
}
}
}
super.visit(orderBy);
}
}

23
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java
View File

@@ -26,6 +26,7 @@ import org.springframework.util.CollectionUtils;
*/
@Slf4j
public class SqlParserSelectHelper {
public static List<FilterExpression> getFilterExpression(String sql) {
PlainSelect plainSelect = getPlainSelect(sql);
if (Objects.isNull(plainSelect)) {
@@ -132,16 +133,12 @@ public class SqlParserSelectHelper {
}
}
List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
if (orderByElements != null) {
if (!CollectionUtils.isEmpty(orderByElements)) {
for (OrderByElement orderByElement : orderByElements) {
Expression expression = orderByElement.getExpression();
if (expression instanceof Column) {
Column column = (Column) expression;
result.add(column.getColumnName());
}
orderByElement.accept(new OrderByAcquireVisitor(result));
}
}
Expression where = plainSelect.getWhere();
if (where != null) {
where.accept(new ExpressionVisitorAdapter() {
@@ -181,7 +178,17 @@ public class SqlParserSelectHelper {
for (SelectItem selectItem : selectItems) {
selectItem.accept(visitor);
}
return visitor.hasAggregateFunction();
boolean selectFunction = visitor.hasAggregateFunction();
if (selectFunction) {
return true;
}
GroupByElement groupBy = plainSelect.getGroupBy();
if (Objects.nonNull(groupBy)) {
GroupByVisitor replaceVisitor = new GroupByVisitor();
groupBy.accept(replaceVisitor);
return replaceVisitor.isHasAggregateFunction();
}
return false;
}
}

25
common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserUpdateHelper.java
View File

@@ -94,6 +94,31 @@ public class SqlParserUpdateHelper {
return selectStatement.toString();
}
public static String replaceFunction(String sql, Map<String, String> functionMap) {
Select selectStatement = SqlParserSelectHelper.getSelect(sql);
SelectBody selectBody = selectStatement.getSelectBody();
if (!(selectBody instanceof PlainSelect)) {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectBody;
//1. replace where dataDiff function
Expression where = plainSelect.getWhere();
FunctionNameReplaceVisitor visitor = new FunctionNameReplaceVisitor(functionMap);
if (Objects.nonNull(where)) {
where.accept(visitor);
}
GroupByElement groupBy = plainSelect.getGroupBy();
if (Objects.nonNull(groupBy)) {
GroupByFunctionReplaceVisitor replaceVisitor = new GroupByFunctionReplaceVisitor(functionMap);
groupBy.accept(replaceVisitor);
}
for (SelectItem selectItem : plainSelect.getSelectItems()) {
selectItem.accept(visitor);
}
return selectStatement.toString();
}
public static String replaceFunction(String sql) {
Select selectStatement = SqlParserSelectHelper.getSelect(sql);