topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project] supersonic 0.7.0 version backend update (#20)

Browse Source 
Co-authored-by: kanedai <kanedai@tencent.com>
This commit is contained in:
daikon
2023-07-31 11:09:58 +08:00
committed by GitHub
parent 078a81038f
commit e2b2d31429
675 changed files with 13089 additions and 13536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
common/src/main/java/com/tencent/supersonic/common/nlp/ItemDO.java
View File

@@ -1,49 +0,0 @@
package com.tencent.supersonic.common.nlp;
import com.google.common.base.Objects;
import java.io.Serializable;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
@Data
@Setter
@Getter
public class ItemDO implements Serializable {
private Integer domain;
private Integer itemId;
private String name;
private String bizName;
private Long useCnt = 0L;
public ItemDO() {
}
public ItemDO(Integer domain, Integer itemId, String name, String bizName) {
this.domain = domain;
this.itemId = itemId;
this.name = name;
this.bizName = bizName;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ItemDO itemDO = (ItemDO) o;
return Objects.equal(domain, itemDO.domain) && Objects.equal(itemId,
itemDO.itemId) && Objects.equal(name, itemDO.name)
&& Objects.equal(bizName, itemDO.bizName) && Objects.equal(
useCnt, itemDO.useCnt);
}
@Override
public int hashCode() {
return Objects.hashCode(domain, itemId, name, bizName, useCnt);
}
}

52
common/src/main/java/com/tencent/supersonic/common/nlp/MapResult.java
View File

@@ -1,52 +0,0 @@
package com.tencent.supersonic.common.nlp;
import java.io.Serializable;
import java.util.List;
import java.util.Objects;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class MapResult implements Serializable {
private String name;
private List<String> natures;
private int offset = 0;
private double similarity;
private String detectWord;
public MapResult() {
}
public MapResult(String name, List<String> natures, String detectWord) {
this.name = name;
this.natures = natures;
this.detectWord = detectWord;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
MapResult that = (MapResult) o;
return Objects.equals(name, that.name) && Objects.equals(natures, that.natures);
}
@Override
public int hashCode() {
return Objects.hash(name, natures);
}
public void setOffset(int offset) {
this.offset = offset;
}
}

54
common/src/main/java/com/tencent/supersonic/common/nlp/NatureType.java
View File

@@ -1,54 +0,0 @@
package com.tencent.supersonic.common.nlp;
import org.apache.commons.lang3.StringUtils;
/***
* nature type
* such as : metric、dimension etc.
*/
public enum NatureType {
METRIC("metric"),
DIMENSION("dimension"),
VALUE("value"),
DOMAIN("dm"),
ENTITY("entity"),
NUMBER("m"),
SUFFIX("suffix");
private String type;
public static String NATURE_SPILT = "_";
public static String SPACE = " ";
NatureType(String type) {
this.type = type;
}
public String getType() {
return NATURE_SPILT + type;
}
public static NatureType getNatureType(String nature) {
if (StringUtils.isEmpty(nature) || !nature.startsWith(NATURE_SPILT)) {
return null;
}
for (NatureType natureType : values()) {
if (nature.endsWith(natureType.getType())) {
return natureType;
}
}
//domain
String[] natures = nature.split(NatureType.NATURE_SPILT);
if (natures.length == 2 && StringUtils.isNumeric(natures[1])) {
return DOMAIN;
}
//dimension value
if (natures.length == 3 && StringUtils.isNumeric(natures[1]) && StringUtils.isNumeric(natures[2])) {
return VALUE;
}
return null;
}
}

36
common/src/main/java/com/tencent/supersonic/common/nlp/WordNature.java
View File

@@ -1,36 +0,0 @@
package com.tencent.supersonic.common.nlp;
import java.util.Objects;
import lombok.Data;
import lombok.ToString;
/***
* word nature
*/
@Data
@ToString
public class WordNature {
private String word;
private String nature;
private String natureWithFrequency;
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
WordNature that = (WordNature) o;
return Objects.equals(word, that.word) && Objects.equals(natureWithFrequency, that.natureWithFrequency);
}
@Override
public int hashCode() {
return Objects.hash(word, natureWithFrequency);
}
}

2
common/src/main/java/com/tencent/supersonic/common/pojo/Aggregator.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import java.util.List;
import javax.validation.constraints.NotBlank;

11
common/src/main/java/com/tencent/supersonic/common/constant/Constants.java → common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.constant;
package com.tencent.supersonic.common.pojo;
import java.util.regex.Pattern;
@@ -41,7 +41,10 @@ public class Constants {
public static final String DAY = "DAY";
public static final String DAY_FORMAT = "yyyy-MM-dd";
public static final String MONTH_FORMAT = "yyyy-MM";
public static final String TIMES_FORMAT = "yyyy-MM-dd HH:mm:ss.S";
public static final String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
public static final String DAY_FORMAT_INT = "YYYYMMDD";
public static final String MONTH_FORMAT_INT = "YYYYMM";
public static final String MONTH = "MONTH";
public static final String WEEK = "WEEK";
public static final String YEAR = "YEAR";
@@ -53,4 +56,10 @@ public class Constants {
public static final String PAGESIZE_LOWER = "pageSize";
public static final String TRUE_LOWER = "true";
public static final String NULL = "null";
public static final Double MAX_SIMILARITY = 1.0d;
public static final String CONTEXT = "CONTEXT";
public static final String BRACKETS_START = "[";
public static final String BRACKETS_END = "]";
}

13
common/src/main/java/com/tencent/supersonic/common/pojo/DataFormat.java Normal file
View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.common.pojo;
import lombok.Data;
@Data
public class DataFormat {
private boolean needMultiply100;
private Integer decimalPlaces;
}

12
common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java
View File

@@ -2,8 +2,6 @@ package com.tencent.supersonic.common.pojo;
import static java.time.LocalDate.now;
import com.tencent.supersonic.common.constant.Constants;
import java.util.ArrayList;
import java.util.List;
@@ -47,10 +45,10 @@ public class DateConf {
public enum DateMode {
/**
* date mode
* 1 - between, continuous value,
* 2 - list discrete value,
* 3 - recent time units,
* 4 - advance time until data is available
* 1 - BETWEEN_CONTINUOUS, continuous static value, [startDate, endDate]
* 2 - LIST_DISCRETE, discrete static value, [dateList]
* 3 - RECENT_UNITS, dynamic time related to the actual available time of the element, [unit, period]
* 4 - AVAILABLE_TIME, dynamic time which guaranteed to query some data, [startDate, endDate]
*/
BETWEEN_CONTINUOUS, LIST_DISCRETE, RECENT_UNITS, AVAILABLE_TIME
}
@@ -75,4 +73,4 @@ public class DateConf {
sb.append('}');
return sb.toString();
}
}
}

2
common/src/main/java/com/tencent/supersonic/common/pojo/Order.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.common.pojo;
import static com.tencent.supersonic.common.constant.Constants.ASC_UPPER;
import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER;
import com.google.common.base.Objects;
import javax.validation.constraints.NotBlank;

2
common/src/main/java/com/tencent/supersonic/common/request/PageBaseReq.java → common/src/main/java/com/tencent/supersonic/common/pojo/PageBaseReq.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.request;
package com.tencent.supersonic.common.pojo;
import lombok.Data;

20
common/src/main/java/com/tencent/supersonic/common/pojo/QueryAuthorization.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.common.pojo;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@AllArgsConstructor
@Data
@ToString
@NoArgsConstructor
public class QueryAuthorization {
private String domainName;
private List<String> dimensionFilters;
private List<String> dimensionFiltersDesc;
private String message;
}

28
common/src/main/java/com/tencent/supersonic/common/pojo/QueryColumn.java Normal file
View File

@@ -0,0 +1,28 @@
package com.tencent.supersonic.common.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class QueryColumn {
private String name;
private String type;
private String nameEn;
private String showType;
private Boolean authorized = true;
private String dataFormatType;
private DataFormat dataFormat;
public QueryColumn(String nameEn, String type) {
this.type = type;
this.nameEn = nameEn;
}
public void setType(String type) {
this.type = type == null ? null : type;
}
}

2
common/src/main/java/com/tencent/supersonic/common/util/RecordInfo.java → common/src/main/java/com/tencent/supersonic/common/pojo/RecordInfo.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util;
package com.tencent.supersonic.common.pojo;
import com.google.common.base.Objects;
import java.util.Date;

2
common/src/main/java/com/tencent/supersonic/common/result/ResultData.java → common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.result;
package com.tencent.supersonic.common.pojo;
import lombok.Data;

2
common/src/main/java/com/tencent/supersonic/common/result/ReturnCode.java → common/src/main/java/com/tencent/supersonic/common/pojo/ReturnCode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.result;
package com.tencent.supersonic.common.pojo;
public enum ReturnCode {
SUCCESS(200, "success"),

50
common/src/main/java/com/tencent/supersonic/common/pojo/SchemaItem.java
View File

@@ -1,50 +0,0 @@
package com.tencent.supersonic.common.pojo;
import com.google.common.base.Objects;
import com.tencent.supersonic.common.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.enums.StatusEnum;
import com.tencent.supersonic.common.enums.TypeEnums;
import com.tencent.supersonic.common.util.RecordInfo;
import lombok.Data;
@Data
public class SchemaItem extends RecordInfo {
private Long id;
private String name;
private String bizName;
private String description;
private Integer status = StatusEnum.ONLINE.getCode();
private TypeEnums typeEnum;
private Integer sensitiveLevel = SensitiveLevelEnum.LOW.getCode();
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
if (!super.equals(o)) {
return false;
}
SchemaItem that = (SchemaItem) o;
return Objects.equal(id, that.id) && Objects.equal(name, that.name)
&& Objects.equal(bizName, that.bizName) && Objects.equal(
description, that.description) && Objects.equal(status, that.status)
&& typeEnum == that.typeEnum && Objects.equal(sensitiveLevel, that.sensitiveLevel);
}
@Override
public int hashCode() {
return Objects.hashCode(super.hashCode(), id, name, bizName, description, status, typeEnum, sensitiveLevel);
}
}

8
common/src/main/java/com/tencent/supersonic/common/enums/AggOperatorEnum.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum AggOperatorEnum {
@@ -16,15 +16,17 @@ public enum AggOperatorEnum {
PERCENTILE("PERCENTILE"),
RATIO_ROLL("RATIO_ROLL"),
RATIO_OVER("RATIO_OVER"),
UNKNOWN("UNKNOWN");
private String operator;
AggOperatorEnum(String operator) {
this.operator = operator;
}
private String operator;
public String getOperator() {
return operator;
}

2
common/src/main/java/com/tencent/supersonic/common/enums/AggregateTypeEnum.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggregateTypeEnum.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum AggregateTypeEnum {
SUM,

2
common/src/main/java/com/tencent/supersonic/common/enums/ConfigMode.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/ConfigMode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum ConfigMode {

2
common/src/main/java/com/tencent/supersonic/common/enums/ErrorCode.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/ErrorCode.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum ErrorCode {

21
common/src/main/java/com/tencent/supersonic/common/pojo/enums/RatioOverType.java Normal file
View File

@@ -0,0 +1,21 @@
package com.tencent.supersonic.common.pojo.enums;
public enum RatioOverType {
DAY_ON_DAY("日环比"),
WEEK_ON_DAY("周同比"),
WEEK_ON_WEEK("周环比"),
MONTH_ON_WEEK("月同比"),
MONTH_ON_MONTH("月环比"),
YEAR_ON_MONTH("年同比"),
YEAR_ON_YEAR("年环比");
private String showName;
RatioOverType(String showName) {
this.showName = showName;
}
public String getShowName() {
return showName;
}
}

2
common/src/main/java/com/tencent/supersonic/common/enums/SensitiveLevelEnum.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/SensitiveLevelEnum.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum SensitiveLevelEnum {

2
common/src/main/java/com/tencent/supersonic/common/enums/SinkDbEnum.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/SinkDbEnum.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum SinkDbEnum {

2
common/src/main/java/com/tencent/supersonic/common/enums/StatusEnum.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/StatusEnum.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum StatusEnum {

2
common/src/main/java/com/tencent/supersonic/common/enums/TaskStatusEnum.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum TaskStatusEnum {

2
common/src/main/java/com/tencent/supersonic/common/enums/TypeEnums.java → common/src/main/java/com/tencent/supersonic/common/pojo/enums/TypeEnums.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.enums;
package com.tencent.supersonic.common.pojo.enums;
public enum TypeEnums {

2
common/src/main/java/com/tencent/supersonic/common/exception/AccessException.java → common/src/main/java/com/tencent/supersonic/common/pojo/exception/AccessException.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.exception;
package com.tencent.supersonic.common.pojo.exception;
public class AccessException extends RuntimeException {

2
common/src/main/java/com/tencent/supersonic/common/exception/CommonException.java → common/src/main/java/com/tencent/supersonic/common/pojo/exception/CommonException.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.exception;
package com.tencent.supersonic.common.pojo.exception;
import lombok.AllArgsConstructor;
import lombok.Data;

2
common/src/main/java/com/tencent/supersonic/common/exception/InvalidArgumentException.java → common/src/main/java/com/tencent/supersonic/common/pojo/exception/InvalidArgumentException.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.exception;
package com.tencent.supersonic.common.pojo.exception;
public class InvalidArgumentException extends RuntimeException {

2
common/src/main/java/com/tencent/supersonic/common/exception/InvalidPermissionException.java → common/src/main/java/com/tencent/supersonic/common/pojo/exception/InvalidPermissionException.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.exception;
package com.tencent.supersonic.common.pojo.exception;
public class InvalidPermissionException extends RuntimeException {

2
common/src/main/java/com/tencent/supersonic/common/util/mapper/BeanMapper.java → common/src/main/java/com/tencent/supersonic/common/util/BeanMapper.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.mapper;
package com.tencent.supersonic.common.util;
import java.beans.PropertyDescriptor;
import java.util.HashSet;

2
common/src/main/java/com/tencent/supersonic/common/util/context/ContextUtils.java → common/src/main/java/com/tencent/supersonic/common/util/ContextUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.context;
package com.tencent.supersonic.common.util;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeansException;

43
common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java Normal file
View File

@@ -0,0 +1,43 @@
package com.tencent.supersonic.common.util;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
public class DateUtils {
public static Integer currentYear(){
Date date = new Date();
SimpleDateFormat dateFormat= new SimpleDateFormat("yyyy-MM-dd");
String time=dateFormat.format(date).replaceAll("-","");
int year = Integer.parseInt(time.substring(0,4));
return year;
}
public static DateTimeFormatter getDateFormatter(String date, String[] formats) {
for (int i = 0; i < formats.length; i++) {
String format = formats[i];
SimpleDateFormat dateFormat = new SimpleDateFormat(format);
try {
dateFormat.parse(date);
return DateTimeFormatter.ofPattern(format);
} catch (Exception e) {
}
}
return DateTimeFormatter.ofPattern(formats[0]);
}
public static DateTimeFormatter getTimeFormatter(String date, String[] formats) {
for (int i = 0; i < formats.length; i++) {
String format = formats[i];
try {
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(format);
LocalDateTime.parse(date,dateTimeFormatter);
return dateTimeFormatter;
} catch (Exception e) {
}
}
return DateTimeFormatter.ofPattern(formats[0]);
}
}

2
common/src/main/java/com/tencent/supersonic/common/util/http/HttpClientResult.java → common/src/main/java/com/tencent/supersonic/common/util/HttpClientResult.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.http;
package com.tencent.supersonic.common.util;
import java.io.Serializable;

16
common/src/main/java/com/tencent/supersonic/common/util/http/HttpClientUtils.java → common/src/main/java/com/tencent/supersonic/common/util/HttpClientUtils.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.common.util.http;
package com.tencent.supersonic.common.util;
import com.tencent.supersonic.common.util.retry.RetryUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -231,7 +230,7 @@ public class HttpClientUtils {
public static HttpClientResult doPost(String url, Map<String, String> header, Map<String, String> params) {
return doPost(url, null, null, header, params);
}
/**
* Send a get request; Without request header and request parameters
*
@@ -537,7 +536,6 @@ public class HttpClientUtils {
HttpPost httpPost = null;
try {
// File uploadFile = ResourceUtils.getFile("classpath:" + fullFilePath);
File uploadFile = new File(fullFilePath);
inputStream = new FileInputStream(uploadFile);
@@ -641,9 +639,8 @@ public class HttpClientUtils {
public static final String METHOD_NAME = "DELETE";
@Override
public String getMethod() {
return METHOD_NAME;
public HttpDeleteWithBody() {
super();
}
public HttpDeleteWithBody(final String uri) {
@@ -656,8 +653,9 @@ public class HttpClientUtils {
setURI(uri);
}
public HttpDeleteWithBody() {
super();
@Override
public String getMethod() {
return METHOD_NAME;
}
}

6
common/src/main/java/com/tencent/supersonic/common/util/json/JsonUtil.java → common/src/main/java/com/tencent/supersonic/common/util/JsonUtil.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.json;
package com.tencent.supersonic.common.util;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonParser;
@@ -19,6 +19,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@@ -49,6 +51,8 @@ public class JsonUtil {
objectMapper.configure(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS, true);
// Java8日期时间类支持
objectMapper.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY);
objectMapper.registerModule(new JavaTimeModule());
}
/**

2
common/src/main/java/com/tencent/supersonic/common/util/coder/MD5Util.java → common/src/main/java/com/tencent/supersonic/common/util/MD5Util.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.coder;
package com.tencent.supersonic.common.util;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;

2
common/src/main/java/com/tencent/supersonic/common/util/mybatis/PageUtils.java → common/src/main/java/com/tencent/supersonic/common/util/PageUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.mybatis;
package com.tencent.supersonic.common.util;
import com.github.pagehelper.Page;
import com.github.pagehelper.PageInfo;

2
common/src/main/java/com/tencent/supersonic/common/util/retry/RetryUtils.java → common/src/main/java/com/tencent/supersonic/common/util/RetryUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.retry;
package com.tencent.supersonic.common.util;
import java.util.function.Supplier;

2
common/src/main/java/com/tencent/supersonic/common/util/context/S2ThreadContext.java → common/src/main/java/com/tencent/supersonic/common/util/S2ThreadContext.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.context;
package com.tencent.supersonic.common.util;
import com.alibaba.ttl.TransmittableThreadLocal;

4
common/src/main/java/com/tencent/supersonic/common/util/context/ThreadContext.java → common/src/main/java/com/tencent/supersonic/common/util/ThreadContext.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.context;
package com.tencent.supersonic.common.util;
import lombok.Builder;
@@ -6,8 +6,6 @@ import lombok.Data;
import lombok.ToString;
import java.util.Map;
import java.util.Map;
@Builder
@ToString
@Data

2
common/src/main/java/com/tencent/supersonic/common/util/context/ThreadContextConfig.java → common/src/main/java/com/tencent/supersonic/common/util/ThreadContextConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.context;
package com.tencent.supersonic.common.util;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

2
common/src/main/java/com/tencent/supersonic/common/util/yaml/YamlUtils.java → common/src/main/java/com/tencent/supersonic/common/util/YamlUtils.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.yaml;
package com.tencent.supersonic.common.util;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.parser.Feature;

47
common/src/main/java/com/tencent/supersonic/common/util/calcite/SqlParseUtils.java
View File

@@ -19,6 +19,7 @@ import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.util.SqlString;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
/**
* sql parse utils
@@ -66,6 +67,8 @@ public class SqlParseUtils {
case ORDER_BY:
handlerOrderBy(sqlNode, sqlParserInfo);
break;
default:
break;
}
}
@@ -91,31 +94,50 @@ public class SqlParseUtils {
* @param sqlParserInfo
*/
private static void handlerSelect(SqlNode select, SqlParserInfo sqlParserInfo) {
List<String> allFields = sqlParserInfo.getAllFields();
SqlSelect sqlSelect = (SqlSelect) select;
SqlNodeList selectList = sqlSelect.getSelectList();
List<String> allFields = sqlParserInfo.getAllFields();
selectList.getList().forEach(list -> {
Set<String> selectFields = handlerField(list);
sqlParserInfo.getSelectFields().addAll(selectFields);
allFields.addAll(selectFields);
});
String tableName = handlerFrom(sqlSelect.getFrom());
sqlParserInfo.setTableName(tableName);
Set<String> selectFields = handlerSelectField(sqlSelect);
allFields.addAll(selectFields);
}
private static Set<String> handlerSelectField(SqlSelect sqlSelect) {
Set<String> results = new HashSet<>();
if (sqlSelect.getFrom() instanceof SqlBasicCall) {
Set<String> formFields = handlerField(sqlSelect.getFrom());
results.addAll(formFields);
}
sqlSelect.getSelectList().getList().forEach(list -> {
Set<String> selectFields = handlerField(list);
results.addAll(selectFields);
});
if (sqlSelect.hasWhere()) {
allFields.addAll(handlerField(sqlSelect.getWhere()));
Set<String> whereFields = handlerField(sqlSelect.getWhere());
results.addAll(whereFields);
}
if (sqlSelect.hasOrderBy()) {
allFields.addAll(handlerField(sqlSelect.getOrderList()));
Set<String> orderByFields = handlerField(sqlSelect.getOrderList());
results.addAll(orderByFields);
}
SqlNodeList group = sqlSelect.getGroup();
if (group != null) {
group.forEach(groupField -> {
allFields.addAll(handlerField(groupField));
Set<String> groupByFields = handlerField(groupField);
results.addAll(groupByFields);
});
}
return results;
}
/**
@@ -135,6 +157,8 @@ public class SqlParseUtils {
SqlNode sqlNode = sqlBasicCall.getOperandList().get(0);
SqlSelect sqlSelect = (SqlSelect) sqlNode;
return handlerFrom(sqlSelect.getFrom());
default:
break;
}
return "";
}
@@ -155,7 +179,14 @@ public class SqlParseUtils {
break;
case IDENTIFIER:
SqlIdentifier sqlIdentifier = (SqlIdentifier) field;
fields.add(sqlIdentifier.getSimple());
String simpleName = sqlIdentifier.getSimple();
if (StringUtils.isNotEmpty(simpleName)) {
fields.add(simpleName);
}
break;
case SELECT:
SqlSelect sqlSelect = (SqlSelect) field;
fields.addAll(handlerSelectField(sqlSelect));
break;
default:
if (field instanceof SqlBasicCall) {
@@ -254,6 +285,8 @@ public class SqlParseUtils {
SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode;
SqlSelect query = (SqlSelect) sqlOrderBy.query;
return query.getSelectList();
default:
break;
}
return null;
}

48
common/src/test/java/com/tencent/supersonic/common/util/calcite/SqlParseUtilsTest.java
View File

@@ -18,28 +18,35 @@ class SqlParseUtilsTest {
void addAliasToSql() throws SqlParseException {
String addAliasToSql = SqlParseUtils.addAliasToSql(
"select sum(pv) from ( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1");
"select sum(pv) from ( select * from t_1 "
+ "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1");
Assert.assertTrue(addAliasToSql.toLowerCase().contains("as `pv`"));
Assert.assertTrue(addAliasToSql.toLowerCase().contains("as pv"));
}
@Test
void addFieldToSql() throws SqlParseException {
String addFieldToSql = SqlParseUtils.addFieldsToSql(
"select pv from ( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1",
"select pv from ( select * from t_1 "
+ "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1",
Collections.singletonList("uv"));
Assert.assertTrue(addFieldToSql.toLowerCase().contains("uv"));
addFieldToSql = SqlParseUtils.addFieldsToSql(
"select uv from ( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1 order by play_count desc limit 10",
"select uv from ( select * from t_1 "
+ "where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1 "
+ "order by play_count desc limit 10",
Collections.singletonList("pv"));
Assert.assertTrue(addFieldToSql.toLowerCase().contains("pv"));
addFieldToSql = SqlParseUtils.addFieldsToSql(
"select uv from ( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1 where user_id = '张三' order by play_count desc limit 10",
"select uv from "
+ "( select * from t_1 where sys_imp_date >= '2023-07-07' "
+ " and sys_imp_date <= '2023-07-07' "
+ ") as t_sub_1 "
+ "where user_id = '张三' order by play_count desc limit 10",
Collections.singletonList("pv"));
Assert.assertTrue(addFieldToSql.toLowerCase().contains("pv"));
}
@@ -49,7 +56,9 @@ class SqlParseUtilsTest {
void getSqlParseInfo() {
SqlParserInfo sqlParserInfo = SqlParseUtils.getSqlParseInfo(
"select pv from ( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1 ");
"select pv from "
+ "( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' )"
+ " as t_sub_1 ");
Assert.assertTrue(sqlParserInfo.getTableName().equalsIgnoreCase("t_1"));
@@ -59,13 +68,11 @@ class SqlParseUtilsTest {
Assert.assertTrue(collect.contains("pv"));
Assert.assertTrue(!collect.contains("uv"));
List<String> selectFields = sqlParserInfo.getSelectFields().stream().map(field -> field.toLowerCase())
.collect(Collectors.toList());
Assert.assertTrue(selectFields.contains("pv"));
Assert.assertTrue(!selectFields.contains("uv"));
sqlParserInfo = SqlParseUtils.getSqlParseInfo(
"select uv from t_1 order by play_count desc limit 10");
@@ -82,10 +89,13 @@ class SqlParseUtilsTest {
Assert.assertTrue(!selectFields.contains("pv"));
Assert.assertTrue(!selectFields.contains("play_count"));
sqlParserInfo = SqlParseUtils.getSqlParseInfo(
"select uv from ( select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' ) as t_sub_1 where user_id = '1' order by play_count desc limit 10");
"select uv from "
+ "( "
+ " select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' "
+ ") as t_sub_1 "
+ "where user_id = '1' order by play_count desc limit 10"
);
Assert.assertTrue(sqlParserInfo.getTableName().equalsIgnoreCase("t_1"));
collect = sqlParserInfo.getAllFields().stream().map(field -> field.toLowerCase())
@@ -103,4 +113,18 @@ class SqlParseUtilsTest {
Assert.assertTrue(!selectFields.contains("play_count"));
}
@Test
void getWhereFieldTest() {
SqlParserInfo sqlParserInfo = SqlParseUtils.getSqlParseInfo(
"select uv from "
+ " ( "
+ " select * from t_1 where sys_imp_date >= '2023-07-07' and sys_imp_date <= '2023-07-07' and user_id = 22 "
+ " ) as t_sub_1 "
+ " where user_name_元 = 'zhangsan' order by play_count desc limit 10"
);
List<String> collect = sqlParserInfo.getAllFields().stream().map(field -> field.toLowerCase())
.collect(Collectors.toList());
Assert.assertTrue(collect.contains("user_id"));
}
}