topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]Remove fixed TimeEnums fields.

Browse Source
This commit is contained in:
jerryjzhang
2024-12-11 21:33:43 +08:00
parent f97ac1da83
commit 8c44c9f42f
36 changed files with 142 additions and 722 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoFormatProcessor.java
View File

@@ -8,7 +8,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
@@ -21,12 +20,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
/**
@@ -216,9 +210,6 @@ public class ParseInfoFormatProcessor implements ParseResultProcessor {
}
private static boolean isPartitionDimension(DataSetSchema dataSetSchema, String sqlFieldName) {
if (TimeDimensionEnum.containsTimeDimension(sqlFieldName)) {
return true;
}
if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
return false;

61
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java
View File

@@ -1,35 +1,17 @@
package com.tencent.supersonic.common.jsqlparser;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.Parenthesis;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.*;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SelectItem;
import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
import net.sf.jsqlparser.statement.select.SetOperationList;
import net.sf.jsqlparser.statement.select.*;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
/** Sql Parser add Helper */
@Slf4j
@@ -144,42 +126,7 @@ public class SqlAddHelper {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectStatement;
List<String> chNameList = TimeDimensionEnum.getChNameList();
Boolean dateWhere = false;
for (String chName : chNameList) {
if (expression.toString().contains(chName)) {
dateWhere = true;
}
}
List<PlainSelect> plainSelectList = SqlSelectHelper.getWithItem(selectStatement);
if (!CollectionUtils.isEmpty(plainSelectList) && dateWhere) {
List<String> withNameList = SqlSelectHelper.getWithName(sql);
for (int i = 0; i < plainSelectList.size(); i++) {
if (plainSelectList.get(i).getFromItem() instanceof Table) {
Table table = (Table) plainSelectList.get(i).getFromItem();
if (withNameList.contains(table.getName())) {
continue;
}
}
Set<String> result = new HashSet<>();
List<PlainSelect> subPlainSelectList = new ArrayList<>();
subPlainSelectList.add(plainSelectList.get(i));
SqlSelectHelper.getWhereFields(subPlainSelectList, result);
if (TimeDimensionEnum.containsZhTimeDimension(new ArrayList<>(result))) {
continue;
}
Expression subWhere = plainSelectList.get(i).getWhere();
addWhere(plainSelectList.get(i), subWhere, expression);
}
return selectStatement.toString();
}
if (plainSelect.getFromItem() instanceof ParenthesedSelect && dateWhere) {
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem();
PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect();
Expression subWhere = subPlainSelect.getWhere();
addWhere(subPlainSelect, subWhere, expression);
return selectStatement.toString();
}
Expression where = plainSelect.getWhere();
addWhere(plainSelect, where, expression);

6
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.common.jsqlparser;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.statement.select.PlainSelect;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@@ -12,7 +12,7 @@ import java.util.Objects;
@Slf4j
public class SqlDateSelectHelper {
public static DateVisitor.DateBoundInfo getDateBoundInfo(String sql) {
public static DateVisitor.DateBoundInfo getDateBoundInfo(String sql, String dateField) {
List<PlainSelect> plainSelectList = SqlSelectHelper.getPlainSelect(sql);
if (plainSelectList.size() != 1) {
return null;
@@ -25,7 +25,7 @@ public class SqlDateSelectHelper {
if (Objects.isNull(where)) {
return null;
}
DateVisitor dateVisitor = new DateVisitor(TimeDimensionEnum.getChNameList());
DateVisitor dateVisitor = new DateVisitor(Collections.singletonList(dateField));
where.accept(dateVisitor);
return dateVisitor.getDateBoundInfo();
}

15
common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import lombok.Data;
import org.springframework.util.CollectionUtils;
@@ -40,6 +39,8 @@ public class DateConf {
private boolean groupByDate;
private String dateField;
public List<String> getDateList() {
if (!CollectionUtils.isEmpty(dateList)) {
return dateList;
@@ -49,18 +50,6 @@ public class DateConf {
return DateUtils.getDateList(startDateStr, endDateStr, getPeriod());
}
public String getGroupByTimeDimension() {
if (DatePeriodEnum.DAY.equals(period)) {
return TimeDimensionEnum.DAY.getName();
} else if (DatePeriodEnum.WEEK.equals(period)) {
return TimeDimensionEnum.WEEK.getName();
} else if (DatePeriodEnum.MONTH.equals(period)) {
return TimeDimensionEnum.MONTH.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
}
@Override
public boolean equals(Object o) {
if (this == o) {

72
common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java
View File

@@ -1,73 +1,7 @@
package com.tencent.supersonic.common.pojo.enums;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public enum TimeDimensionEnum {
DAY("sys_imp_date", "数据日期"),
WEEK("sys_imp_week", "数据日期_周"),
MONTH("sys_imp_month", "数据日期_月");
private String name;
private String chName;
TimeDimensionEnum(String name, String chName) {
this.name = name;
this.chName = chName;
}
public static boolean containsTimeDimension(String fieldName) {
if (getNameList().contains(fieldName) || getChNameList().contains(fieldName)) {
return true;
}
return false;
}
public static List<String> getNameList() {
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getName)
.collect(Collectors.toList());
}
public static List<String> getChNameList() {
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getChName)
.collect(Collectors.toList());
}
public static Map<String, String> getChNameToNameMap() {
return Arrays.stream(TimeDimensionEnum.values()).collect(Collectors
.toMap(TimeDimensionEnum::getChName, TimeDimensionEnum::getName, (k1, k2) -> k1));
}
public static Map<String, String> getNameToNameMap() {
return Arrays.stream(TimeDimensionEnum.values()).collect(Collectors
.toMap(TimeDimensionEnum::getName, TimeDimensionEnum::getName, (k1, k2) -> k1));
}
public String getName() {
return name;
}
public String getChName() {
return chName;
}
/**
* Determine if a time dimension field is included in a Chinese/English text field
*
* @param fields field
* @return true/false
*/
public static boolean containsZhTimeDimension(List<String> fields) {
if (CollectionUtils.isEmpty(fields)) {
return false;
}
return fields.stream().anyMatch(field -> containsTimeDimension(field));
}
DAY,
WEEK,
MONTH;
}

117
common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java
View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -21,22 +20,14 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.regex.Pattern;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.*;
@Slf4j
@Component
@Data
public class DateModeUtils {
private final String sysDateCol = TimeDimensionEnum.DAY.getName();
private final String sysDateMonthCol = TimeDimensionEnum.MONTH.getName();
private final String sysDateWeekCol = TimeDimensionEnum.WEEK.getName();
@Value("${s2.query.parameter.sys.zipper.begin:start_}")
private String sysZipperDateColBegin;
@@ -60,8 +51,8 @@ public class DateModeUtils {
public String hasDataModeStr(ItemDateResp dateDate, DateConf dateInfo) {
if (Objects.isNull(dateDate) || StringUtils.isEmpty(dateDate.getStartDate())
|| StringUtils.isEmpty(dateDate.getStartDate())) {
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(),
sysDateCol, dateInfo.getEndDate());
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
dateInfo.getStartDate(), dateInfo.getDateField(), dateInfo.getEndDate());
} else {
log.info("dateDate:{}", dateDate);
}
@@ -79,27 +70,28 @@ public class DateModeUtils {
dateFormatStr, ChronoUnit.DAYS);
LocalDate dateMax = endData;
LocalDate dateMin = dateMax.minusDays(unit - 1);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol,
dateMax);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
dateMin, dateInfo.getDateField(), dateMax);
}
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(),
dateFormatStr, ChronoUnit.MONTHS);
return generateMonthSql(endData, unit, dateFormatStr);
return generateMonthSql(endData, unit, dateFormatStr, dateInfo);
}
}
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(),
sysDateCol, dateInfo.getEndDate());
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
dateInfo.getStartDate(), dateInfo.getDateField(), dateInfo.getEndDate());
}
public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr) {
public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr,
DateConf dateConf) {
LocalDate dateMax = endData;
List<String> months = generateMonthStr(dateMax, unit, dateFormatStr);
if (!CollectionUtils.isEmpty(months)) {
StringJoiner joiner = new StringJoiner(",");
months.stream().forEach(month -> joiner.add("'" + month + "'"));
return String.format("(%s in (%s))", sysDateCol, joiner.toString());
return String.format("(%s in (%s))", dateConf.getDateField(), joiner.toString());
}
return "";
}
@@ -116,8 +108,8 @@ public class DateModeUtils {
public String recentDayStr(ItemDateResp dateDate, DateConf dateInfo) {
ImmutablePair<String, String> dayRange = recentDay(dateDate, dateInfo);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dayRange.left, sysDateCol,
dayRange.right);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dayRange.left,
dateInfo.getDateField(), dayRange.right);
}
public ImmutablePair<String, String> recentDay(ItemDateResp dateDate, DateConf dateInfo) {
@@ -134,24 +126,24 @@ public class DateModeUtils {
return ImmutablePair.of(start, dateDate.getEndDate());
}
public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr) {
public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr, DateConf dateInfo) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
String endStr = endData.format(formatter);
String start = endData.minusMonths(unit).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol,
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), start, dateInfo.getDateField(),
endStr);
}
public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) {
List<ImmutablePair<String, String>> range = recentMonth(dateDate, dateInfo);
if (range.size() == 1) {
return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, range.get(0).left,
sysDateMonthCol, range.get(0).right);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), range.get(0).left,
dateInfo.getDateField(), range.get(0).right);
}
if (range.size() > 0) {
StringJoiner joiner = new StringJoiner(",");
range.stream().forEach(month -> joiner.add("'" + month.left + "'"));
return String.format("(%s in (%s))", sysDateCol, joiner.toString());
return String.format("(%s in (%s))", dateInfo.getDateField(), joiner.toString());
}
return "";
}
@@ -181,17 +173,17 @@ public class DateModeUtils {
return ret;
}
public String recentWeekStr(LocalDate endData, Long unit) {
public String recentWeekStr(LocalDate endData, Long unit, DateConf dataInfo) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT);
String start = endData.minusDays(unit * 7).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, start, sysDateWeekCol,
return String.format("(%s >= '%s' and %s <= '%s')", dataInfo.getDateField(), start, dataInfo.getDateField(),
endData.format(formatter));
}
public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) {
ImmutablePair<String, String> dayRange = recentWeek(dateDate, dateInfo);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, dayRange.left,
sysDateWeekCol, dayRange.right);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dayRange.left,
dateInfo.getDateField(), dayRange.right);
}
public ImmutablePair<String, String> recentWeek(ItemDateResp dateDate, DateConf dateInfo) {
@@ -242,26 +234,27 @@ public class DateModeUtils {
* @return
*/
public String betweenDateStr(DateConf dateInfo) {
String dateField = dateInfo.getDateField();
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
// startDate YYYYMM
if (!dateInfo.getStartDate().contains(Constants.MINUS)) {
return String.format("%s >= '%s' and %s <= '%s'", sysDateMonthCol,
dateInfo.getStartDate(), sysDateMonthCol, dateInfo.getEndDate());
return String.format("%s >= '%s' and %s <= '%s'", dateField,
dateInfo.getStartDate(), dateField, dateInfo.getEndDate());
}
LocalDate endData =
LocalDate.parse(dateInfo.getEndDate(), DateTimeFormatter.ofPattern(DAY_FORMAT));
LocalDate startData = LocalDate.parse(dateInfo.getStartDate(),
DateTimeFormatter.ofPattern(DAY_FORMAT));
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT);
return String.format("%s >= '%s' and %s <= '%s'", sysDateMonthCol,
startData.format(formatter), sysDateMonthCol, endData.format(formatter));
return String.format("%s >= '%s' and %s <= '%s'", dateField,
startData.format(formatter), dateField, endData.format(formatter));
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
return String.format("%s >= '%s' and %s <= '%s'", sysDateWeekCol,
dateInfo.getStartDate(), sysDateWeekCol, dateInfo.getEndDate());
return String.format("%s >= '%s' and %s <= '%s'", dateField, dateInfo.getStartDate(),
dateField, dateInfo.getEndDate());
}
return String.format("%s >= '%s' and %s <= '%s'", sysDateCol, dateInfo.getStartDate(),
sysDateCol, dateInfo.getEndDate());
return String.format("%s >= '%s' and %s <= '%s'", dateField, dateInfo.getStartDate(),
dateField, dateInfo.getEndDate());
}
/**
@@ -273,12 +266,12 @@ public class DateModeUtils {
public String listDateStr(DateConf dateInfo) {
StringJoiner joiner = new StringJoiner(COMMA);
dateInfo.getDateList().stream().forEach(date -> joiner.add(APOSTROPHE + date + APOSTROPHE));
String dateCol = sysDateCol;
String dateCol = dateInfo.getDateField();
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
dateCol = sysDateMonthCol;
dateCol = dateInfo.getDateField();
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
dateCol = sysDateWeekCol;
dateCol = dateInfo.getDateField();
}
return String.format("(%s in (%s))", dateCol, joiner.toString());
}
@@ -299,25 +292,26 @@ public class DateModeUtils {
if (DatePeriodEnum.DAY.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
LocalDate dateMin = dateMax.minusDays(unit - 1);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol,
dateMax);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dateMin,
dateInfo.getDateField(), dateMax);
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return recentWeekStr(dateMax, unit.longValue());
return recentWeekStr(dateMax, unit.longValue(), dateInfo);
}
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT);
return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT, dateInfo);
}
if (DatePeriodEnum.YEAR.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return recentMonthStr(dateMax, unit.longValue() * 12, MONTH_FORMAT);
return recentMonthStr(dateMax, unit.longValue() * 12, MONTH_FORMAT, dateInfo);
}
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol,
LocalDate.now().minusDays(2), sysDateCol, LocalDate.now().minusDays(1));
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
LocalDate.now().minusDays(2), dateInfo.getDateField(),
LocalDate.now().minusDays(1));
}
public String getDateWhereStr(DateConf dateInfo) {
@@ -349,32 +343,7 @@ public class DateModeUtils {
}
public String getSysDateCol(DateConf dateInfo) {
if (DatePeriodEnum.DAY.equals(dateInfo.getPeriod())) {
return sysDateCol;
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
return sysDateWeekCol;
}
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
return sysDateMonthCol;
}
return "";
return dateInfo.getDateField();
}
public boolean isDateStr(String date) {
return Pattern.matches("[\\d\\s-:]+", date);
}
public DatePeriodEnum getPeriodByCol(String col) {
if (sysDateCol.equalsIgnoreCase(col)) {
return DatePeriodEnum.DAY;
}
if (sysDateWeekCol.equalsIgnoreCase(col)) {
return DatePeriodEnum.WEEK;
}
if (sysDateMonthCol.equalsIgnoreCase(col)) {
return DatePeriodEnum.MONTH;
}
return null;
}
}

12
common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java
View File

@@ -2,8 +2,10 @@ package com.tencent.supersonic.common.jsqlparser;
import com.tencent.supersonic.common.jsqlparser.DateVisitor.DateBoundInfo;
import org.junit.Assert;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@Disabled
class SqlDateSelectHelperTest {
@Test
@@ -11,31 +13,31 @@ class SqlDateSelectHelperTest {
String sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1";
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getLowerBound(), ">=");
Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getLowerBound(), ">");
Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getUpperBound(), "<=");
Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getUpperBound(), "<");
Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-10-17' "
+ "AND 数据日期 <= '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getUpperBound(), "<=");
Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17");
Assert.assertEquals(dateBoundInfo.getLowerBound(), ">=");

7
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/DataSetSchema.java
View File

@@ -89,12 +89,7 @@ public class DataSetSchema implements Serializable {
}
public SchemaElement getPartitionDimension() {
for (SchemaElement dimension : dimensions) {
if (dimension.isPartitionTime()) {
return dimension;
}
}
return null;
return dimensions.stream().filter(SchemaElement::isPartitionTime).findFirst().orElse(null);
}
public SchemaElement getPrimaryKey() {

23
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -13,12 +12,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
/**
@@ -48,17 +42,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
protected Map<String, String> getFieldNameMap(ChatQueryContext chatQueryContext,
Long dataSetId) {
Map<String, String> result = getFieldNameMapFromDB(chatQueryContext, dataSetId);
if (chatQueryContext.containsPartitionDimensions(dataSetId)) {
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
}
return result;
return getFieldNameMapFromDB(chatQueryContext, dataSetId);
}
private static Map<String, String> getFieldNameMapFromDB(ChatQueryContext chatQueryContext,
@@ -126,7 +110,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
}
return elements.stream();
}).collect(Collectors.toSet());
dimensions.add(TimeDimensionEnum.DAY.getChName());
return dimensions;
}
@@ -142,8 +125,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
SemanticParseInfo semanticParseInfo) {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL();
Set<String> removeFieldNames = new HashSet<>();
removeFieldNames.addAll(TimeDimensionEnum.getChNameList());
removeFieldNames.addAll(TimeDimensionEnum.getNameList());
Map<String, String> fieldNameMap =
getFieldNameMapFromDB(chatQueryContext, semanticParseInfo.getDataSetId());
removeFieldNames.removeIf(fieldName -> fieldNameMap.containsKey(fieldName));

5
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java
View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.jsqlparser.SqlValidHelper;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
@@ -53,10 +52,6 @@ public class GroupByCorrector extends BaseSemanticCorrector {
if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) {
return false;
}
// if only date in select not add group by.
if (selectFields.size() == 1 && TimeDimensionEnum.containsZhTimeDimension(selectFields)) {
return false;
}
if (SqlSelectHelper.hasGroupBy(correctS2SQL)) {
log.debug("No need to add 'group by', existed 'group by' in s2sql:{}", correctS2SQL);
return false;

16
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java
View File

@@ -1,14 +1,8 @@
package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.jsqlparser.AggregateEnum;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlAsHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.jsqlparser.*;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -21,11 +15,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
/** Perform schema corrections on the Schema information in S2SQL. */
@@ -144,8 +134,6 @@ public class SchemaCorrector extends BaseSemanticCorrector {
Set<String> removeFieldNames = whereExpressionList.stream()
.filter(fieldExpression -> StringUtils.isBlank(fieldExpression.getFunction()))
.filter(fieldExpression -> !TimeDimensionEnum
.containsTimeDimension(fieldExpression.getFieldName()))
.filter(fieldExpression -> FilterOperatorEnum.EQUALS.getValue()
.equals(fieldExpression.getOperator()))
.filter(fieldExpression -> dimensions.contains(fieldExpression.getFieldName()))

7
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlDateSelectHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -44,8 +43,7 @@ public class TimeCorrector extends BaseSemanticCorrector {
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())
|| TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
return;
}
String partitionDimension = dataSetSchema.getPartitionDimension().getName();
@@ -75,7 +73,8 @@ public class TimeCorrector extends BaseSemanticCorrector {
private void addLowerBoundDate(SemanticParseInfo semanticParseInfo) {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL();
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL);
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL,
semanticParseInfo.getDateInfo().getDateField());
if (dateBoundInfo != null && StringUtils.isBlank(dateBoundInfo.getLowerBound())
&& StringUtils.isNotBlank(dateBoundInfo.getUpperBound())

6
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java
View File

@@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.chat.parser.rule;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.parser.SemanticParser;
@@ -57,6 +59,10 @@ public class TimeRangeParser implements SemanticParser {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
SemanticParseInfo parseInfo = query.getParseInfo();
if (queryContext.containsPartitionDimensions(parseInfo.getDataSetId())) {
DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap()
.get(parseInfo.getDataSetId());
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
dateConf.setDateField(partitionDimension.getName());
parseInfo.setDateInfo(dateConf);
}
parseInfo.setScore(parseInfo.getScore() + dateConf.getDetectWord().length());

23
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java
View File

@@ -4,18 +4,8 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.request.*;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.BaseSemanticQuery;
import com.tencent.supersonic.headless.chat.query.QueryManager;
@@ -25,13 +15,8 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.stream.Collectors;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TERM;
@@ -233,8 +218,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
protected void convertBizNameToName(DataSetSchema dataSetSchema,
QueryStructReq queryStructReq) {
Map<String, String> bizNameToName = dataSetSchema.getBizNameToName();
bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap());
List<Order> orders = queryStructReq.getOrders();
if (CollectionUtils.isNotEmpty(orders)) {
for (Order order : orders) {

8
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java
View File

@@ -3,14 +3,13 @@ package com.tencent.supersonic.headless.chat.query.rule.detail;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery;
import lombok.extern.slf4j.Slf4j;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -33,10 +32,13 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery {
chatQueryContext.getSemanticSchema().getDataSetSchemaMap();
DataSetSchema dataSetSchema = dataSetSchemaMap.get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getDetailTypeTimeDefaultConfig();
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
if (Objects.nonNull(partitionDimension) && Objects.nonNull(timeDefaultConfig)
&& Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
DateConf dateInfo = new DateConf();
dateInfo.setDateField(partitionDimension.getName());
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().minusDays(unit).toString();
String endDate = startDate;

12
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java
View File

@@ -3,14 +3,13 @@ package com.tencent.supersonic.headless.chat.query.rule.metric;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery;
import lombok.extern.slf4j.Slf4j;
import java.time.LocalDate;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC;
@@ -40,10 +39,12 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap()
.get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig();
DateConf dateInfo = new DateConf();
// 加上时间!=-1 判断
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
if (Objects.nonNull(partitionDimension) && Objects.nonNull(timeDefaultConfig)
&& Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
DateConf dateInfo = new DateConf();
dateInfo.setDateField(partitionDimension.getName());
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().minusDays(unit).toString();
String endDate = startDate;
@@ -55,7 +56,6 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
dateInfo.setPeriod(timeDefaultConfig.getPeriod());
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
// 时间不为-1才设置时间，所以移到这里
parseInfo.setDateInfo(dateInfo);
}
}

19
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java
View File

@@ -7,9 +7,7 @@ import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
@@ -22,13 +20,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@@ -179,14 +171,7 @@ public class QueryReqBuilder {
if (Objects.isNull(dateConf)) {
return "";
}
String dateField = TimeDimensionEnum.DAY.getName();
if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) {
dateField = TimeDimensionEnum.MONTH.getName();
}
if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) {
dateField = TimeDimensionEnum.WEEK.getName();
}
return dateField;
return dateConf.getDateField();
}
public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo,

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java
View File

@@ -41,7 +41,6 @@ public class DefaultDimValueConverter implements QueryConverter {
}
String sql = queryStatement.getSqlQueryParam().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql).stream()
.filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;

17
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
@@ -350,15 +349,8 @@ public class MetricRatioConverter implements QueryConverter {
return CollectionUtils.isEmpty(groups) ? aggStr : String.join(",", groups) + "," + aggStr;
}
private String getGroupDimWithOutTime(StructQueryParam structQueryParam) {
String timeDim = getTimeDim(structQueryParam);
return structQueryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim))
.collect(Collectors.joining(","));
}
private static String getTimeDim(StructQueryParam structQueryParam) {
DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
return dateModeUtils.getSysDateCol(structQueryParam.getDateInfo());
return structQueryParam.getDateInfo().getDateField();
}
private static String getLimit(StructQueryParam structQueryParam) {
@@ -385,13 +377,6 @@ public class MetricRatioConverter implements QueryConverter {
return sqlGenerateUtils.getSelectField(agg);
}
private String getGroupBy(StructQueryParam structQueryParam) {
if (CollectionUtils.isEmpty(structQueryParam.getGroups())) {
return "";
}
return "group by " + String.join(",", structQueryParam.getGroups());
}
private static String getOrderBy(StructQueryParam structQueryParam) {
return "order by " + getTimeDim(structQueryParam) + " desc";
}

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
@@ -115,8 +114,8 @@ public class SqlQueryConverter implements QueryConverter {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(),
SchemaItem::getBizName, (k1, k2) -> k1));
dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.DAY.getName());
// dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
// TimeDimensionEnum.DAY.getName());
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase()))
@@ -167,8 +166,6 @@ public class SqlQueryConverter implements QueryConverter {
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
dimensionResults.putAll(metricResults);
return dimensionResults;
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java
View File

@@ -2,8 +2,6 @@ package com.tencent.supersonic.headless.core.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
@@ -53,7 +51,7 @@ public class DataTransformUtils {
private static String getRowKey(Map<String, Object> originalRow, List<String> groups) {
List<Object> values = Lists.newArrayList();
for (String key : originalRow.keySet()) {
if (groups.contains(key) && !TimeDimensionEnum.getNameList().contains(key)) {
if (groups.contains(key)) {
values.add(originalRow.get(key));
}
}
@@ -61,12 +59,6 @@ public class DataTransformUtils {
}
private static String getTimeDimension(DateConf dateConf) {
if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
return dateConf.getDateField();
}
}

20
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -7,7 +7,6 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
@@ -29,19 +28,11 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
/** tools functions to analyze queryStructReq */
@Component
@@ -68,13 +59,8 @@ public class SqlGenerateUtils {
if (group.contains(JOIN_UNDERLINE)) {
group = group.split(JOIN_UNDERLINE)[1];
}
if (!TimeDimensionEnum.getNameList().contains(group)) {
locate++;
sb.append(group).append(" as ").append("name").append(locate).append(",");
} else {
sb.append(group).append(",");
}
}
locate = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {
locate++;
@@ -271,10 +257,6 @@ public class SqlGenerateUtils {
return true;
}
public String generateInternalMetricName(String modelBizName) {
return modelBizName + UNDERLINE + executorConfig.getInternalMetricNameSuffix();
}
public String generateDerivedMetric(final List<MetricSchemaResp> metricResps,
final Set<String> allFields, final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps, final String expression,

102
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java
View File

@@ -1,102 +0,0 @@
package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** generate system time dimension tools */
@Slf4j
public class SysTimeDimensionBuilder {
// Defines the regular expression pattern for the time keyword
private static final Pattern TIME_KEYWORD_PATTERN =
Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b",
Pattern.CASE_INSENSITIVE);
public static void addSysTimeDimension(List<Dimension> dims, DbAdaptor engineAdaptor) {
log.debug("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor);
Dimension timeDim = getTimeDim(dims);
if (timeDim == null) {
timeDim = Dimension.getDefault();
// todo not find the time dimension
return;
}
dims.add(generateSysDayDimension(timeDim, engineAdaptor));
dims.add(generateSysWeekDimension(timeDim, engineAdaptor));
dims.add(generateSysMonthDimension(timeDim, engineAdaptor));
log.debug("addSysTimeDimension after:{}, engineAdaptor:{}", dims, engineAdaptor);
}
private static Dimension generateSysDayDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.DAY.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.DAY.name().toLowerCase());
typeParams.setIsPrimary("true");
dim.setTypeParams(typeParams);
return dim;
}
private static Dimension generateSysWeekDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.WEEK.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.WEEK.name().toLowerCase());
typeParams.setIsPrimary("false");
dim.setTypeParams(typeParams);
return dim;
}
private static Dimension generateSysMonthDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.MONTH.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.MONTH.name().toLowerCase());
typeParams.setIsPrimary("false");
dim.setTypeParams(typeParams);
return dim;
}
private static boolean containsTimeKeyword(String fieldName) {
Matcher matcher = TIME_KEYWORD_PATTERN.matcher(fieldName);
return matcher.find();
}
// Check whether the time field contains keywords,Generation time expression
private static String generateTimeExpr(Dimension timeDim, String dateType,
DbAdaptor engineAdaptor) {
String bizName = timeDim.getBizName();
String dateFormat = timeDim.getDateFormat();
if (containsTimeKeyword(bizName)) {
String bizNameWithBackticks = String.format("`%s`", bizName);
return engineAdaptor.getDateFormat(dateType, dateFormat, bizNameWithBackticks);
} else {
return engineAdaptor.getDateFormat(dateType, dateFormat, bizName);
}
}
private static Dimension getTimeDim(List<Dimension> timeDims) {
for (Dimension dim : timeDims) {
if (dim.getType().equals(DimensionType.partition_time)) {
return dim;
}
}
return null;
}
}

43
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java
View File

@@ -4,26 +4,12 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.SqlEvaluation;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.ChatWorkflowState;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector;
import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector;
@@ -40,13 +26,7 @@ import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -223,9 +203,6 @@ public class S2ChatLayerService implements ChatLayerService {
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
// topN metrics
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()).limit(topN)
@@ -256,20 +233,6 @@ public class S2ChatLayerService implements ChatLayerService {
return termMap;
}
/**
* * get time dimension SchemaElementMatch
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element = SchemaElement.builder().dataSetId(dataSetId)
.dataSetName(dataSetName).type(SchemaElementType.DIMENSION)
.bizName(TimeDimensionEnum.DAY.getName()).build();
return SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName())
.word(TimeDimensionEnum.DAY.getChName()).similarity(1L)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName())

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -5,8 +5,9 @@ import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.*;
import com.tencent.supersonic.headless.api.pojo.response.*;
@@ -210,8 +211,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
List<Dimension> timeDims = modelResp.getTimeDimension();
if (CollectionUtils.isNotEmpty(timeDims)) {
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(),
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate());
queryDimValueReq.getDateInfo().getDateField(),
queryDimValueReq.getDateInfo().getStartDate(),
queryDimValueReq.getDateInfo().getDateField(),
queryDimValueReq.getDateInfo().getEndDate());
}
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%"

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
View File

@@ -7,14 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.*;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@@ -30,8 +23,6 @@ public class ModelYamlManager {
public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp,
DatabaseResp databaseResp) {
ModelDetail modelDetail = modelResp.getModelDetail();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
dataModelYamlTpl.setType(databaseResp.getType());
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);

35
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -7,13 +7,7 @@ import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.FileUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.*;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
@@ -36,21 +30,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.*;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@@ -258,7 +241,7 @@ public class DownloadServiceImpl implements DownloadService {
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setGroups(dimensionResps.stream().map(DimensionResp::getBizName)
.collect(Collectors.toList()));
queryStructReq.getGroups().add(0, getTimeDimension(dateConf));
queryStructReq.getGroups().add(0, dateConf.getDateField());
Aggregator aggregator = new Aggregator();
aggregator.setColumn(metricResp.getBizName());
queryStructReq.setAggregators(Lists.newArrayList(aggregator));
@@ -268,16 +251,6 @@ public class DownloadServiceImpl implements DownloadService {
return queryStructReq;
}
private String getTimeDimension(DateConf dateConf) {
if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
}
private Map<String, List<MetricResp>> getMetricMap(List<MetricResp> metricResps) {
for (MetricResp metricResp : metricResps) {
List<DrillDownDimension> drillDownDimensions =

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -723,7 +723,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateInfo = queryMetricReq.getDateInfo();
if (Objects.nonNull(dateInfo) && dateInfo.isGroupByDate()) {
queryStructReq.getGroups().add(dateInfo.getGroupByTimeDimension());
queryStructReq.getGroups().add(dateInfo.getDateField());
}
if (!CollectionUtils.isEmpty(dimensionBizNames)) {
queryStructReq.getGroups().addAll(dimensionBizNames);

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
@@ -26,12 +25,7 @@ import org.springframework.util.StringUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
@Service
@Slf4j
@@ -121,8 +115,8 @@ public class TagQueryServiceImpl implements TagQueryService {
User user) {
String sqlPattern = "select max(%s) as %s from tbl where %s is not null";
String sql = String.format(sqlPattern, TimeDimensionEnum.DAY.getName(), maxDateAlias,
tag.getBizName());
String sql = String.format(sqlPattern, itemValueReq.getDateConf().getDateField(),
maxDateAlias, tag.getBizName());
// 添加时间过滤信息
log.info("[queryTagDateFromDbBySql] calculate the maximum time start");
@@ -140,8 +134,8 @@ public class TagQueryServiceImpl implements TagQueryService {
String end = LocalDate.now().minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
sql = sql + String.format(" and ( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
end);
itemValueReq.getDateConf().getDateField(), start,
itemValueReq.getDateConf().getDateField(), end);
}
}
}

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java
View File

@@ -2,17 +2,7 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DimensionConstants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
@@ -21,11 +11,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
public class DataSetSchemaBuilder {
@@ -206,12 +192,6 @@ public class DataSetSchemaBuilder {
private static void setDefaultTimeFormat(SchemaElement dimToAdd,
DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
if (null != dimensionTimeTypeParams && TimeDimensionEnum.DAY.name()
.equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT,
DateUtils.DEFAULT_DATE_FORMAT);
} else {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
}
}
}

57
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
View File

@@ -1,17 +1,7 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.*;
import com.tencent.supersonic.common.pojo.enums.*;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.Dimension;
@@ -20,13 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
@@ -45,22 +29,9 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.*;
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
import static com.tencent.supersonic.common.pojo.Constants.POUND;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.*;
@Slf4j
@Component
@@ -456,14 +427,14 @@ public class DictUtils {
return joiner.toString();
}
public String defaultDateFilter() {
public String defaultDateFilter(DateConf dateConf) {
String format = itemValueDateFormat;
String start = LocalDate.now().minusDays(itemValueDateStart)
.format(DateTimeFormatter.ofPattern(format));
String end = LocalDate.now().minusDays(itemValueDateEnd)
.format(DateTimeFormatter.ofPattern(format));
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
start, TimeDimensionEnum.DAY.getName(), end);
return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start,
dateConf.getDateField(), end);
}
private String generateDictDateFilter(DictItemResp dictItemResp) {
@@ -473,7 +444,7 @@ public class DictUtils {
}
// 未进行设置
if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) {
return defaultDateFilter();
return defaultDateFilter(config.getDateConf());
}
// 全表扫描
if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) {
@@ -481,9 +452,9 @@ public class DictUtils {
}
// 静态日期
if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) {
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
config.getDateConf().getStartDate(), TimeDimensionEnum.DAY.getName(),
config.getDateConf().getEndDate());
return String.format("( %s >= '%s' and %s <= '%s' )",
config.getDateConf().getDateField(), config.getDateConf().getStartDate(),
config.getDateConf().getDateField(), config.getDateConf().getEndDate());
}
// 动态日期
if (DateConf.DateMode.RECENT.equals(config.getDateConf().getDateMode())) {
@@ -519,8 +490,8 @@ public class DictUtils {
String end = LocalDate.now().minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
return String.format("( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
end);
dictItemResp.getConfig().getDateConf().getDateField(), start,
dictItemResp.getConfig().getDateConf().getDateField(), end);
}
}
return "";

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java
View File

@@ -2,14 +2,9 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.MetricService;
import lombok.extern.slf4j.Slf4j;
@@ -61,9 +56,6 @@ public class MetricDrillDownChecker {
}
}
for (String dimensionBizName : groupByFields) {
if (TimeDimensionEnum.containsTimeDimension(dimensionBizName)) {
continue;
}
List<MetricResp> metricResps = getMetrics(metricFields, semanticSchemaResp);
if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) {
DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName);

65
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -2,13 +2,10 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
@@ -31,16 +28,7 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
@@ -274,55 +262,4 @@ public class QueryStructUtils {
return Triple.of("", "", "");
}
public DateConf getDateConfBySql(String sql) {
List<FieldExpression> fieldExpressions = SqlSelectHelper.getFilterExpression(sql);
if (!CollectionUtils.isEmpty(fieldExpressions)) {
Set<String> dateList = new HashSet<>();
String startDate = "";
String endDate = "";
DatePeriodEnum period = null;
for (FieldExpression f : fieldExpressions) {
if (Objects.isNull(f.getFieldName())
|| !internalCols.contains(f.getFieldName().toLowerCase())) {
continue;
}
if (Objects.isNull(f.getFieldValue())
|| !dateModeUtils.isDateStr(f.getFieldValue().toString())) {
continue;
}
period = dateModeUtils.getPeriodByCol(f.getFieldName().toLowerCase());
if (period == null) {
continue;
}
if ("=".equals(f.getOperator())) {
dateList.add(f.getFieldValue().toString());
} else if ("<".equals(f.getOperator()) || "<=".equals(f.getOperator())) {
if (startDate.isEmpty()
|| startDate.compareTo(f.getFieldValue().toString()) > 0) {
startDate = f.getFieldValue().toString();
}
} else if (">".equals(f.getOperator()) || ">=".equals(f.getOperator())) {
if (endDate.isEmpty() || endDate.compareTo(f.getFieldValue().toString()) < 0) {
endDate = f.getFieldValue().toString();
}
}
}
if (period != null) {
DateConf dateConf = new DateConf();
dateConf.setPeriod(period);
if (!CollectionUtils.isEmpty(dateList)) {
dateConf.setDateList(new ArrayList<>(dateList));
dateConf.setDateMode(DateMode.LIST);
return dateConf;
}
if (!"".equals(startDate) && !"".equals(endDate)) {
dateConf.setStartDate(startDate);
dateConf.setEndDate(endDate);
dateConf.setDateMode(DateMode.BETWEEN);
return dateConf;
}
}
}
return null;
}
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
@@ -55,10 +54,6 @@ public class QueryUtils {
private void populateNamePairs(SemanticSchemaResp semanticSchemaResp,
Map<String, String> namePair, Map<String, String> nameTypePair) {
for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) {
namePair.put(timeDimensionEnum.getName(), "date");
nameTypePair.put(timeDimensionEnum.getName(), "DATE");
}
semanticSchemaResp.getMetrics().forEach(metricDesc -> {
namePair.put(metricDesc.getBizName(), metricDesc.getName());
nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name());

1
launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java
View File

@@ -89,6 +89,7 @@ public class BaseTest extends BaseApplication {
dateConf.setDateMode(DateMode.BETWEEN);
dateConf.setEndDate(now().plusDays(0).toString());
dateConf.setStartDate(now().plusDays(-365).toString());
dateConf.setDateField("imp_date");
queryStructReq.setDateInfo(dateConf);
List<Order> orders = new ArrayList<>();

1
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -74,6 +74,7 @@ public class DataUtils {
public static DateConf getDateConf(DateConf.DateMode dateMode, Integer unit,
DatePeriodEnum period, String startDate, String endDate) {
DateConf dateInfo = new DateConf();
dateInfo.setDateField("imp_date");
dateInfo.setUnit(unit);
dateInfo.setDateMode(dateMode);
dateInfo.setPeriod(period);