topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 02:46:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][chat&headless]Remove deprecated system time fields.

Browse Source
This commit is contained in:
jerryjzhang
2024-12-28 19:50:11 +08:00
parent 6f5e477e3c
commit 6486257c9e
61 changed files with 281 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoFormatProcessor.java
View File

@@ -8,7 +8,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
@@ -21,12 +20,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
/**
@@ -222,9 +216,6 @@ public class ParseInfoFormatProcessor implements ParseResultProcessor {
}
private static boolean isPartitionDimension(DataSetSchema dataSetSchema, String sqlFieldName) {
if (TimeDimensionEnum.containsTimeDimension(sqlFieldName)) {
return true;
}
if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
return false;

61
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java
View File

@@ -1,35 +1,17 @@
package com.tencent.supersonic.common.jsqlparser;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.Parenthesis;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.*;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.select.GroupByElement;
import net.sf.jsqlparser.statement.select.OrderByElement;
import net.sf.jsqlparser.statement.select.ParenthesedSelect;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.SelectItem;
import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
import net.sf.jsqlparser.statement.select.SetOperationList;
import net.sf.jsqlparser.statement.select.*;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
/** Sql Parser add Helper */
@Slf4j
@@ -144,42 +126,7 @@ public class SqlAddHelper {
return sql;
}
PlainSelect plainSelect = (PlainSelect) selectStatement;
List<String> chNameList = TimeDimensionEnum.getChNameList();
Boolean dateWhere = false;
for (String chName : chNameList) {
if (expression.toString().contains(chName)) {
dateWhere = true;
}
}
List<PlainSelect> plainSelectList = SqlSelectHelper.getWithItem(selectStatement);
if (!CollectionUtils.isEmpty(plainSelectList) && dateWhere) {
List<String> withNameList = SqlSelectHelper.getWithName(sql);
for (int i = 0; i < plainSelectList.size(); i++) {
if (plainSelectList.get(i).getFromItem() instanceof Table) {
Table table = (Table) plainSelectList.get(i).getFromItem();
if (withNameList.contains(table.getName())) {
continue;
}
}
Set<String> result = new HashSet<>();
List<PlainSelect> subPlainSelectList = new ArrayList<>();
subPlainSelectList.add(plainSelectList.get(i));
SqlSelectHelper.getWhereFields(subPlainSelectList, result);
if (TimeDimensionEnum.containsZhTimeDimension(new ArrayList<>(result))) {
continue;
}
Expression subWhere = plainSelectList.get(i).getWhere();
addWhere(plainSelectList.get(i), subWhere, expression);
}
return selectStatement.toString();
}
if (plainSelect.getFromItem() instanceof ParenthesedSelect && dateWhere) {
ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem();
PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect();
Expression subWhere = subPlainSelect.getWhere();
addWhere(subPlainSelect, subWhere, expression);
return selectStatement.toString();
}
Expression where = plainSelect.getWhere();
addWhere(plainSelect, where, expression);

6
common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.common.jsqlparser;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.statement.select.PlainSelect;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@@ -12,7 +12,7 @@ import java.util.Objects;
@Slf4j
public class SqlDateSelectHelper {
public static DateVisitor.DateBoundInfo getDateBoundInfo(String sql) {
public static DateVisitor.DateBoundInfo getDateBoundInfo(String sql, String dateField) {
List<PlainSelect> plainSelectList = SqlSelectHelper.getPlainSelect(sql);
if (plainSelectList.size() != 1) {
return null;
@@ -25,7 +25,7 @@ public class SqlDateSelectHelper {
if (Objects.isNull(where)) {
return null;
}
DateVisitor dateVisitor = new DateVisitor(TimeDimensionEnum.getChNameList());
DateVisitor dateVisitor = new DateVisitor(Collections.singletonList(dateField));
where.accept(dateVisitor);
return dateVisitor.getDateBoundInfo();
}

15
common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java
View File

@@ -1,7 +1,6 @@
package com.tencent.supersonic.common.pojo;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import lombok.Data;
import org.springframework.util.CollectionUtils;
@@ -40,6 +39,8 @@ public class DateConf {
private boolean groupByDate;
private String dateField;
public List<String> getDateList() {
if (!CollectionUtils.isEmpty(dateList)) {
return dateList;
@@ -49,18 +50,6 @@ public class DateConf {
return DateUtils.getDateList(startDateStr, endDateStr, getPeriod());
}
public String getGroupByTimeDimension() {
if (DatePeriodEnum.DAY.equals(period)) {
return TimeDimensionEnum.DAY.getName();
} else if (DatePeriodEnum.WEEK.equals(period)) {
return TimeDimensionEnum.WEEK.getName();
} else if (DatePeriodEnum.MONTH.equals(period)) {
return TimeDimensionEnum.MONTH.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
}
@Override
public boolean equals(Object o) {
if (this == o) {

70
common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java
View File

@@ -1,73 +1,5 @@
package com.tencent.supersonic.common.pojo.enums;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public enum TimeDimensionEnum {
DAY("sys_imp_date", "数据日期"),
WEEK("sys_imp_week", "数据日期_周"),
MONTH("sys_imp_month", "数据日期_月");
private String name;
private String chName;
TimeDimensionEnum(String name, String chName) {
this.name = name;
this.chName = chName;
}
public static boolean containsTimeDimension(String fieldName) {
if (getNameList().contains(fieldName) || getChNameList().contains(fieldName)) {
return true;
}
return false;
}
public static List<String> getNameList() {
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getName)
.collect(Collectors.toList());
}
public static List<String> getChNameList() {
return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getChName)
.collect(Collectors.toList());
}
public static Map<String, String> getChNameToNameMap() {
return Arrays.stream(TimeDimensionEnum.values()).collect(Collectors
.toMap(TimeDimensionEnum::getChName, TimeDimensionEnum::getName, (k1, k2) -> k1));
}
public static Map<String, String> getNameToNameMap() {
return Arrays.stream(TimeDimensionEnum.values()).collect(Collectors
.toMap(TimeDimensionEnum::getName, TimeDimensionEnum::getName, (k1, k2) -> k1));
}
public String getName() {
return name;
}
public String getChName() {
return chName;
}
/**
* Determine if a time dimension field is included in a Chinese/English text field
*
* @param fields field
* @return true/false
*/
public static boolean containsZhTimeDimension(List<String> fields) {
if (CollectionUtils.isEmpty(fields)) {
return false;
}
return fields.stream().anyMatch(field -> containsTimeDimension(field));
}
DAY, WEEK, MONTH;
}

122
common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java
View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -21,22 +20,14 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.StringJoiner;
import java.util.regex.Pattern;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.*;
@Slf4j
@Component
@Data
public class DateModeUtils {
private final String sysDateCol = TimeDimensionEnum.DAY.getName();
private final String sysDateMonthCol = TimeDimensionEnum.MONTH.getName();
private final String sysDateWeekCol = TimeDimensionEnum.WEEK.getName();
@Value("${s2.query.parameter.sys.zipper.begin:start_}")
private String sysZipperDateColBegin;
@@ -60,8 +51,8 @@ public class DateModeUtils {
public String hasDataModeStr(ItemDateResp dateDate, DateConf dateInfo) {
if (Objects.isNull(dateDate) || StringUtils.isEmpty(dateDate.getStartDate())
|| StringUtils.isEmpty(dateDate.getStartDate())) {
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(),
sysDateCol, dateInfo.getEndDate());
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
dateInfo.getStartDate(), dateInfo.getDateField(), dateInfo.getEndDate());
} else {
log.info("dateDate:{}", dateDate);
}
@@ -79,27 +70,28 @@ public class DateModeUtils {
dateFormatStr, ChronoUnit.DAYS);
LocalDate dateMax = endData;
LocalDate dateMin = dateMax.minusDays(unit - 1);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol,
dateMax);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
dateMin, dateInfo.getDateField(), dateMax);
}
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(),
dateFormatStr, ChronoUnit.MONTHS);
return generateMonthSql(endData, unit, dateFormatStr);
return generateMonthSql(endData, unit, dateFormatStr, dateInfo);
}
}
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(),
sysDateCol, dateInfo.getEndDate());
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
dateInfo.getStartDate(), dateInfo.getDateField(), dateInfo.getEndDate());
}
public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr) {
public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr,
DateConf dateConf) {
LocalDate dateMax = endData;
List<String> months = generateMonthStr(dateMax, unit, dateFormatStr);
if (!CollectionUtils.isEmpty(months)) {
StringJoiner joiner = new StringJoiner(",");
months.stream().forEach(month -> joiner.add("'" + month + "'"));
return String.format("(%s in (%s))", sysDateCol, joiner.toString());
return String.format("(%s in (%s))", dateConf.getDateField(), joiner.toString());
}
return "";
}
@@ -116,8 +108,8 @@ public class DateModeUtils {
public String recentDayStr(ItemDateResp dateDate, DateConf dateInfo) {
ImmutablePair<String, String> dayRange = recentDay(dateDate, dateInfo);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dayRange.left, sysDateCol,
dayRange.right);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dayRange.left,
dateInfo.getDateField(), dayRange.right);
}
public ImmutablePair<String, String> recentDay(ItemDateResp dateDate, DateConf dateInfo) {
@@ -134,24 +126,25 @@ public class DateModeUtils {
return ImmutablePair.of(start, dateDate.getEndDate());
}
public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr) {
public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr,
DateConf dateInfo) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
String endStr = endData.format(formatter);
String start = endData.minusMonths(unit).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol,
endStr);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), start,
dateInfo.getDateField(), endStr);
}
public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) {
List<ImmutablePair<String, String>> range = recentMonth(dateDate, dateInfo);
if (range.size() == 1) {
return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, range.get(0).left,
sysDateMonthCol, range.get(0).right);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
range.get(0).left, dateInfo.getDateField(), range.get(0).right);
}
if (range.size() > 0) {
StringJoiner joiner = new StringJoiner(",");
range.stream().forEach(month -> joiner.add("'" + month.left + "'"));
return String.format("(%s in (%s))", sysDateCol, joiner.toString());
return String.format("(%s in (%s))", dateInfo.getDateField(), joiner.toString());
}
return "";
}
@@ -181,17 +174,17 @@ public class DateModeUtils {
return ret;
}
public String recentWeekStr(LocalDate endData, Long unit) {
public String recentWeekStr(LocalDate endData, Long unit, DateConf dataInfo) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT);
String start = endData.minusDays(unit * 7).format(formatter);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, start, sysDateWeekCol,
endData.format(formatter));
return String.format("(%s >= '%s' and %s <= '%s')", dataInfo.getDateField(), start,
dataInfo.getDateField(), endData.format(formatter));
}
public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) {
ImmutablePair<String, String> dayRange = recentWeek(dateDate, dateInfo);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, dayRange.left,
sysDateWeekCol, dayRange.right);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dayRange.left,
dateInfo.getDateField(), dayRange.right);
}
public ImmutablePair<String, String> recentWeek(ItemDateResp dateDate, DateConf dateInfo) {
@@ -242,26 +235,27 @@ public class DateModeUtils {
* @return
*/
public String betweenDateStr(DateConf dateInfo) {
String dateField = dateInfo.getDateField();
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
// startDate YYYYMM
if (!dateInfo.getStartDate().contains(Constants.MINUS)) {
return String.format("%s >= '%s' and %s <= '%s'", sysDateMonthCol,
dateInfo.getStartDate(), sysDateMonthCol, dateInfo.getEndDate());
return String.format("%s >= '%s' and %s <= '%s'", dateField,
dateInfo.getStartDate(), dateField, dateInfo.getEndDate());
}
LocalDate endData =
LocalDate.parse(dateInfo.getEndDate(), DateTimeFormatter.ofPattern(DAY_FORMAT));
LocalDate startData = LocalDate.parse(dateInfo.getStartDate(),
DateTimeFormatter.ofPattern(DAY_FORMAT));
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT);
return String.format("%s >= '%s' and %s <= '%s'", sysDateMonthCol,
startData.format(formatter), sysDateMonthCol, endData.format(formatter));
return String.format("%s >= '%s' and %s <= '%s'", dateField,
startData.format(formatter), dateField, endData.format(formatter));
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
return String.format("%s >= '%s' and %s <= '%s'", sysDateWeekCol,
dateInfo.getStartDate(), sysDateWeekCol, dateInfo.getEndDate());
return String.format("%s >= '%s' and %s <= '%s'", dateField, dateInfo.getStartDate(),
dateField, dateInfo.getEndDate());
}
return String.format("%s >= '%s' and %s <= '%s'", sysDateCol, dateInfo.getStartDate(),
sysDateCol, dateInfo.getEndDate());
return String.format("%s >= '%s' and %s <= '%s'", dateField, dateInfo.getStartDate(),
dateField, dateInfo.getEndDate());
}
/**
@@ -273,12 +267,12 @@ public class DateModeUtils {
public String listDateStr(DateConf dateInfo) {
StringJoiner joiner = new StringJoiner(COMMA);
dateInfo.getDateList().stream().forEach(date -> joiner.add(APOSTROPHE + date + APOSTROPHE));
String dateCol = sysDateCol;
String dateCol = dateInfo.getDateField();
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
dateCol = sysDateMonthCol;
dateCol = dateInfo.getDateField();
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
dateCol = sysDateWeekCol;
dateCol = dateInfo.getDateField();
}
return String.format("(%s in (%s))", dateCol, joiner.toString());
}
@@ -299,25 +293,26 @@ public class DateModeUtils {
if (DatePeriodEnum.DAY.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
LocalDate dateMin = dateMax.minusDays(unit - 1);
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol,
dateMax);
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dateMin,
dateInfo.getDateField(), dateMax);
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return recentWeekStr(dateMax, unit.longValue());
return recentWeekStr(dateMax, unit.longValue(), dateInfo);
}
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT);
return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT, dateInfo);
}
if (DatePeriodEnum.YEAR.equals(dateInfo.getPeriod())) {
LocalDate dateMax = LocalDate.now().minusDays(1);
return recentMonthStr(dateMax, unit.longValue() * 12, MONTH_FORMAT);
return recentMonthStr(dateMax, unit.longValue() * 12, MONTH_FORMAT, dateInfo);
}
return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol,
LocalDate.now().minusDays(2), sysDateCol, LocalDate.now().minusDays(1));
return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(),
LocalDate.now().minusDays(2), dateInfo.getDateField(),
LocalDate.now().minusDays(1));
}
public String getDateWhereStr(DateConf dateInfo) {
@@ -349,32 +344,7 @@ public class DateModeUtils {
}
public String getSysDateCol(DateConf dateInfo) {
if (DatePeriodEnum.DAY.equals(dateInfo.getPeriod())) {
return sysDateCol;
}
if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) {
return sysDateWeekCol;
}
if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) {
return sysDateMonthCol;
}
return "";
return dateInfo.getDateField();
}
public boolean isDateStr(String date) {
return Pattern.matches("[\\d\\s-:]+", date);
}
public DatePeriodEnum getPeriodByCol(String col) {
if (sysDateCol.equalsIgnoreCase(col)) {
return DatePeriodEnum.DAY;
}
if (sysDateWeekCol.equalsIgnoreCase(col)) {
return DatePeriodEnum.WEEK;
}
if (sysDateMonthCol.equalsIgnoreCase(col)) {
return DatePeriodEnum.MONTH;
}
return null;
}
}

10
common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java
View File

@@ -11,31 +11,31 @@ class SqlDateSelectHelperTest {
String sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1";
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getLowerBound(), ">=");
Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getLowerBound(), ">");
Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getUpperBound(), "<=");
Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 "
+ "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getUpperBound(), "<");
Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17");
sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-10-17' "
+ "AND 数据日期 <= '2023-11-17' GROUP BY 维度1";
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql);
dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期");
Assert.assertEquals(dateBoundInfo.getUpperBound(), "<=");
Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17");
Assert.assertEquals(dateBoundInfo.getLowerBound(), ">=");

17
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java
View File

@@ -32,6 +32,16 @@ public class Dimension {
this.type = type;
this.isCreateDimension = isCreateDimension;
this.bizName = bizName;
this.expr = bizName;
}
public Dimension(String name, String bizName, String expr, DimensionType type,
Integer isCreateDimension) {
this.name = name;
this.type = type;
this.isCreateDimension = isCreateDimension;
this.bizName = bizName;
this.expr = expr;
}
public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension,
@@ -45,12 +55,7 @@ public class Dimension {
this.bizName = bizName;
}
public static Dimension getDefault() {
return new Dimension("数据日期", "imp_date", DimensionType.partition_time, 0, "imp_date",
Constants.DAY_FORMAT, new DimensionTimeTypeParams("false", "day"));
}
public String getFieldName() {
return bizName;
return expr;
}
}

10
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java
View File

@@ -23,11 +23,20 @@ public class Measure {
private String alias;
public Measure(String name, String bizName, String expr, String agg, Integer isCreateMetric) {
this.name = name;
this.agg = agg;
this.isCreateMetric = isCreateMetric;
this.bizName = bizName;
this.expr = expr;
}
public Measure(String name, String bizName, String agg, Integer isCreateMetric) {
this.name = name;
this.agg = agg;
this.isCreateMetric = isCreateMetric;
this.bizName = bizName;
this.expr = bizName;
}
public Measure(String bizName, String constraint) {
@@ -38,4 +47,5 @@ public class Measure {
public String getFieldName() {
return expr;
}
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java
View File

@@ -8,5 +8,5 @@ import java.util.List;
@Data
public class MetricDefineByMeasureParams extends MetricDefineParams {
private List<MeasureParam> measures = Lists.newArrayList();
private List<Measure> measures = Lists.newArrayList();
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java
View File

@@ -18,6 +18,8 @@ public class ModelDetail {
private String queryType;
private String dbType;
private String sqlQuery;
private String tableQuery;

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.api.pojo.enums;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import java.util.List;
@@ -32,7 +32,7 @@ public enum MetricType {
return true;
}
if (MetricDefineType.MEASURE.equals(metricDefineType)) {
List<MeasureParam> measures = typeParams.getMeasures();
List<Measure> measures = typeParams.getMeasures();
if (measures.size() > 1) {
return true;
}

14
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java
View File

@@ -1,14 +1,26 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Sets;
import lombok.Data;
import lombok.ToString;
import java.util.List;
import java.util.Set;
@Data
@ToString(callSuper = true)
public class DimSchemaResp extends DimensionResp {
private Long useCnt = 0L;
private Set<String> fields = Sets.newHashSet();
@Override
public boolean equals(Object o) {
return super.equals(o);
}
@Override
public int hashCode() {
return super.hashCode();
}
}

25
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java
View File

@@ -2,13 +2,9 @@ package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.collections.CollectionUtils;
@@ -69,6 +65,19 @@ public class MetricResp extends SchemaItem {
private boolean containsPartitionDimensions;
public void setMetricDefinition(MetricDefineType type, MetricDefineParams params) {
if (MetricDefineType.MEASURE.equals(type)) {
assert params instanceof MetricDefineByMeasureParams;
metricDefineByMeasureParams = (MetricDefineByMeasureParams) params;
} else if (MetricDefineType.FIELD.equals(type)) {
assert params instanceof MetricDefineByFieldParams;
metricDefineByFieldParams = (MetricDefineByFieldParams) params;
} else if (MetricDefineType.METRIC.equals(type)) {
assert params instanceof MetricDefineByMetricParams;
metricDefineByMetricParams = (MetricDefineByMetricParams) params;
}
}
public void setClassifications(String tag) {
if (StringUtils.isBlank(tag)) {
classifications = Lists.newArrayList();
@@ -105,4 +114,8 @@ public class MetricResp extends SchemaItem {
}
return "";
}
public boolean isDerived() {
return MetricType.isDerived(metricDefineType, metricDefineByMeasureParams);
}
}

15
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java
View File

@@ -1,11 +1,26 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Sets;
import lombok.Data;
import lombok.ToString;
import java.util.Set;
@Data
@ToString(callSuper = true)
public class MetricSchemaResp extends MetricResp {
private Long useCnt = 0L;
private Set<String> fields = Sets.newHashSet();
@Override
public boolean equals(Object o) {
return super.equals(o);
}
@Override
public int hashCode() {
return super.hashCode();
}
}

21
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java
View File

@@ -1,17 +1,9 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Field;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import lombok.*;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
@@ -26,6 +18,7 @@ import java.util.stream.Collectors;
@ToString(callSuper = true)
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class ModelResp extends SchemaItem {
private Long domainId;
@@ -62,6 +55,14 @@ public class ModelResp extends SchemaItem {
return isOpen != null && isOpen == 1;
}
public List<Measure> getMeasures() {
return modelDetail != null ? modelDetail.getMeasures() : Lists.newArrayList();
}
public List<Identify> getIdentifiers() {
return modelDetail != null ? modelDetail.getIdentifiers() : Lists.newArrayList();
}
public List<Dimension> getTimeDimension() {
if (modelDetail == null) {
return Lists.newArrayList();

24
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -13,12 +12,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
/**
@@ -47,18 +41,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
protected Map<String, String> getFieldNameMap(ChatQueryContext chatQueryContext,
Long dataSetId) {
Map<String, String> result = getFieldNameMapFromDB(chatQueryContext, dataSetId);
if (chatQueryContext.containsPartitionDimensions(dataSetId)) {
result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName());
result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName());
result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName());
result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName());
}
return result;
return getFieldNameMapFromDB(chatQueryContext, dataSetId);
}
private static Map<String, String> getFieldNameMapFromDB(ChatQueryContext chatQueryContext,
@@ -126,7 +109,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
}
return elements.stream();
}).collect(Collectors.toSet());
dimensions.add(TimeDimensionEnum.DAY.getChName());
return dimensions;
}
@@ -142,8 +124,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector {
SemanticParseInfo semanticParseInfo) {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL();
Set<String> removeFieldNames = new HashSet<>();
removeFieldNames.addAll(TimeDimensionEnum.getChNameList());
removeFieldNames.addAll(TimeDimensionEnum.getNameList());
Map<String, String> fieldNameMap =
getFieldNameMapFromDB(chatQueryContext, semanticParseInfo.getDataSetId());
removeFieldNames.removeIf(fieldName -> fieldNameMap.containsKey(fieldName));

5
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java
View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.jsqlparser.SqlValidHelper;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
@@ -53,10 +52,6 @@ public class GroupByCorrector extends BaseSemanticCorrector {
if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) {
return false;
}
// if only date in select not add group by.
if (selectFields.size() == 1 && TimeDimensionEnum.containsZhTimeDimension(selectFields)) {
return false;
}
if (SqlSelectHelper.hasGroupBy(correctS2SQL)) {
log.debug("No need to add 'group by', existed 'group by' in s2sql:{}", correctS2SQL);
return false;

16
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java
View File

@@ -1,14 +1,8 @@
package com.tencent.supersonic.headless.chat.corrector;
import com.tencent.supersonic.common.jsqlparser.AggregateEnum;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlAsHelper;
import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.jsqlparser.*;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -21,11 +15,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
/** Perform schema corrections on the Schema information in S2SQL. */
@@ -144,8 +134,6 @@ public class SchemaCorrector extends BaseSemanticCorrector {
Set<String> removeFieldNames = whereExpressionList.stream()
.filter(fieldExpression -> StringUtils.isBlank(fieldExpression.getFunction()))
.filter(fieldExpression -> !TimeDimensionEnum
.containsTimeDimension(fieldExpression.getFieldName()))
.filter(fieldExpression -> FilterOperatorEnum.EQUALS.getValue()
.equals(fieldExpression.getOperator()))
.filter(fieldExpression -> dimensions.contains(fieldExpression.getFieldName()))

7
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java
View File

@@ -5,7 +5,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlDateSelectHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
@@ -44,8 +43,7 @@ public class TimeCorrector extends BaseSemanticCorrector {
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension())
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())
|| TimeDimensionEnum.containsZhTimeDimension(whereFields)) {
|| Objects.isNull(dataSetSchema.getPartitionDimension().getName())) {
return;
}
String partitionDimension = dataSetSchema.getPartitionDimension().getName();
@@ -75,7 +73,8 @@ public class TimeCorrector extends BaseSemanticCorrector {
private void addLowerBoundDate(SemanticParseInfo semanticParseInfo) {
String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL();
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL);
DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL,
semanticParseInfo.getDateInfo().getDateField());
if (dateBoundInfo != null && StringUtils.isBlank(dateBoundInfo.getLowerBound())
&& StringUtils.isNotBlank(dateBoundInfo.getUpperBound())

12
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java
View File

@@ -2,7 +2,10 @@ package com.tencent.supersonic.headless.chat.parser.llm;
import com.tencent.supersonic.common.jsqlparser.SqlValidHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Text2SQLExemplar;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.QueryManager;
@@ -50,6 +53,15 @@ public class LLMResponseService {
parseInfo.setQueryMode(semanticQuery.getQueryMode());
parseInfo.getSqlInfo().setParsedS2SQL(s2SQL);
parseInfo.getSqlInfo().setCorrectedS2SQL(s2SQL);
DataSetSchema dataSetSchema =
queryCtx.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId());
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
if (Objects.nonNull(partitionDimension)) {
DateConf dateConf = new DateConf();
dateConf.setDateField(partitionDimension.getName());
parseInfo.setDateInfo(dateConf);
}
queryCtx.getCandidateQueries().add(semanticQuery);
}

6
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java
View File

@@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.chat.parser.rule;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.parser.SemanticParser;
@@ -57,6 +59,10 @@ public class TimeRangeParser implements SemanticParser {
for (SemanticQuery query : queryContext.getCandidateQueries()) {
SemanticParseInfo parseInfo = query.getParseInfo();
if (queryContext.containsPartitionDimensions(parseInfo.getDataSetId())) {
DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap()
.get(parseInfo.getDataSetId());
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
dateConf.setDateField(partitionDimension.getName());
parseInfo.setDateInfo(dateConf);
}
parseInfo.setScore(parseInfo.getScore() + dateConf.getDetectWord().length());

23
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java
View File

@@ -4,18 +4,8 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.request.*;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.BaseSemanticQuery;
import com.tencent.supersonic.headless.chat.query.QueryManager;
@@ -25,13 +15,8 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.stream.Collectors;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TERM;
@@ -233,8 +218,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
protected void convertBizNameToName(DataSetSchema dataSetSchema,
QueryStructReq queryStructReq) {
Map<String, String> bizNameToName = dataSetSchema.getBizNameToName();
bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap());
List<Order> orders = queryStructReq.getOrders();
if (CollectionUtils.isNotEmpty(orders)) {
for (Order order : orders) {

8
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java
View File

@@ -3,14 +3,13 @@ package com.tencent.supersonic.headless.chat.query.rule.detail;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery;
import lombok.extern.slf4j.Slf4j;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -33,10 +32,13 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery {
chatQueryContext.getSemanticSchema().getDataSetSchemaMap();
DataSetSchema dataSetSchema = dataSetSchemaMap.get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getDetailTypeTimeDefaultConfig();
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
if (Objects.nonNull(partitionDimension) && Objects.nonNull(timeDefaultConfig)
&& Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
DateConf dateInfo = new DateConf();
dateInfo.setDateField(partitionDimension.getName());
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().minusDays(unit).toString();
String endDate = startDate;

13
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java
View File

@@ -3,14 +3,13 @@ package com.tencent.supersonic.headless.chat.query.rule.metric;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery;
import lombok.extern.slf4j.Slf4j;
import java.time.LocalDate;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC;
@@ -40,10 +39,12 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap()
.get(parseInfo.getDataSetId());
TimeDefaultConfig timeDefaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig();
DateConf dateInfo = new DateConf();
// 加上时间!=-1 判断
if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit())
SchemaElement partitionDimension = dataSetSchema.getPartitionDimension();
if (Objects.nonNull(partitionDimension) && Objects.nonNull(timeDefaultConfig)
&& Objects.nonNull(timeDefaultConfig.getUnit())
&& timeDefaultConfig.getUnit() != -1) {
DateConf dateInfo = new DateConf();
dateInfo.setDateField(partitionDimension.getName());
int unit = timeDefaultConfig.getUnit();
String startDate = LocalDate.now().minusDays(unit).toString();
String endDate = startDate;
@@ -55,8 +56,8 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
dateInfo.setPeriod(timeDefaultConfig.getPeriod());
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
// 时间不为-1才设置时间，所以移到这里
parseInfo.setDateInfo(dateInfo);
}
}
}

19
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java
View File

@@ -7,9 +7,7 @@ import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
@@ -22,13 +20,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@@ -179,14 +171,7 @@ public class QueryReqBuilder {
if (Objects.isNull(dateConf)) {
return "";
}
String dateField = TimeDimensionEnum.DAY.getName();
if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) {
dateField = TimeDimensionEnum.MONTH.getName();
}
if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) {
dateField = TimeDimensionEnum.WEEK.getName();
}
return dateField;
return dateConf.getDateField();
}
public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo,

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java
View File

@@ -1,16 +1,13 @@
package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel;
import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric;
import lombok.Data;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
@Data
@@ -28,4 +25,11 @@ public class Ontology {
.collect(Collectors.toList());
}
public EngineType getDatabaseType() {
if (Objects.nonNull(database)) {
return database.getType();
}
return null;
}
}

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -9,6 +9,7 @@ import org.apache.commons.lang3.tuple.Triple;
public class QueryStatement {
private Long dataSetId;
private String dataSetName;
private String sql;
private String errMsg;
private StructQuery structQuery;

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.core.translator;
import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQuery;
import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer;
@@ -26,7 +25,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
return;
}
try {
for (QueryParser parser : ComponentFactory.getQueryParser()) {
for (QueryParser parser : ComponentFactory.getQueryParsers()) {
if (parser.accept(queryStatement)) {
log.debug("QueryConverter accept [{}]", parser.getClass().getName());
parser.parse(queryStatement);
@@ -63,9 +62,6 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
}
private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception {
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
log.info("parse with ontology: [{}]", ontologyQuery);
SqlQuery sqlQuery = queryStatement.getSqlQuery();
String ontologyQuerySql = sqlQuery.getSql();
String ontologyInnerTable = sqlQuery.getTable();
@@ -74,7 +70,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
List<Pair<String, String>> tables = new ArrayList<>();
tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql));
if (sqlQuery.isSupportWith()) {
EngineType engineType = queryStatement.getOntology().getDatabase().getType();
EngineType engineType = queryStatement.getOntology().getDatabaseType();
if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) {
String withSql = "with " + tables.stream()
.map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.translator.parser;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension;
import lombok.extern.slf4j.Slf4j;
@@ -40,9 +39,8 @@ public class DefaultDimValueParser implements QueryParser {
return;
}
String sql = queryStatement.getSqlQuery().getSql();
List<String> whereFields = SqlSelectHelper.getWhereFields(sql).stream()
.filter(field -> !TimeDimensionEnum.containsTimeDimension(field))
.collect(Collectors.toList());
List<String> whereFields =
SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toList());
if (!CollectionUtils.isEmpty(whereFields)) {
return;
}

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java
View File

@@ -6,7 +6,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
@@ -131,8 +130,6 @@ public class SqlQueryParser implements QueryParser {
Map<String, String> dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(),
SchemaItem::getBizName, (k1, k2) -> k1));
dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(),
TimeDimensionEnum.DAY.getName());
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase()))
@@ -315,8 +312,6 @@ public class SqlQueryParser implements QueryParser {
entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName()))
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1));
dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap());
dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap());
dimensionResults.putAll(metricResults);
return dimensionResults;
}

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Dimension.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.core.translator.parser.s2sql;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import lombok.Builder;
import lombok.Data;

11
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/DimensionTimeTypeParams.java
View File

@@ -1,11 +0,0 @@
package com.tencent.supersonic.headless.core.translator.parser.s2sql;
import lombok.Data;
@Data
public class DimensionTimeTypeParams {
private String isPrimary;
private String timeGranularity;
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java
View File

@@ -54,7 +54,7 @@ public class ComponentFactory {
return queryAccelerators;
}
public static List<QueryParser> getQueryParser() {
public static List<QueryParser> getQueryParsers() {
if (queryParsers.isEmpty()) {
initQueryParsers();
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java
View File

@@ -2,8 +2,6 @@ package com.tencent.supersonic.headless.core.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
@@ -53,7 +51,7 @@ public class DataTransformUtils {
private static String getRowKey(Map<String, Object> originalRow, List<String> groups) {
List<Object> values = Lists.newArrayList();
for (String key : originalRow.keySet()) {
if (groups.contains(key) && !TimeDimensionEnum.getNameList().contains(key)) {
if (groups.contains(key)) {
values.add(originalRow.get(key));
}
}
@@ -61,12 +59,6 @@ public class DataTransformUtils {
}
private static String getTimeDimension(DateConf dateConf) {
if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
return dateConf.getDateField();
}
}

21
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -7,7 +7,6 @@ import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.StringUtil;
@@ -29,19 +28,10 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.*;
/** tools functions to analyze queryStructReq */
@Component
@@ -68,12 +58,7 @@ public class SqlGenerateUtils {
if (group.contains(JOIN_UNDERLINE)) {
group = group.split(JOIN_UNDERLINE)[1];
}
if (!TimeDimensionEnum.getNameList().contains(group)) {
locate++;
sb.append(group).append(" as ").append("name").append(locate).append(",");
} else {
sb.append(group).append(",");
}
sb.append(group).append(",");
}
locate = 0;
for (Aggregator agg : queryStructCmd.getAggregators()) {

102
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java
View File

@@ -1,102 +0,0 @@
package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/** generate system time dimension tools */
@Slf4j
public class SysTimeDimensionBuilder {
// Defines the regular expression pattern for the time keyword
private static final Pattern TIME_KEYWORD_PATTERN =
Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b",
Pattern.CASE_INSENSITIVE);
public static void addSysTimeDimension(List<Dimension> dims, DbAdaptor engineAdaptor) {
log.debug("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor);
Dimension timeDim = getTimeDim(dims);
if (timeDim == null) {
timeDim = Dimension.getDefault();
// todo not find the time dimension
return;
}
dims.add(generateSysDayDimension(timeDim, engineAdaptor));
dims.add(generateSysWeekDimension(timeDim, engineAdaptor));
dims.add(generateSysMonthDimension(timeDim, engineAdaptor));
log.debug("addSysTimeDimension after:{}, engineAdaptor:{}", dims, engineAdaptor);
}
private static Dimension generateSysDayDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.DAY.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.DAY.name().toLowerCase());
typeParams.setIsPrimary("true");
dim.setTypeParams(typeParams);
return dim;
}
private static Dimension generateSysWeekDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.WEEK.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.WEEK.name().toLowerCase());
typeParams.setIsPrimary("false");
dim.setTypeParams(typeParams);
return dim;
}
private static Dimension generateSysMonthDimension(Dimension timeDim, DbAdaptor engineAdaptor) {
Dimension dim = new Dimension();
dim.setBizName(TimeDimensionEnum.MONTH.getName());
dim.setType(DimensionType.partition_time);
dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(),
engineAdaptor));
DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams();
typeParams.setTimeGranularity(TimeDimensionEnum.MONTH.name().toLowerCase());
typeParams.setIsPrimary("false");
dim.setTypeParams(typeParams);
return dim;
}
private static boolean containsTimeKeyword(String fieldName) {
Matcher matcher = TIME_KEYWORD_PATTERN.matcher(fieldName);
return matcher.find();
}
// Check whether the time field contains keywords,Generation time expression
private static String generateTimeExpr(Dimension timeDim, String dateType,
DbAdaptor engineAdaptor) {
String bizName = timeDim.getBizName();
String dateFormat = timeDim.getDateFormat();
if (containsTimeKeyword(bizName)) {
String bizNameWithBackticks = String.format("`%s`", bizName);
return engineAdaptor.getDateFormat(dateType, dateFormat, bizNameWithBackticks);
} else {
return engineAdaptor.getDateFormat(dateType, dateFormat, bizName);
}
}
private static Dimension getTimeDim(List<Dimension> timeDims) {
for (Dimension dim : timeDims) {
if (dim.getType().equals(DimensionType.partition_time)) {
return dim;
}
}
return null;
}
}

43
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java
View File

@@ -4,26 +4,12 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.SqlEvaluation;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.ChatWorkflowState;
import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp;
import com.tencent.supersonic.headless.api.pojo.response.MapResp;
import com.tencent.supersonic.headless.api.pojo.response.ParseResp;
import com.tencent.supersonic.headless.api.pojo.response.SearchResult;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector;
import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector;
@@ -40,13 +26,7 @@ import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -223,9 +203,6 @@ public class S2ChatLayerService implements ChatLayerService {
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed())
.limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet());
SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName);
dimensions.add(timeDimensionMatch);
// topN metrics
Set<SchemaElementMatch> metrics = semanticSchema.getMetrics(dataSetId).stream()
.sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()).limit(topN)
@@ -256,20 +233,6 @@ public class S2ChatLayerService implements ChatLayerService {
return termMap;
}
/**
* * get time dimension SchemaElementMatch
*/
private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) {
SchemaElement element = SchemaElement.builder().dataSetId(dataSetId)
.dataSetName(dataSetName).type(SchemaElementType.DIMENSION)
.bizName(TimeDimensionEnum.DAY.getName()).build();
return SchemaElementMatch.builder().element(element)
.detectWord(TimeDimensionEnum.DAY.getChName())
.word(TimeDimensionEnum.DAY.getChName()).similarity(1L)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build();
}
private Function<SchemaElement, SchemaElementMatch> mergeFunction() {
return schemaElement -> SchemaElementMatch.builder().element(schemaElement)
.frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName())

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java
View File

@@ -5,8 +5,8 @@ import com.google.common.collect.Sets;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.*;
import com.tencent.supersonic.headless.api.pojo.response.*;
@@ -206,12 +206,6 @@ public class S2SemanticLayerService implements SemanticLayerService {
ModelResp modelResp = modelResps.get(0);
String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(),
modelResp.getName());
List<Dimension> timeDims = modelResp.getTimeDimension();
if (CollectionUtils.isNotEmpty(timeDims)) {
sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql,
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(),
TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate());
}
if (StringUtils.isNotBlank(queryDimValueReq.getValue())) {
sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%"
+ queryDimValueReq.getValue() + "%'";
@@ -284,6 +278,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
&& StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) {
queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL());
queryStatement.setDataSetId(semanticQueryReq.getDataSetId());
queryStatement.setDataSetName(semanticQueryReq.getDataSetName());
queryStatement.setIsTranslated(true);
}
return queryStatement;
@@ -298,6 +293,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setDataSetId(queryReq.getDataSetId());
queryStatement.setDataSetName(queryReq.getDataSetName());
queryStatement.setSemanticSchema(semanticSchemaResp);
queryStatement.setOntology(semanticSchemaManager.buildOntology(semanticSchemaResp));
return queryStatement;

17
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java
View File

@@ -1,19 +1,10 @@
package com.tencent.supersonic.headless.server.manager;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.FieldParam;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.*;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
@@ -47,7 +38,7 @@ public class MetricYamlManager {
MetricDefineByMeasureParams metricDefineParams =
metric.getMetricDefineByMeasureParams();
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
List<MeasureParam> measures = metricDefineParams.getMeasures();
List<Measure> measures = metricDefineParams.getMeasures();
metricTypeParamsYamlTpl.setMeasures(
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
} else if (MetricDefineType.FIELD.equals(metric.getMetricDefineType())) {
@@ -68,7 +59,7 @@ public class MetricYamlManager {
return metricYamlTpl;
}
public static MeasureYamlTpl convert(MeasureParam measure) {
public static MeasureYamlTpl convert(Measure measure) {
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
measureYamlTpl.setName(measure.getBizName());
measureYamlTpl.setConstraint(measure.getConstraint());

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
View File

@@ -1,17 +1,10 @@
package com.tencent.supersonic.headless.server.manager;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
@@ -30,8 +23,6 @@ public class ModelYamlManager {
public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp,
DatabaseResp databaseResp) {
ModelDetail modelDetail = modelResp.getModelDetail();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
dataModelYamlTpl.setType(databaseResp.getType());
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
@@ -61,8 +52,7 @@ public class ModelYamlManager {
dimensionYamlTpl.setExpr(dim.getBizName());
}
if (dim.getTypeParams() != null) {
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl =
new DimensionTimeTypeParamsTpl();
DimensionTimeTypeParams dimensionTimeTypeParamsTpl = new DimensionTimeTypeParams();
dimensionTimeTypeParamsTpl.setIsPrimary(dim.getTypeParams().getIsPrimary());
dimensionTimeTypeParamsTpl.setTimeGranularity(dim.getTypeParams().getTimeGranularity());
dimensionYamlTpl.setTypeParams(dimensionTimeTypeParamsTpl);

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -187,24 +187,12 @@ public class SemanticSchemaManager {
if (Objects.nonNull(dimensionYamlTpl.getExt())) {
dimension.setExt(dimensionYamlTpl.getExt());
}
dimension.setDimensionTimeTypeParams(
getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams()));
dimension.setDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams());
dimensions.add(dimension);
}
return dimensions;
}
private static DimensionTimeTypeParams getDimensionTimeTypeParams(
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) {
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
if (dimensionTimeTypeParamsTpl != null) {
dimensionTimeTypeParams
.setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity());
dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary());
}
return dimensionTimeTypeParams;
}
private static List<Identify> getIdentify(List<IdentifyYamlTpl> identifyYamlTpls) {
List<Identify> identifies = new ArrayList<>();
for (IdentifyYamlTpl identifyYamlTpl : identifyYamlTpls) {

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java
View File

@@ -1,11 +0,0 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
import lombok.Data;
@Data
public class DimensionTimeTypeParamsTpl {
private String isPrimary;
private String timeGranularity;
}

3
headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.headless.server.pojo.yaml;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import lombok.Data;
import java.util.List;
@@ -19,7 +20,7 @@ public class DimensionYamlTpl {
private String expr;
private DimensionTimeTypeParamsTpl typeParams;
private DimensionTimeTypeParams typeParams;
private DataTypeEnums dataType;

31
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -5,13 +5,7 @@ import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.FileUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.*;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MetaFilter;
@@ -35,21 +29,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.*;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@@ -268,13 +251,7 @@ public class DownloadServiceImpl implements DownloadService {
}
private String getTimeDimension(DateConf dateConf) {
if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.MONTH.getName();
} else if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) {
return TimeDimensionEnum.WEEK.getName();
} else {
return TimeDimensionEnum.DAY.getName();
}
return dateConf.getDateField();
}
private Map<String, List<MetricResp>> getMetricMap(List<MetricResp> metricResps) {

11
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View File

@@ -385,8 +385,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
return true;
}
} else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) {
List<MeasureParam> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
List<String> fieldNameDepended = measures.stream().map(MeasureParam::getBizName)
List<Measure> measures = metricResp.getMetricDefineByMeasureParams().getMeasures();
List<String> fieldNameDepended = measures.stream().map(Measure::getBizName)
// measure bizName = model bizName_fieldName
.map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", ""))
.collect(Collectors.toList());
@@ -679,12 +679,11 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
}
// Measure define will get from first measure
List<Measure> measures = modelResp.getModelDetail().getMeasures();
List<MeasureParam> measureParams =
metricResp.getMetricDefineByMeasureParams().getMeasures();
List<Measure> measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures();
if (CollectionUtils.isEmpty(measureParams)) {
return "";
}
MeasureParam firstMeasure = measureParams.get(0);
Measure firstMeasure = measureParams.get(0);
for (Measure measure : measures) {
if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) {
@@ -727,7 +726,7 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateInfo = queryMetricReq.getDateInfo();
if (Objects.nonNull(dateInfo) && dateInfo.isGroupByDate()) {
queryStructReq.getGroups().add(dateInfo.getGroupByTimeDimension());
queryStructReq.getGroups().add(dateInfo.getDateField());
}
if (!CollectionUtils.isEmpty(dimensionBizNames)) {
queryStructReq.getGroups().addAll(dimensionBizNames);

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.ValueDistribution;
@@ -26,12 +25,7 @@ import org.springframework.util.StringUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
@Service
@Slf4j
@@ -121,8 +115,8 @@ public class TagQueryServiceImpl implements TagQueryService {
User user) {
String sqlPattern = "select max(%s) as %s from tbl where %s is not null";
String sql = String.format(sqlPattern, TimeDimensionEnum.DAY.getName(), maxDateAlias,
tag.getBizName());
String sql = String.format(sqlPattern, itemValueReq.getDateConf().getDateField(),
maxDateAlias, tag.getBizName());
// 添加时间过滤信息
log.info("[queryTagDateFromDbBySql] calculate the maximum time start");
@@ -140,8 +134,8 @@ public class TagQueryServiceImpl implements TagQueryService {
String end = LocalDate.now().minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
sql = sql + String.format(" and ( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
end);
itemValueReq.getDateConf().getDateField(), start,
itemValueReq.getDateConf().getDateField(), end);
}
}
}

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java
View File

@@ -206,12 +206,6 @@ public class DataSetSchemaBuilder {
private static void setDefaultTimeFormat(SchemaElement dimToAdd,
DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
if (null != dimensionTimeTypeParams && TimeDimensionEnum.DAY.name()
.equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT,
DateUtils.DEFAULT_DATE_FORMAT);
} else {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
}
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat);
}
}

57
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
View File

@@ -1,17 +1,7 @@
package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.pojo.*;
import com.tencent.supersonic.common.pojo.enums.*;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.Dimension;
@@ -20,13 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
@@ -45,22 +29,9 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.*;
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
import static com.tencent.supersonic.common.pojo.Constants.POUND;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.*;
@Slf4j
@Component
@@ -456,14 +427,14 @@ public class DictUtils {
return joiner.toString();
}
public String defaultDateFilter() {
public String defaultDateFilter(DateConf dateConf) {
String format = itemValueDateFormat;
String start = LocalDate.now().minusDays(itemValueDateStart)
.format(DateTimeFormatter.ofPattern(format));
String end = LocalDate.now().minusDays(itemValueDateEnd)
.format(DateTimeFormatter.ofPattern(format));
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
start, TimeDimensionEnum.DAY.getName(), end);
return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start,
dateConf.getDateField(), end);
}
private String generateDictDateFilter(DictItemResp dictItemResp) {
@@ -473,7 +444,7 @@ public class DictUtils {
}
// 未进行设置
if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) {
return defaultDateFilter();
return defaultDateFilter(config.getDateConf());
}
// 全表扫描
if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) {
@@ -481,9 +452,9 @@ public class DictUtils {
}
// 静态日期
if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) {
return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(),
config.getDateConf().getStartDate(), TimeDimensionEnum.DAY.getName(),
config.getDateConf().getEndDate());
return String.format("( %s >= '%s' and %s <= '%s' )",
config.getDateConf().getDateField(), config.getDateConf().getStartDate(),
config.getDateConf().getDateField(), config.getDateConf().getEndDate());
}
// 动态日期
if (DateConf.DateMode.RECENT.equals(config.getDateConf().getDateMode())) {
@@ -519,8 +490,8 @@ public class DictUtils {
String end = LocalDate.now().minusDays(0)
.format(DateTimeFormatter.ofPattern(dateFormat));
return String.format("( %s > '%s' and %s <= '%s' )",
TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(),
end);
dictItemResp.getConfig().getDateConf().getDateField(), start,
dictItemResp.getConfig().getDateConf().getDateField(), end);
}
}
return "";

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java
View File

@@ -46,9 +46,9 @@ public class MetricCheckUtils {
throw new InvalidArgumentException("指标定义参数不可为空");
}
expr = typeParams.getExpr();
if (CollectionUtils.isEmpty(typeParams.getFields())) {
throw new InvalidArgumentException("定义指标的字段列表参数不可为空");
}
// if (CollectionUtils.isEmpty(typeParams.getFields())) {
// throw new InvalidArgumentException("定义指标的字段列表参数不可为空");
// }
if (!hasAggregateFunction(expr)) {
throw new InvalidArgumentException("基于字段来创建指标,表达式中必须包含聚合函数");
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java
View File

@@ -2,14 +2,9 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.*;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.MetricService;
import lombok.extern.slf4j.Slf4j;
@@ -61,9 +56,6 @@ public class MetricDrillDownChecker {
}
}
for (String dimensionBizName : groupByFields) {
if (TimeDimensionEnum.containsTimeDimension(dimensionBizName)) {
continue;
}
List<MetricResp> metricResps = getMetrics(metricFields, semanticSchemaResp);
if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) {
DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName);

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java
View File

@@ -7,21 +7,8 @@ import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.util.BeanMapper;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.ModelSchema;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.*;
import com.tencent.supersonic.headless.api.pojo.enums.*;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
@@ -34,12 +21,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
public class ModelConverter {
@@ -141,7 +123,7 @@ public class ModelConverter {
metricReq.setModelId(modelDO.getId());
MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
exprTypeParams.setExpr(measure.getBizName());
MeasureParam measureParam = new MeasureParam();
Measure measureParam = new Measure();
BeanMapper.mapper(measure, measureParam);
exprTypeParams.setMeasures(Lists.newArrayList(measureParam));
metricReq.setMetricDefineByMeasureParams(exprTypeParams);

65
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -2,13 +2,10 @@ package com.tencent.supersonic.headless.server.utils;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.tencent.supersonic.common.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.ItemDateResp;
import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
@@ -31,16 +28,7 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
@@ -274,55 +262,4 @@ public class QueryStructUtils {
return Triple.of("", "", "");
}
public DateConf getDateConfBySql(String sql) {
List<FieldExpression> fieldExpressions = SqlSelectHelper.getFilterExpression(sql);
if (!CollectionUtils.isEmpty(fieldExpressions)) {
Set<String> dateList = new HashSet<>();
String startDate = "";
String endDate = "";
DatePeriodEnum period = null;
for (FieldExpression f : fieldExpressions) {
if (Objects.isNull(f.getFieldName())
|| !internalCols.contains(f.getFieldName().toLowerCase())) {
continue;
}
if (Objects.isNull(f.getFieldValue())
|| !dateModeUtils.isDateStr(f.getFieldValue().toString())) {
continue;
}
period = dateModeUtils.getPeriodByCol(f.getFieldName().toLowerCase());
if (period == null) {
continue;
}
if ("=".equals(f.getOperator())) {
dateList.add(f.getFieldValue().toString());
} else if ("<".equals(f.getOperator()) || "<=".equals(f.getOperator())) {
if (startDate.isEmpty()
|| startDate.compareTo(f.getFieldValue().toString()) > 0) {
startDate = f.getFieldValue().toString();
}
} else if (">".equals(f.getOperator()) || ">=".equals(f.getOperator())) {
if (endDate.isEmpty() || endDate.compareTo(f.getFieldValue().toString()) < 0) {
endDate = f.getFieldValue().toString();
}
}
}
if (period != null) {
DateConf dateConf = new DateConf();
dateConf.setPeriod(period);
if (!CollectionUtils.isEmpty(dateList)) {
dateConf.setDateList(new ArrayList<>(dateList));
dateConf.setDateMode(DateMode.LIST);
return dateConf;
}
if (!"".equals(startDate) && !"".equals(endDate)) {
dateConf.setStartDate(startDate);
dateConf.setEndDate(endDate);
dateConf.setDateMode(DateMode.BETWEEN);
return dateConf;
}
}
}
return null;
}
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.utils;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.enums.SemanticType;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
@@ -55,10 +54,6 @@ public class QueryUtils {
private void populateNamePairs(SemanticSchemaResp semanticSchemaResp,
Map<String, String> namePair, Map<String, String> nameTypePair) {
for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) {
namePair.put(timeDimensionEnum.getName(), "date");
nameTypePair.put(timeDimensionEnum.getName(), "DATE");
}
semanticSchemaResp.getMetrics().forEach(metricDesc -> {
namePair.put(metricDesc.getBizName(), metricDesc.getName());
nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name());

11
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.calcite;
import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.common.pojo.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
@@ -76,7 +77,7 @@ class HeadlessParserServiceTest {
dimension.setName("imp_date");
dimension.setExpr("imp_date");
dimension.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
dimensionTimeTypeParams.setIsPrimary("true");
dimensionTimeTypeParams.setTimeGranularity("day");
dimension.setTypeParams(dimensionTimeTypeParams);
@@ -87,7 +88,7 @@ class HeadlessParserServiceTest {
dimension2.setName("sys_imp_date");
dimension2.setExpr("imp_date");
dimension2.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams2 = new DimensionTimeTypeParamsTpl();
DimensionTimeTypeParams dimensionTimeTypeParams2 = new DimensionTimeTypeParams();
dimensionTimeTypeParams2.setIsPrimary("true");
dimensionTimeTypeParams2.setTimeGranularity("day");
dimension2.setTypeParams(dimensionTimeTypeParams2);
@@ -97,7 +98,7 @@ class HeadlessParserServiceTest {
dimension3.setName("sys_imp_week");
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
dimension3.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
dimensionTimeTypeParams3.setIsPrimary("true");
dimensionTimeTypeParams3.setTimeGranularity("day");
dimension3.setTypeParams(dimensionTimeTypeParams3);
@@ -201,7 +202,7 @@ class HeadlessParserServiceTest {
dimension.setName("sys_imp_date");
dimension.setExpr("imp_date");
dimension.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl();
DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
dimensionTimeTypeParams.setIsPrimary("true");
dimensionTimeTypeParams.setTimeGranularity("day");
dimension.setTypeParams(dimensionTimeTypeParams);
@@ -212,7 +213,7 @@ class HeadlessParserServiceTest {
dimension3.setName("sys_imp_week");
dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))");
dimension3.setType("time");
DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl();
DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams();
dimensionTimeTypeParams3.setIsPrimary("true");
dimensionTimeTypeParams3.setTimeGranularity("week");
dimension3.setTypeParams(dimensionTimeTypeParams3);

160
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java
View File

@@ -1,160 +0,0 @@
package com.tencent.supersonic.headless.server.service;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.DataFormat;
import com.tencent.supersonic.common.pojo.User;
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.headless.api.pojo.DrillDownDimension;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.server.facade.service.ChatLayerService;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository;
import com.tencent.supersonic.headless.server.service.impl.DataSetServiceImpl;
import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl;
import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.context.ApplicationEventPublisher;
import java.util.HashMap;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
public class MetricServiceImplTest {
@Test
void createMetric() throws Exception {
MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
ModelService modelService = Mockito.mock(ModelService.class);
MetricService metricService = mockMetricService(metricRepository, modelService);
MetricReq metricReq = buildMetricReq();
when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
MetricResp actualMetricResp = metricService.createMetric(metricReq, User.getDefaultUser());
MetricResp expectedMetricResp = buildExpectedMetricResp();
Assertions.assertEquals(expectedMetricResp, actualMetricResp);
}
@Test
void updateMetric() throws Exception {
MetricRepository metricRepository = Mockito.mock(MetricRepository.class);
ModelService modelService = Mockito.mock(ModelService.class);
MetricService metricService = mockMetricService(metricRepository, modelService);
MetricReq metricReq = buildMetricUpdateReq();
when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp());
when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList());
MetricDO metricDO = MetricConverter.convert2MetricDO(buildMetricReq());
when(metricRepository.getMetricById(metricDO.getId())).thenReturn(metricDO);
MetricResp actualMetricResp = metricService.updateMetric(metricReq, User.getDefaultUser());
MetricResp expectedMetricResp = buildExpectedMetricResp();
Assertions.assertEquals(expectedMetricResp, actualMetricResp);
}
private MetricService mockMetricService(MetricRepository metricRepository,
ModelService modelService) {
AliasGenerateHelper aliasGenerateHelper = Mockito.mock(AliasGenerateHelper.class);
CollectService collectService = Mockito.mock(CollectService.class);
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
TagMetaService tagMetaService = Mockito.mock(TagMetaService.class);
ChatLayerService chatLayerService = Mockito.mock(ChatLayerService.class);
return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper,
collectService, dataSetService, eventPublisher, dimensionService, tagMetaService,
chatLayerService);
}
private MetricReq buildMetricReq() {
MetricReq metricReq = new MetricReq();
metricReq.setId(1L);
metricReq.setName("hr部门的访问次数");
metricReq.setBizName("pv");
metricReq.setDescription("SuperSonic的访问情况");
metricReq.setAlias("pv");
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
metricReq.setModelId(2L);
metricReq.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
DataFormat dataFormat = new DataFormat();
dataFormat.setDecimalPlaces(3);
dataFormat.setNeedMultiply100(false);
metricReq.setDataFormat(dataFormat);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setMetricDefineByMeasureParams(typeParams);
metricReq.setClassifications(Lists.newArrayList("核心指标"));
metricReq.setRelateDimension(RelateDimension.builder().drillDownDimensions(
Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false)))
.build());
metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
metricReq.setExt(new HashMap<>());
return metricReq;
}
private MetricResp buildExpectedMetricResp() {
MetricResp metricResp = new MetricResp();
metricResp.setId(1L);
metricResp.setName("hr部门的访问次数");
metricResp.setBizName("pv");
metricResp.setDescription("SuperSonic的访问情况");
metricResp.setAlias("pv");
metricResp.setMetricDefineType(MetricDefineType.MEASURE);
metricResp.setModelId(2L);
metricResp.setDataFormatType(DataFormatTypeEnum.PERCENT.getName());
DataFormat dataFormat = new DataFormat();
dataFormat.setDecimalPlaces(3);
dataFormat.setNeedMultiply100(false);
metricResp.setDataFormat(dataFormat);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricResp.setMetricDefineByMeasureParams(typeParams);
metricResp.setClassifications("核心指标");
metricResp.setRelateDimension(RelateDimension.builder().drillDownDimensions(
Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false)))
.build());
metricResp.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode());
metricResp.setExt(new HashMap<>());
metricResp.setTypeEnum(TypeEnums.METRIC);
metricResp.setIsCollect(false);
metricResp.setType(MetricType.DERIVED.name());
metricResp.setStatus(StatusEnum.ONLINE.getCode());
return metricResp;
}
private MetricReq buildMetricUpdateReq() {
MetricReq metricReq = new MetricReq();
metricReq.setId(1L);
metricReq.setName("hr部门的访问次数");
metricReq.setBizName("pv");
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams();
typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"),
new MeasureParam("s2_uv", "department='hr'")));
typeParams.setExpr("s2_pv/s2_uv");
metricReq.setMetricDefineByMeasureParams(typeParams);
return metricReq;
}
private ModelResp mockModelResp() {
ModelResp modelResp = new ModelResp();
modelResp.setId(2L);
modelResp.setDomainId(1L);
return modelResp;
}
}

3
launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java
View File

@@ -19,6 +19,7 @@ public class WebConfig implements WebMvcConfigurer {
public void addViewControllers(ViewControllerRegistry registry) {
registry.addViewController("/").setViewName("redirect:/webapp/");
registry.addViewController("/webapp/").setViewName("forward:/webapp/index.html");
registry.addViewController("/webapp/**/{path:[^\\.]*}").setViewName("forward:/webapp/index.html");
registry.addViewController("/webapp/**/{path:[^\\.]*}")
.setViewName("forward:/webapp/index.html");
}
}

13
launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java
View File

@@ -60,6 +60,7 @@ public class S2VisitsDemo extends S2BaseDemo {
DimensionResp userDimension = getDimension("user_name", userModel);
updateMetric(stayTimeModel, departmentDimension, userDimension);
updateMetric_pv(pvUvModel, departmentDimension, userDimension, metricPv);
addMetric_uv(pvUvModel, departmentDimension);
// create dict conf for dimensions
enableDimensionValue(departmentDimension);
@@ -307,9 +308,9 @@ public class S2VisitsDemo extends S2BaseDemo {
metricReq.setClassifications(Collections.singletonList("核心指标"));
MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams();
metricTypeParams.setExpr("s2_stay_time_statis_stay_hours");
List<MeasureParam> measures = new ArrayList<>();
MeasureParam measure = new MeasureParam("s2_stay_time_statis_stay_hours", "",
AggOperatorEnum.SUM.getOperator());
List<Measure> measures = new ArrayList<>();
Measure measure = new Measure("停留时长", "s2_stay_time_statis_stay_hours",
AggOperatorEnum.SUM.getOperator(), 0);
measures.add(measure);
metricTypeParams.setMeasures(measures);
metricReq.setMetricDefineByMeasureParams(metricTypeParams);
@@ -329,9 +330,9 @@ public class S2VisitsDemo extends S2BaseDemo {
metricReq.setDescription("一段时间内用户的访问次数");
MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams();
metricTypeParams.setExpr("s2_pv_uv_statis_pv");
List<MeasureParam> measures = new ArrayList<>();
MeasureParam measure =
new MeasureParam("s2_pv_uv_statis_pv", "", AggOperatorEnum.SUM.getOperator());
List<Measure> measures = new ArrayList<>();
Measure measure =
new Measure("访问次数", "s2_pv_uv_statis_pv", AggOperatorEnum.SUM.getOperator(), 0);
measures.add(measure);
metricTypeParams.setMeasures(measures);
metricReq.setMetricDefineByMeasureParams(metricTypeParams);

1
launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java
View File

@@ -39,6 +39,7 @@ public class MetricTest extends BaseTest {
}
@Test
@SetSystemProperty(key = "s2.test", value = "true")
public void testMetricModel() throws Exception {
QueryResult actualResult = submitNewChat("超音数 访问次数", agent.getId());

1
launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java
View File

@@ -106,6 +106,7 @@ public class BaseTest extends BaseApplication {
dateConf.setDateMode(DateMode.BETWEEN);
dateConf.setEndDate(now().plusDays(0).toString());
dateConf.setStartDate(now().plusDays(-365).toString());
dateConf.setDateField("imp_date");
queryStructReq.setDateInfo(dateConf);
List<Order> orders = new ArrayList<>();

1
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -79,6 +79,7 @@ public class DataUtils {
dateInfo.setPeriod(period);
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
dateInfo.setDateField("imp_date");
return dateInfo;
}