diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoFormatProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoFormatProcessor.java index 71c021e2a..dbc013ebd 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoFormatProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoFormatProcessor.java @@ -8,7 +8,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; @@ -21,12 +20,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; /** @@ -222,9 +216,6 @@ public class ParseInfoFormatProcessor implements ParseResultProcessor { } private static boolean isPartitionDimension(DataSetSchema dataSetSchema, String sqlFieldName) { - if (TimeDimensionEnum.containsTimeDimension(sqlFieldName)) { - return true; - } if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension()) || Objects.isNull(dataSetSchema.getPartitionDimension().getName())) { return false; diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java index b7918c5e7..c90f89695 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlAddHelper.java @@ -1,35 +1,17 @@ package com.tencent.supersonic.common.jsqlparser; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.Function; -import net.sf.jsqlparser.expression.LongValue; -import net.sf.jsqlparser.expression.Parenthesis; -import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.*; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; import net.sf.jsqlparser.expression.operators.conditional.OrExpression; import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; import net.sf.jsqlparser.expression.operators.relational.EqualsTo; import net.sf.jsqlparser.schema.Column; -import net.sf.jsqlparser.schema.Table; -import net.sf.jsqlparser.statement.select.GroupByElement; -import net.sf.jsqlparser.statement.select.OrderByElement; -import net.sf.jsqlparser.statement.select.ParenthesedSelect; -import net.sf.jsqlparser.statement.select.PlainSelect; -import net.sf.jsqlparser.statement.select.Select; -import net.sf.jsqlparser.statement.select.SelectItem; -import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; -import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.select.*; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; /** Sql Parser add Helper */ @Slf4j @@ -144,42 +126,7 @@ public class SqlAddHelper { return sql; } PlainSelect plainSelect = (PlainSelect) selectStatement; - List chNameList = TimeDimensionEnum.getChNameList(); - Boolean dateWhere = false; - for (String chName : chNameList) { - if (expression.toString().contains(chName)) { - dateWhere = true; - } - } - List plainSelectList = SqlSelectHelper.getWithItem(selectStatement); - if (!CollectionUtils.isEmpty(plainSelectList) && dateWhere) { - List withNameList = SqlSelectHelper.getWithName(sql); - for (int i = 0; i < plainSelectList.size(); i++) { - if (plainSelectList.get(i).getFromItem() instanceof Table) { - Table table = (Table) plainSelectList.get(i).getFromItem(); - if (withNameList.contains(table.getName())) { - continue; - } - } - Set result = new HashSet<>(); - List subPlainSelectList = new ArrayList<>(); - subPlainSelectList.add(plainSelectList.get(i)); - SqlSelectHelper.getWhereFields(subPlainSelectList, result); - if (TimeDimensionEnum.containsZhTimeDimension(new ArrayList<>(result))) { - continue; - } - Expression subWhere = plainSelectList.get(i).getWhere(); - addWhere(plainSelectList.get(i), subWhere, expression); - } - return selectStatement.toString(); - } - if (plainSelect.getFromItem() instanceof ParenthesedSelect && dateWhere) { - ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); - PlainSelect subPlainSelect = parenthesedSelect.getPlainSelect(); - Expression subWhere = subPlainSelect.getWhere(); - addWhere(subPlainSelect, subWhere, expression); - return selectStatement.toString(); - } + Expression where = plainSelect.getWhere(); addWhere(plainSelect, where, expression); diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java index 1cf70c5b0..3ea330318 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelper.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.common.jsqlparser; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.statement.select.PlainSelect; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -12,7 +12,7 @@ import java.util.Objects; @Slf4j public class SqlDateSelectHelper { - public static DateVisitor.DateBoundInfo getDateBoundInfo(String sql) { + public static DateVisitor.DateBoundInfo getDateBoundInfo(String sql, String dateField) { List plainSelectList = SqlSelectHelper.getPlainSelect(sql); if (plainSelectList.size() != 1) { return null; @@ -25,7 +25,7 @@ public class SqlDateSelectHelper { if (Objects.isNull(where)) { return null; } - DateVisitor dateVisitor = new DateVisitor(TimeDimensionEnum.getChNameList()); + DateVisitor dateVisitor = new DateVisitor(Collections.singletonList(dateField)); where.accept(dateVisitor); return dateVisitor.getDateBoundInfo(); } diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java index f162395bf..790f2ec72 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/DateConf.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.common.pojo; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; import lombok.Data; import org.springframework.util.CollectionUtils; @@ -40,6 +39,8 @@ public class DateConf { private boolean groupByDate; + private String dateField; + public List getDateList() { if (!CollectionUtils.isEmpty(dateList)) { return dateList; @@ -49,18 +50,6 @@ public class DateConf { return DateUtils.getDateList(startDateStr, endDateStr, getPeriod()); } - public String getGroupByTimeDimension() { - if (DatePeriodEnum.DAY.equals(period)) { - return TimeDimensionEnum.DAY.getName(); - } else if (DatePeriodEnum.WEEK.equals(period)) { - return TimeDimensionEnum.WEEK.getName(); - } else if (DatePeriodEnum.MONTH.equals(period)) { - return TimeDimensionEnum.MONTH.getName(); - } else { - return TimeDimensionEnum.DAY.getName(); - } - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java index b12e2c9c1..b21071e59 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeDimensionEnum.java @@ -1,73 +1,5 @@ package com.tencent.supersonic.common.pojo.enums; -import org.springframework.util.CollectionUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - public enum TimeDimensionEnum { - DAY("sys_imp_date", "数据日期"), - - WEEK("sys_imp_week", "数据日期_周"), - - MONTH("sys_imp_month", "数据日期_月"); - - private String name; - - private String chName; - - TimeDimensionEnum(String name, String chName) { - this.name = name; - this.chName = chName; - } - - public static boolean containsTimeDimension(String fieldName) { - if (getNameList().contains(fieldName) || getChNameList().contains(fieldName)) { - return true; - } - return false; - } - - public static List getNameList() { - return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getName) - .collect(Collectors.toList()); - } - - public static List getChNameList() { - return Arrays.stream(TimeDimensionEnum.values()).map(TimeDimensionEnum::getChName) - .collect(Collectors.toList()); - } - - public static Map getChNameToNameMap() { - return Arrays.stream(TimeDimensionEnum.values()).collect(Collectors - .toMap(TimeDimensionEnum::getChName, TimeDimensionEnum::getName, (k1, k2) -> k1)); - } - - public static Map getNameToNameMap() { - return Arrays.stream(TimeDimensionEnum.values()).collect(Collectors - .toMap(TimeDimensionEnum::getName, TimeDimensionEnum::getName, (k1, k2) -> k1)); - } - - public String getName() { - return name; - } - - public String getChName() { - return chName; - } - - /** - * Determine if a time dimension field is included in a Chinese/English text field - * - * @param fields field - * @return true/false - */ - public static boolean containsZhTimeDimension(List fields) { - if (CollectionUtils.isEmpty(fields)) { - return false; - } - return fields.stream().anyMatch(field -> containsTimeDimension(field)); - } + DAY, WEEK, MONTH; } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java index 75ee5a597..1cd84cfee 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java @@ -4,7 +4,6 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -21,22 +20,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.StringJoiner; -import java.util.regex.Pattern; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.*; @Slf4j @Component @Data public class DateModeUtils { - private final String sysDateCol = TimeDimensionEnum.DAY.getName(); - private final String sysDateMonthCol = TimeDimensionEnum.MONTH.getName(); - private final String sysDateWeekCol = TimeDimensionEnum.WEEK.getName(); - @Value("${s2.query.parameter.sys.zipper.begin:start_}") private String sysZipperDateColBegin; @@ -60,8 +51,8 @@ public class DateModeUtils { public String hasDataModeStr(ItemDateResp dateDate, DateConf dateInfo) { if (Objects.isNull(dateDate) || StringUtils.isEmpty(dateDate.getStartDate()) || StringUtils.isEmpty(dateDate.getStartDate())) { - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(), - sysDateCol, dateInfo.getEndDate()); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), + dateInfo.getStartDate(), dateInfo.getDateField(), dateInfo.getEndDate()); } else { log.info("dateDate:{}", dateDate); } @@ -79,27 +70,28 @@ public class DateModeUtils { dateFormatStr, ChronoUnit.DAYS); LocalDate dateMax = endData; LocalDate dateMin = dateMax.minusDays(unit - 1); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, - dateMax); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), + dateMin, dateInfo.getDateField(), dateMax); } if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) { Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateFormatStr, ChronoUnit.MONTHS); - return generateMonthSql(endData, unit, dateFormatStr); + return generateMonthSql(endData, unit, dateFormatStr, dateInfo); } } - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(), - sysDateCol, dateInfo.getEndDate()); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), + dateInfo.getStartDate(), dateInfo.getDateField(), dateInfo.getEndDate()); } - public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr) { + public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr, + DateConf dateConf) { LocalDate dateMax = endData; List months = generateMonthStr(dateMax, unit, dateFormatStr); if (!CollectionUtils.isEmpty(months)) { StringJoiner joiner = new StringJoiner(","); months.stream().forEach(month -> joiner.add("'" + month + "'")); - return String.format("(%s in (%s))", sysDateCol, joiner.toString()); + return String.format("(%s in (%s))", dateConf.getDateField(), joiner.toString()); } return ""; } @@ -116,8 +108,8 @@ public class DateModeUtils { public String recentDayStr(ItemDateResp dateDate, DateConf dateInfo) { ImmutablePair dayRange = recentDay(dateDate, dateInfo); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dayRange.left, sysDateCol, - dayRange.right); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dayRange.left, + dateInfo.getDateField(), dayRange.right); } public ImmutablePair recentDay(ItemDateResp dateDate, DateConf dateInfo) { @@ -134,24 +126,25 @@ public class DateModeUtils { return ImmutablePair.of(start, dateDate.getEndDate()); } - public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr) { + public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr, + DateConf dateInfo) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr); String endStr = endData.format(formatter); String start = endData.minusMonths(unit).format(formatter); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol, - endStr); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), start, + dateInfo.getDateField(), endStr); } public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) { List> range = recentMonth(dateDate, dateInfo); if (range.size() == 1) { - return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, range.get(0).left, - sysDateMonthCol, range.get(0).right); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), + range.get(0).left, dateInfo.getDateField(), range.get(0).right); } if (range.size() > 0) { StringJoiner joiner = new StringJoiner(","); range.stream().forEach(month -> joiner.add("'" + month.left + "'")); - return String.format("(%s in (%s))", sysDateCol, joiner.toString()); + return String.format("(%s in (%s))", dateInfo.getDateField(), joiner.toString()); } return ""; } @@ -181,17 +174,17 @@ public class DateModeUtils { return ret; } - public String recentWeekStr(LocalDate endData, Long unit) { + public String recentWeekStr(LocalDate endData, Long unit, DateConf dataInfo) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT); String start = endData.minusDays(unit * 7).format(formatter); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, start, sysDateWeekCol, - endData.format(formatter)); + return String.format("(%s >= '%s' and %s <= '%s')", dataInfo.getDateField(), start, + dataInfo.getDateField(), endData.format(formatter)); } public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) { ImmutablePair dayRange = recentWeek(dateDate, dateInfo); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, dayRange.left, - sysDateWeekCol, dayRange.right); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dayRange.left, + dateInfo.getDateField(), dayRange.right); } public ImmutablePair recentWeek(ItemDateResp dateDate, DateConf dateInfo) { @@ -242,26 +235,27 @@ public class DateModeUtils { * @return */ public String betweenDateStr(DateConf dateInfo) { + String dateField = dateInfo.getDateField(); if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) { // startDate YYYYMM if (!dateInfo.getStartDate().contains(Constants.MINUS)) { - return String.format("%s >= '%s' and %s <= '%s'", sysDateMonthCol, - dateInfo.getStartDate(), sysDateMonthCol, dateInfo.getEndDate()); + return String.format("%s >= '%s' and %s <= '%s'", dateField, + dateInfo.getStartDate(), dateField, dateInfo.getEndDate()); } LocalDate endData = LocalDate.parse(dateInfo.getEndDate(), DateTimeFormatter.ofPattern(DAY_FORMAT)); LocalDate startData = LocalDate.parse(dateInfo.getStartDate(), DateTimeFormatter.ofPattern(DAY_FORMAT)); DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT); - return String.format("%s >= '%s' and %s <= '%s'", sysDateMonthCol, - startData.format(formatter), sysDateMonthCol, endData.format(formatter)); + return String.format("%s >= '%s' and %s <= '%s'", dateField, + startData.format(formatter), dateField, endData.format(formatter)); } if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) { - return String.format("%s >= '%s' and %s <= '%s'", sysDateWeekCol, - dateInfo.getStartDate(), sysDateWeekCol, dateInfo.getEndDate()); + return String.format("%s >= '%s' and %s <= '%s'", dateField, dateInfo.getStartDate(), + dateField, dateInfo.getEndDate()); } - return String.format("%s >= '%s' and %s <= '%s'", sysDateCol, dateInfo.getStartDate(), - sysDateCol, dateInfo.getEndDate()); + return String.format("%s >= '%s' and %s <= '%s'", dateField, dateInfo.getStartDate(), + dateField, dateInfo.getEndDate()); } /** @@ -273,12 +267,12 @@ public class DateModeUtils { public String listDateStr(DateConf dateInfo) { StringJoiner joiner = new StringJoiner(COMMA); dateInfo.getDateList().stream().forEach(date -> joiner.add(APOSTROPHE + date + APOSTROPHE)); - String dateCol = sysDateCol; + String dateCol = dateInfo.getDateField(); if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) { - dateCol = sysDateMonthCol; + dateCol = dateInfo.getDateField(); } if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) { - dateCol = sysDateWeekCol; + dateCol = dateInfo.getDateField(); } return String.format("(%s in (%s))", dateCol, joiner.toString()); } @@ -299,25 +293,26 @@ public class DateModeUtils { if (DatePeriodEnum.DAY.equals(dateInfo.getPeriod())) { LocalDate dateMax = LocalDate.now().minusDays(1); LocalDate dateMin = dateMax.minusDays(unit - 1); - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, - dateMax); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), dateMin, + dateInfo.getDateField(), dateMax); } if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) { LocalDate dateMax = LocalDate.now().minusDays(1); - return recentWeekStr(dateMax, unit.longValue()); + return recentWeekStr(dateMax, unit.longValue(), dateInfo); } if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) { LocalDate dateMax = LocalDate.now().minusDays(1); - return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT); + return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT, dateInfo); } if (DatePeriodEnum.YEAR.equals(dateInfo.getPeriod())) { LocalDate dateMax = LocalDate.now().minusDays(1); - return recentMonthStr(dateMax, unit.longValue() * 12, MONTH_FORMAT); + return recentMonthStr(dateMax, unit.longValue() * 12, MONTH_FORMAT, dateInfo); } - return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, - LocalDate.now().minusDays(2), sysDateCol, LocalDate.now().minusDays(1)); + return String.format("(%s >= '%s' and %s <= '%s')", dateInfo.getDateField(), + LocalDate.now().minusDays(2), dateInfo.getDateField(), + LocalDate.now().minusDays(1)); } public String getDateWhereStr(DateConf dateInfo) { @@ -349,32 +344,7 @@ public class DateModeUtils { } public String getSysDateCol(DateConf dateInfo) { - if (DatePeriodEnum.DAY.equals(dateInfo.getPeriod())) { - return sysDateCol; - } - if (DatePeriodEnum.WEEK.equals(dateInfo.getPeriod())) { - return sysDateWeekCol; - } - if (DatePeriodEnum.MONTH.equals(dateInfo.getPeriod())) { - return sysDateMonthCol; - } - return ""; + return dateInfo.getDateField(); } - public boolean isDateStr(String date) { - return Pattern.matches("[\\d\\s-:]+", date); - } - - public DatePeriodEnum getPeriodByCol(String col) { - if (sysDateCol.equalsIgnoreCase(col)) { - return DatePeriodEnum.DAY; - } - if (sysDateWeekCol.equalsIgnoreCase(col)) { - return DatePeriodEnum.WEEK; - } - if (sysDateMonthCol.equalsIgnoreCase(col)) { - return DatePeriodEnum.MONTH; - } - return null; - } } diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java index 046609b07..76fba2a9d 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlDateSelectHelperTest.java @@ -11,31 +11,31 @@ class SqlDateSelectHelperTest { String sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-11-17' GROUP BY 维度1"; - DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期"); Assert.assertEquals(dateBoundInfo.getLowerBound(), ">="); Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17"); sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 > '2023-11-17' GROUP BY 维度1"; - dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期"); Assert.assertEquals(dateBoundInfo.getLowerBound(), ">"); Assert.assertEquals(dateBoundInfo.getLowerDate(), "2023-11-17"); sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; - dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期"); Assert.assertEquals(dateBoundInfo.getUpperBound(), "<="); Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 < '2023-11-17' GROUP BY 维度1"; - dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期"); Assert.assertEquals(dateBoundInfo.getUpperBound(), "<"); Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); sql = "SELECT 维度1,sum(播放量) FROM 数据库 " + "WHERE (歌手名 = '张三') AND 数据日期 >= '2023-10-17' " + "AND 数据日期 <= '2023-11-17' GROUP BY 维度1"; - dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql); + dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(sql, "数据日期"); Assert.assertEquals(dateBoundInfo.getUpperBound(), "<="); Assert.assertEquals(dateBoundInfo.getUpperDate(), "2023-11-17"); Assert.assertEquals(dateBoundInfo.getLowerBound(), ">="); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java index 6ed5b0450..1e0539d98 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java @@ -32,6 +32,16 @@ public class Dimension { this.type = type; this.isCreateDimension = isCreateDimension; this.bizName = bizName; + this.expr = bizName; + } + + public Dimension(String name, String bizName, String expr, DimensionType type, + Integer isCreateDimension) { + this.name = name; + this.type = type; + this.isCreateDimension = isCreateDimension; + this.bizName = bizName; + this.expr = expr; } public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension, @@ -45,12 +55,7 @@ public class Dimension { this.bizName = bizName; } - public static Dimension getDefault() { - return new Dimension("数据日期", "imp_date", DimensionType.partition_time, 0, "imp_date", - Constants.DAY_FORMAT, new DimensionTimeTypeParams("false", "day")); - } - public String getFieldName() { - return bizName; + return expr; } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java index b39521be3..ad18851b8 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java @@ -23,11 +23,20 @@ public class Measure { private String alias; + public Measure(String name, String bizName, String expr, String agg, Integer isCreateMetric) { + this.name = name; + this.agg = agg; + this.isCreateMetric = isCreateMetric; + this.bizName = bizName; + this.expr = expr; + } + public Measure(String name, String bizName, String agg, Integer isCreateMetric) { this.name = name; this.agg = agg; this.isCreateMetric = isCreateMetric; this.bizName = bizName; + this.expr = bizName; } public Measure(String bizName, String constraint) { @@ -38,4 +47,5 @@ public class Measure { public String getFieldName() { return expr; } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java index c186d8b67..78793246c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricDefineByMeasureParams.java @@ -8,5 +8,5 @@ import java.util.List; @Data public class MetricDefineByMeasureParams extends MetricDefineParams { - private List measures = Lists.newArrayList(); + private List measures = Lists.newArrayList(); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index 8c9fc1753..26a334e27 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -18,6 +18,8 @@ public class ModelDetail { private String queryType; + private String dbType; + private String sqlQuery; private String tableQuery; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java index c5cc84035..93f537e0f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/MetricType.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.enums; -import com.tencent.supersonic.headless.api.pojo.MeasureParam; +import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import java.util.List; @@ -32,7 +32,7 @@ public enum MetricType { return true; } if (MetricDefineType.MEASURE.equals(metricDefineType)) { - List measures = typeParams.getMeasures(); + List measures = typeParams.getMeasures(); if (measures.size() > 1) { return true; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java index aca3deeed..a3732b20b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java @@ -1,14 +1,26 @@ package com.tencent.supersonic.headless.api.pojo.response; +import com.google.common.collect.Sets; import lombok.Data; import lombok.ToString; -import java.util.List; +import java.util.Set; @Data @ToString(callSuper = true) public class DimSchemaResp extends DimensionResp { private Long useCnt = 0L; + private Set fields = Sets.newHashSet(); + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java index 5f1397ee8..dfde7fec1 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricResp.java @@ -2,13 +2,9 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DataFormat; -import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; -import com.tencent.supersonic.headless.api.pojo.RelateDimension; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; +import com.tencent.supersonic.headless.api.pojo.enums.MetricType; import lombok.Data; import lombok.ToString; import org.apache.commons.collections.CollectionUtils; @@ -69,6 +65,19 @@ public class MetricResp extends SchemaItem { private boolean containsPartitionDimensions; + public void setMetricDefinition(MetricDefineType type, MetricDefineParams params) { + if (MetricDefineType.MEASURE.equals(type)) { + assert params instanceof MetricDefineByMeasureParams; + metricDefineByMeasureParams = (MetricDefineByMeasureParams) params; + } else if (MetricDefineType.FIELD.equals(type)) { + assert params instanceof MetricDefineByFieldParams; + metricDefineByFieldParams = (MetricDefineByFieldParams) params; + } else if (MetricDefineType.METRIC.equals(type)) { + assert params instanceof MetricDefineByMetricParams; + metricDefineByMetricParams = (MetricDefineByMetricParams) params; + } + } + public void setClassifications(String tag) { if (StringUtils.isBlank(tag)) { classifications = Lists.newArrayList(); @@ -105,4 +114,8 @@ public class MetricResp extends SchemaItem { } return ""; } + + public boolean isDerived() { + return MetricType.isDerived(metricDefineType, metricDefineByMeasureParams); + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java index a6e334e05..6c7bd78d9 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/MetricSchemaResp.java @@ -1,11 +1,26 @@ package com.tencent.supersonic.headless.api.pojo.response; +import com.google.common.collect.Sets; import lombok.Data; import lombok.ToString; +import java.util.Set; + @Data @ToString(callSuper = true) public class MetricSchemaResp extends MetricResp { private Long useCnt = 0L; + private Set fields = Sets.newHashSet(); + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java index afead8d78..76304ecb0 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java @@ -1,17 +1,9 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.Dimension; -import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.Field; -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; +import lombok.*; import org.springframework.util.CollectionUtils; import java.util.ArrayList; @@ -26,6 +18,7 @@ import java.util.stream.Collectors; @ToString(callSuper = true) @AllArgsConstructor @NoArgsConstructor +@Builder public class ModelResp extends SchemaItem { private Long domainId; @@ -62,6 +55,14 @@ public class ModelResp extends SchemaItem { return isOpen != null && isOpen == 1; } + public List getMeasures() { + return modelDetail != null ? modelDetail.getMeasures() : Lists.newArrayList(); + } + + public List getIdentifiers() { + return modelDetail != null ? modelDetail.getIdentifiers() : Lists.newArrayList(); + } + public List getTimeDimension() { if (modelDetail == null) { return Lists.newArrayList(); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java index e66afe3d6..5a0960adb 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/BaseSemanticCorrector.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.chat.corrector; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; @@ -13,12 +12,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; /** @@ -47,18 +41,7 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { protected Map getFieldNameMap(ChatQueryContext chatQueryContext, Long dataSetId) { - - Map result = getFieldNameMapFromDB(chatQueryContext, dataSetId); - if (chatQueryContext.containsPartitionDimensions(dataSetId)) { - result.put(TimeDimensionEnum.DAY.getChName(), TimeDimensionEnum.DAY.getChName()); - result.put(TimeDimensionEnum.MONTH.getChName(), TimeDimensionEnum.MONTH.getChName()); - result.put(TimeDimensionEnum.WEEK.getChName(), TimeDimensionEnum.WEEK.getChName()); - - result.put(TimeDimensionEnum.DAY.getName(), TimeDimensionEnum.DAY.getChName()); - result.put(TimeDimensionEnum.MONTH.getName(), TimeDimensionEnum.MONTH.getChName()); - result.put(TimeDimensionEnum.WEEK.getName(), TimeDimensionEnum.WEEK.getChName()); - } - return result; + return getFieldNameMapFromDB(chatQueryContext, dataSetId); } private static Map getFieldNameMapFromDB(ChatQueryContext chatQueryContext, @@ -126,7 +109,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { } return elements.stream(); }).collect(Collectors.toSet()); - dimensions.add(TimeDimensionEnum.DAY.getChName()); return dimensions; } @@ -142,8 +124,6 @@ public abstract class BaseSemanticCorrector implements SemanticCorrector { SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); Set removeFieldNames = new HashSet<>(); - removeFieldNames.addAll(TimeDimensionEnum.getChNameList()); - removeFieldNames.addAll(TimeDimensionEnum.getNameList()); Map fieldNameMap = getFieldNameMapFromDB(chatQueryContext, semanticParseInfo.getDataSetId()); removeFieldNames.removeIf(fieldName -> fieldNameMap.containsKey(fieldName)); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java index eb7a4da70..1ad5886c0 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java @@ -4,7 +4,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.jsqlparser.SqlValidHelper; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; @@ -53,10 +52,6 @@ public class GroupByCorrector extends BaseSemanticCorrector { if (CollectionUtils.isEmpty(selectFields) || CollectionUtils.isEmpty(dimensions)) { return false; } - // if only date in select not add group by. - if (selectFields.size() == 1 && TimeDimensionEnum.containsZhTimeDimension(selectFields)) { - return false; - } if (SqlSelectHelper.hasGroupBy(correctS2SQL)) { log.debug("No need to add 'group by', existed 'group by' in s2sql:{}", correctS2SQL); return false; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java index 918713e5d..180f796ad 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/SchemaCorrector.java @@ -1,14 +1,8 @@ package com.tencent.supersonic.headless.chat.corrector; -import com.tencent.supersonic.common.jsqlparser.AggregateEnum; -import com.tencent.supersonic.common.jsqlparser.FieldExpression; -import com.tencent.supersonic.common.jsqlparser.SqlAsHelper; -import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; -import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.common.jsqlparser.*; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; @@ -21,11 +15,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; /** Perform schema corrections on the Schema information in S2SQL. */ @@ -144,8 +134,6 @@ public class SchemaCorrector extends BaseSemanticCorrector { Set removeFieldNames = whereExpressionList.stream() .filter(fieldExpression -> StringUtils.isBlank(fieldExpression.getFunction())) - .filter(fieldExpression -> !TimeDimensionEnum - .containsTimeDimension(fieldExpression.getFieldName())) .filter(fieldExpression -> FilterOperatorEnum.EQUALS.getValue() .equals(fieldExpression.getOperator())) .filter(fieldExpression -> dimensions.contains(fieldExpression.getFieldName())) diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java index 68ff19673..1d6567093 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java @@ -5,7 +5,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlDateSelectHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; import com.tencent.supersonic.headless.api.pojo.QueryConfig; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; @@ -44,8 +43,7 @@ public class TimeCorrector extends BaseSemanticCorrector { DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId); if (Objects.isNull(dataSetSchema) || Objects.isNull(dataSetSchema.getPartitionDimension()) - || Objects.isNull(dataSetSchema.getPartitionDimension().getName()) - || TimeDimensionEnum.containsZhTimeDimension(whereFields)) { + || Objects.isNull(dataSetSchema.getPartitionDimension().getName())) { return; } String partitionDimension = dataSetSchema.getPartitionDimension().getName(); @@ -75,7 +73,8 @@ public class TimeCorrector extends BaseSemanticCorrector { private void addLowerBoundDate(SemanticParseInfo semanticParseInfo) { String correctS2SQL = semanticParseInfo.getSqlInfo().getCorrectedS2SQL(); - DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL); + DateBoundInfo dateBoundInfo = SqlDateSelectHelper.getDateBoundInfo(correctS2SQL, + semanticParseInfo.getDateInfo().getDateField()); if (dateBoundInfo != null && StringUtils.isBlank(dateBoundInfo.getLowerBound()) && StringUtils.isNotBlank(dateBoundInfo.getUpperBound()) diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java index 59b1d8560..c669e7139 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMResponseService.java @@ -2,7 +2,10 @@ package com.tencent.supersonic.headless.chat.parser.llm; import com.tencent.supersonic.common.jsqlparser.SqlValidHelper; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Text2SQLExemplar; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.QueryManager; @@ -50,6 +53,15 @@ public class LLMResponseService { parseInfo.setQueryMode(semanticQuery.getQueryMode()); parseInfo.getSqlInfo().setParsedS2SQL(s2SQL); parseInfo.getSqlInfo().setCorrectedS2SQL(s2SQL); + + DataSetSchema dataSetSchema = + queryCtx.getSemanticSchema().getDataSetSchemaMap().get(parseInfo.getDataSetId()); + SchemaElement partitionDimension = dataSetSchema.getPartitionDimension(); + if (Objects.nonNull(partitionDimension)) { + DateConf dateConf = new DateConf(); + dateConf.setDateField(partitionDimension.getName()); + parseInfo.setDateInfo(dateConf); + } queryCtx.getCandidateQueries().add(semanticQuery); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java index 0dd6d59c1..f2e70e901 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/rule/TimeRangeParser.java @@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.chat.parser.rule; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.parser.SemanticParser; @@ -57,6 +59,10 @@ public class TimeRangeParser implements SemanticParser { for (SemanticQuery query : queryContext.getCandidateQueries()) { SemanticParseInfo parseInfo = query.getParseInfo(); if (queryContext.containsPartitionDimensions(parseInfo.getDataSetId())) { + DataSetSchema dataSetSchema = queryContext.getSemanticSchema().getDataSetSchemaMap() + .get(parseInfo.getDataSetId()); + SchemaElement partitionDimension = dataSetSchema.getPartitionDimension(); + dateConf.setDateField(partitionDimension.getName()); parseInfo.setDateInfo(dateConf); } parseInfo.setScore(parseInfo.getScore() + dateConf.getDetectWord().length()); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java index a2c0895ca..f7c96745f 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java @@ -4,18 +4,8 @@ import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticSchema; -import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; -import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; +import com.tencent.supersonic.headless.api.pojo.*; +import com.tencent.supersonic.headless.api.pojo.request.*; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.BaseSemanticQuery; import com.tencent.supersonic.headless.chat.query.QueryManager; @@ -25,13 +15,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; import java.util.stream.Collectors; import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.TERM; @@ -233,8 +218,6 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { protected void convertBizNameToName(DataSetSchema dataSetSchema, QueryStructReq queryStructReq) { Map bizNameToName = dataSetSchema.getBizNameToName(); - bizNameToName.putAll(TimeDimensionEnum.getNameToNameMap()); - List orders = queryStructReq.getOrders(); if (CollectionUtils.isNotEmpty(orders)) { for (Order order : orders) { diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java index 48d191015..1db715190 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/detail/DetailSemanticQuery.java @@ -3,14 +3,13 @@ package com.tencent.supersonic.headless.chat.query.rule.detail; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; import java.time.LocalDate; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -33,10 +32,13 @@ public abstract class DetailSemanticQuery extends RuleSemanticQuery { chatQueryContext.getSemanticSchema().getDataSetSchemaMap(); DataSetSchema dataSetSchema = dataSetSchemaMap.get(parseInfo.getDataSetId()); TimeDefaultConfig timeDefaultConfig = dataSetSchema.getDetailTypeTimeDefaultConfig(); + SchemaElement partitionDimension = dataSetSchema.getPartitionDimension(); - if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit()) + if (Objects.nonNull(partitionDimension) && Objects.nonNull(timeDefaultConfig) + && Objects.nonNull(timeDefaultConfig.getUnit()) && timeDefaultConfig.getUnit() != -1) { DateConf dateInfo = new DateConf(); + dateInfo.setDateField(partitionDimension.getName()); int unit = timeDefaultConfig.getUnit(); String startDate = LocalDate.now().minusDays(unit).toString(); String endDate = startDate; diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java index a4d787b9e..e41bdc5ed 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/metric/MetricSemanticQuery.java @@ -3,14 +3,13 @@ package com.tencent.supersonic.headless.chat.query.rule.metric; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TimeMode; import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; +import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.query.rule.RuleSemanticQuery; import lombok.extern.slf4j.Slf4j; import java.time.LocalDate; -import java.util.List; import java.util.Objects; import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC; @@ -40,10 +39,12 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { DataSetSchema dataSetSchema = chatQueryContext.getSemanticSchema().getDataSetSchemaMap() .get(parseInfo.getDataSetId()); TimeDefaultConfig timeDefaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig(); - DateConf dateInfo = new DateConf(); - // 加上时间!=-1 判断 - if (Objects.nonNull(timeDefaultConfig) && Objects.nonNull(timeDefaultConfig.getUnit()) + SchemaElement partitionDimension = dataSetSchema.getPartitionDimension(); + if (Objects.nonNull(partitionDimension) && Objects.nonNull(timeDefaultConfig) + && Objects.nonNull(timeDefaultConfig.getUnit()) && timeDefaultConfig.getUnit() != -1) { + DateConf dateInfo = new DateConf(); + dateInfo.setDateField(partitionDimension.getName()); int unit = timeDefaultConfig.getUnit(); String startDate = LocalDate.now().minusDays(unit).toString(); String endDate = startDate; @@ -55,8 +56,8 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { dateInfo.setPeriod(timeDefaultConfig.getPeriod()); dateInfo.setStartDate(startDate); dateInfo.setEndDate(endDate); - // 时间不为-1才设置时间，所以移到这里 parseInfo.setDateInfo(dateInfo); } } + } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java index c84e526ad..22f21f700 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java @@ -7,9 +7,7 @@ import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; -import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SqlInfo; @@ -22,13 +20,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Slf4j @@ -179,14 +171,7 @@ public class QueryReqBuilder { if (Objects.isNull(dateConf)) { return ""; } - String dateField = TimeDimensionEnum.DAY.getName(); - if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) { - dateField = TimeDimensionEnum.MONTH.getName(); - } - if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) { - dateField = TimeDimensionEnum.WEEK.getName(); - } - return dateField; + return dateConf.getDateField(); } public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java index daba74b36..8a1275340 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java @@ -1,16 +1,13 @@ package com.tencent.supersonic.headless.core.pojo; +import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.core.translator.parser.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Metric; import lombok.Data; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Data @@ -28,4 +25,11 @@ public class Ontology { .collect(Collectors.toList()); } + public EngineType getDatabaseType() { + if (Objects.nonNull(database)) { + return database.getType(); + } + return null; + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index 59058fff2..af5a706dc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -9,6 +9,7 @@ import org.apache.commons.lang3.tuple.Triple; public class QueryStatement { private Long dataSetId; + private String dataSetName; private String sql; private String errMsg; private StructQuery structQuery; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index 88e9341e6..764f0b849 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.SqlQuery; import com.tencent.supersonic.headless.core.translator.optimizer.QueryOptimizer; @@ -26,7 +25,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator { return; } try { - for (QueryParser parser : ComponentFactory.getQueryParser()) { + for (QueryParser parser : ComponentFactory.getQueryParsers()) { if (parser.accept(queryStatement)) { log.debug("QueryConverter accept [{}]", parser.getClass().getName()); parser.parse(queryStatement); @@ -63,9 +62,6 @@ public class DefaultSemanticTranslator implements SemanticTranslator { } private void mergeOntologyQuery(QueryStatement queryStatement) throws Exception { - OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); - log.info("parse with ontology: [{}]", ontologyQuery); - SqlQuery sqlQuery = queryStatement.getSqlQuery(); String ontologyQuerySql = sqlQuery.getSql(); String ontologyInnerTable = sqlQuery.getTable(); @@ -74,7 +70,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator { List> tables = new ArrayList<>(); tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql)); if (sqlQuery.isSupportWith()) { - EngineType engineType = queryStatement.getOntology().getDatabase().getType(); + EngineType engineType = queryStatement.getOntology().getDatabaseType(); if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) { String withSql = "with " + tables.stream() .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight())) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java index 42c65ab2d..230a6349f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/DefaultDimValueParser.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.translator.parser; import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.parser.s2sql.Dimension; import lombok.extern.slf4j.Slf4j; @@ -40,9 +39,8 @@ public class DefaultDimValueParser implements QueryParser { return; } String sql = queryStatement.getSqlQuery().getSql(); - List whereFields = SqlSelectHelper.getWhereFields(sql).stream() - .filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) - .collect(Collectors.toList()); + List whereFields = + SqlSelectHelper.getWhereFields(sql).stream().collect(Collectors.toList()); if (!CollectionUtils.isEmpty(whereFields)) { return; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index 314e1cffd..f0c8d3f67 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -6,7 +6,6 @@ import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.SchemaItem; @@ -131,8 +130,6 @@ public class SqlQueryParser implements QueryParser { Map dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream() .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), SchemaItem::getBizName, (k1, k2) -> k1)); - dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(), - TimeDimensionEnum.DAY.getName()); return allFields.stream() .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())) @@ -315,8 +312,6 @@ public class SqlQueryParser implements QueryParser { entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap()); - dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap()); dimensionResults.putAll(metricResults); return dimensionResults; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Dimension.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Dimension.java index 7fcc6aaba..ad3fa704e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Dimension.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Dimension.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.core.translator.parser.s2sql; +import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/DimensionTimeTypeParams.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/DimensionTimeTypeParams.java deleted file mode 100644 index d1d1bbdba..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/DimensionTimeTypeParams.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; - -import lombok.Data; - -@Data -public class DimensionTimeTypeParams { - - private String isPrimary; - - private String timeGranularity; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index edce8fdc3..70ae9f333 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -54,7 +54,7 @@ public class ComponentFactory { return queryAccelerators; } - public static List getQueryParser() { + public static List getQueryParsers() { if (queryParsers.isEmpty()) { initQueryParsers(); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java index 3fcf769cf..582026120 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/DataTransformUtils.java @@ -2,8 +2,6 @@ package com.tencent.supersonic.headless.core.utils; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; @@ -53,7 +51,7 @@ public class DataTransformUtils { private static String getRowKey(Map originalRow, List groups) { List values = Lists.newArrayList(); for (String key : originalRow.keySet()) { - if (groups.contains(key) && !TimeDimensionEnum.getNameList().contains(key)) { + if (groups.contains(key)) { values.add(originalRow.get(key)); } } @@ -61,12 +59,6 @@ public class DataTransformUtils { } private static String getTimeDimension(DateConf dateConf) { - if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) { - return TimeDimensionEnum.MONTH.getName(); - } else if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) { - return TimeDimensionEnum.WEEK.getName(); - } else { - return TimeDimensionEnum.DAY.getName(); - } + return dateConf.getDateField(); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 0c5f87781..26b8ba9a0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -7,7 +7,6 @@ import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.StringUtil; @@ -29,19 +28,10 @@ import org.springframework.util.CollectionUtils; import java.time.LocalDate; import java.time.format.DateTimeFormatter; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.*; /** tools functions to analyze queryStructReq */ @Component @@ -68,12 +58,7 @@ public class SqlGenerateUtils { if (group.contains(JOIN_UNDERLINE)) { group = group.split(JOIN_UNDERLINE)[1]; } - if (!TimeDimensionEnum.getNameList().contains(group)) { - locate++; - sb.append(group).append(" as ").append("name").append(locate).append(","); - } else { - sb.append(group).append(","); - } + sb.append(group).append(","); } locate = 0; for (Aggregator agg : queryStructCmd.getAggregators()) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java deleted file mode 100644 index d17b301e8..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SysTimeDimensionBuilder.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.tencent.supersonic.headless.core.utils; - -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.Dimension; -import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; -import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; -import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** generate system time dimension tools */ -@Slf4j -public class SysTimeDimensionBuilder { - - // Defines the regular expression pattern for the time keyword - private static final Pattern TIME_KEYWORD_PATTERN = - Pattern.compile("\\b(DATE|TIME|TIMESTAMP|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)\\b", - Pattern.CASE_INSENSITIVE); - - public static void addSysTimeDimension(List dims, DbAdaptor engineAdaptor) { - log.debug("addSysTimeDimension before:{}, engineAdaptor:{}", dims, engineAdaptor); - Dimension timeDim = getTimeDim(dims); - if (timeDim == null) { - timeDim = Dimension.getDefault(); - // todo not find the time dimension - return; - } - dims.add(generateSysDayDimension(timeDim, engineAdaptor)); - dims.add(generateSysWeekDimension(timeDim, engineAdaptor)); - dims.add(generateSysMonthDimension(timeDim, engineAdaptor)); - log.debug("addSysTimeDimension after:{}, engineAdaptor:{}", dims, engineAdaptor); - } - - private static Dimension generateSysDayDimension(Dimension timeDim, DbAdaptor engineAdaptor) { - Dimension dim = new Dimension(); - dim.setBizName(TimeDimensionEnum.DAY.getName()); - dim.setType(DimensionType.partition_time); - dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.DAY.name().toLowerCase(), - engineAdaptor)); - DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams(); - typeParams.setTimeGranularity(TimeDimensionEnum.DAY.name().toLowerCase()); - typeParams.setIsPrimary("true"); - dim.setTypeParams(typeParams); - return dim; - } - - private static Dimension generateSysWeekDimension(Dimension timeDim, DbAdaptor engineAdaptor) { - Dimension dim = new Dimension(); - dim.setBizName(TimeDimensionEnum.WEEK.getName()); - dim.setType(DimensionType.partition_time); - dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.WEEK.name().toLowerCase(), - engineAdaptor)); - DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams(); - typeParams.setTimeGranularity(TimeDimensionEnum.WEEK.name().toLowerCase()); - typeParams.setIsPrimary("false"); - dim.setTypeParams(typeParams); - return dim; - } - - private static Dimension generateSysMonthDimension(Dimension timeDim, DbAdaptor engineAdaptor) { - Dimension dim = new Dimension(); - dim.setBizName(TimeDimensionEnum.MONTH.getName()); - dim.setType(DimensionType.partition_time); - dim.setExpr(generateTimeExpr(timeDim, TimeDimensionEnum.MONTH.name().toLowerCase(), - engineAdaptor)); - DimensionTimeTypeParams typeParams = new DimensionTimeTypeParams(); - typeParams.setTimeGranularity(TimeDimensionEnum.MONTH.name().toLowerCase()); - typeParams.setIsPrimary("false"); - dim.setTypeParams(typeParams); - return dim; - } - - private static boolean containsTimeKeyword(String fieldName) { - Matcher matcher = TIME_KEYWORD_PATTERN.matcher(fieldName); - return matcher.find(); - } - - // Check whether the time field contains keywords,Generation time expression - private static String generateTimeExpr(Dimension timeDim, String dateType, - DbAdaptor engineAdaptor) { - String bizName = timeDim.getBizName(); - String dateFormat = timeDim.getDateFormat(); - if (containsTimeKeyword(bizName)) { - String bizNameWithBackticks = String.format("`%s`", bizName); - return engineAdaptor.getDateFormat(dateType, dateFormat, bizNameWithBackticks); - } else { - return engineAdaptor.getDateFormat(dateType, dateFormat, bizName); - } - } - - private static Dimension getTimeDim(List timeDims) { - for (Dimension dim : timeDims) { - if (dim.getType().equals(DimensionType.partition_time)) { - return dim; - } - } - return null; - } -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java index 515c18115..e2cf3d139 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2ChatLayerService.java @@ -4,26 +4,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; -import com.tencent.supersonic.headless.api.pojo.SchemaElement; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticSchema; -import com.tencent.supersonic.headless.api.pojo.SqlEvaluation; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.ChatWorkflowState; import com.tencent.supersonic.headless.api.pojo.request.QueryMapReq; import com.tencent.supersonic.headless.api.pojo.request.QueryNLReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.response.DataSetMapInfo; -import com.tencent.supersonic.headless.api.pojo.response.DataSetResp; -import com.tencent.supersonic.headless.api.pojo.response.MapInfoResp; -import com.tencent.supersonic.headless.api.pojo.response.MapResp; -import com.tencent.supersonic.headless.api.pojo.response.ParseResp; -import com.tencent.supersonic.headless.api.pojo.response.SearchResult; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.chat.ChatQueryContext; import com.tencent.supersonic.headless.chat.corrector.GrammarCorrector; import com.tencent.supersonic.headless.chat.corrector.SchemaCorrector; @@ -40,13 +26,7 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -223,9 +203,6 @@ public class S2ChatLayerService implements ChatLayerService { .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(topN - 1).map(mergeFunction()).collect(Collectors.toSet()); - SchemaElementMatch timeDimensionMatch = getTimeDimension(dataSetId, dataSetName); - dimensions.add(timeDimensionMatch); - // topN metrics Set metrics = semanticSchema.getMetrics(dataSetId).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()).limit(topN) @@ -256,20 +233,6 @@ public class S2ChatLayerService implements ChatLayerService { return termMap; } - /** - * * get time dimension SchemaElementMatch - */ - private SchemaElementMatch getTimeDimension(Long dataSetId, String dataSetName) { - SchemaElement element = SchemaElement.builder().dataSetId(dataSetId) - .dataSetName(dataSetName).type(SchemaElementType.DIMENSION) - .bizName(TimeDimensionEnum.DAY.getName()).build(); - - return SchemaElementMatch.builder().element(element) - .detectWord(TimeDimensionEnum.DAY.getChName()) - .word(TimeDimensionEnum.DAY.getChName()).similarity(1L) - .frequency(BaseWordBuilder.DEFAULT_FREQUENCY).build(); - } - private Function mergeFunction() { return schemaElement -> SchemaElementMatch.builder().element(schemaElement) .frequency(BaseWordBuilder.DEFAULT_FREQUENCY).word(schemaElement.getName()) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index f2cb5b780..6f7e692f3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -5,8 +5,8 @@ import com.google.common.collect.Sets; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.*; +import com.tencent.supersonic.headless.api.pojo.DataSetSchema; +import com.tencent.supersonic.headless.api.pojo.MetaFilter; import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.request.*; import com.tencent.supersonic.headless.api.pojo.response.*; @@ -206,12 +206,6 @@ public class S2SemanticLayerService implements SemanticLayerService { ModelResp modelResp = modelResps.get(0); String sql = String.format("select distinct %s from %s where 1=1", dimensionResp.getName(), modelResp.getName()); - List timeDims = modelResp.getTimeDimension(); - if (CollectionUtils.isNotEmpty(timeDims)) { - sql = String.format("%s and %s >= '%s' and %s <= '%s'", sql, - TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(), - TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate()); - } if (StringUtils.isNotBlank(queryDimValueReq.getValue())) { sql += " AND " + queryDimValueReq.getBizName() + " LIKE '%" + queryDimValueReq.getValue() + "%'"; @@ -284,6 +278,7 @@ public class S2SemanticLayerService implements SemanticLayerService { && StringUtils.isNotBlank(semanticQueryReq.getSqlInfo().getQuerySQL())) { queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL()); queryStatement.setDataSetId(semanticQueryReq.getDataSetId()); + queryStatement.setDataSetName(semanticQueryReq.getDataSetName()); queryStatement.setIsTranslated(true); } return queryStatement; @@ -298,6 +293,7 @@ public class S2SemanticLayerService implements SemanticLayerService { QueryStatement queryStatement = new QueryStatement(); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setDataSetId(queryReq.getDataSetId()); + queryStatement.setDataSetName(queryReq.getDataSetName()); queryStatement.setSemanticSchema(semanticSchemaResp); queryStatement.setOntology(semanticSchemaManager.buildOntology(semanticSchemaResp)); return queryStatement; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java index 750d9d8a1..40793d2da 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/MetricYamlManager.java @@ -1,19 +1,10 @@ package com.tencent.supersonic.headless.server.manager; import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.FieldParam; -import com.tencent.supersonic.headless.api.pojo.MeasureParam; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; -import com.tencent.supersonic.headless.api.pojo.MetricParam; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.server.pojo.yaml.FieldParamYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricParamYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.server.pojo.yaml.*; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; @@ -47,7 +38,7 @@ public class MetricYamlManager { MetricDefineByMeasureParams metricDefineParams = metric.getMetricDefineByMeasureParams(); metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr()); - List measures = metricDefineParams.getMeasures(); + List measures = metricDefineParams.getMeasures(); metricTypeParamsYamlTpl.setMeasures( measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); } else if (MetricDefineType.FIELD.equals(metric.getMetricDefineType())) { @@ -68,7 +59,7 @@ public class MetricYamlManager { return metricYamlTpl; } - public static MeasureYamlTpl convert(MeasureParam measure) { + public static MeasureYamlTpl convert(Measure measure) { MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl(); measureYamlTpl.setName(measure.getBizName()); measureYamlTpl.setConstraint(measure.getConstraint()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index 8075d421a..85abf04c9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -1,17 +1,10 @@ package com.tencent.supersonic.headless.server.manager; -import com.tencent.supersonic.headless.api.pojo.Dimension; -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.ModelDetail; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; -import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; -import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; @@ -30,8 +23,6 @@ public class ModelYamlManager { public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) { ModelDetail modelDetail = modelResp.getModelDetail(); - DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); - SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor); DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl(); dataModelYamlTpl.setType(databaseResp.getType()); BeanUtils.copyProperties(modelDetail, dataModelYamlTpl); @@ -61,8 +52,7 @@ public class ModelYamlManager { dimensionYamlTpl.setExpr(dim.getBizName()); } if (dim.getTypeParams() != null) { - DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl = - new DimensionTimeTypeParamsTpl(); + DimensionTimeTypeParams dimensionTimeTypeParamsTpl = new DimensionTimeTypeParams(); dimensionTimeTypeParamsTpl.setIsPrimary(dim.getTypeParams().getIsPrimary()); dimensionTimeTypeParamsTpl.setTimeGranularity(dim.getTypeParams().getTimeGranularity()); dimensionYamlTpl.setTypeParams(dimensionTimeTypeParamsTpl); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 4b990c803..71d8dac15 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -187,24 +187,12 @@ public class SemanticSchemaManager { if (Objects.nonNull(dimensionYamlTpl.getExt())) { dimension.setExt(dimensionYamlTpl.getExt()); } - dimension.setDimensionTimeTypeParams( - getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams())); + dimension.setDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams()); dimensions.add(dimension); } return dimensions; } - private static DimensionTimeTypeParams getDimensionTimeTypeParams( - DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl) { - DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams(); - if (dimensionTimeTypeParamsTpl != null) { - dimensionTimeTypeParams - .setTimeGranularity(dimensionTimeTypeParamsTpl.getTimeGranularity()); - dimensionTimeTypeParams.setIsPrimary(dimensionTimeTypeParamsTpl.getIsPrimary()); - } - return dimensionTimeTypeParams; - } - private static List getIdentify(List identifyYamlTpls) { List identifies = new ArrayList<>(); for (IdentifyYamlTpl identifyYamlTpl : identifyYamlTpls) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java deleted file mode 100644 index b834b3be6..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionTimeTypeParamsTpl.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.tencent.supersonic.headless.server.pojo.yaml; - -import lombok.Data; - -@Data -public class DimensionTimeTypeParamsTpl { - - private String isPrimary; - - private String timeGranularity; -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java index 8d6f0a0c1..de679b549 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.server.pojo.yaml; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; +import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import lombok.Data; import java.util.List; @@ -19,7 +20,7 @@ public class DimensionYamlTpl { private String expr; - private DimensionTimeTypeParamsTpl typeParams; + private DimensionTimeTypeParams typeParams; private DataTypeEnums dataType; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index beefec3c5..4eb69d614 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -5,13 +5,7 @@ import com.alibaba.excel.ExcelWriter; import com.alibaba.excel.util.FileUtils; import com.alibaba.excel.write.metadata.WriteSheet; import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.pojo.User; -import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.pojo.*; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; import com.tencent.supersonic.headless.api.pojo.MetaFilter; @@ -35,21 +29,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Service; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.net.URLEncoder; import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Slf4j @@ -268,13 +251,7 @@ public class DownloadServiceImpl implements DownloadService { } private String getTimeDimension(DateConf dateConf) { - if (DatePeriodEnum.MONTH.equals(dateConf.getPeriod())) { - return TimeDimensionEnum.MONTH.getName(); - } else if (DatePeriodEnum.WEEK.equals(dateConf.getPeriod())) { - return TimeDimensionEnum.WEEK.getName(); - } else { - return TimeDimensionEnum.DAY.getName(); - } + return dateConf.getDateField(); } private Map> getMetricMap(List metricResps) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 936e050b2..a6251ed13 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -385,8 +385,8 @@ public class MetricServiceImpl extends ServiceImpl return true; } } else if (MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType())) { - List measures = metricResp.getMetricDefineByMeasureParams().getMeasures(); - List fieldNameDepended = measures.stream().map(MeasureParam::getBizName) + List measures = metricResp.getMetricDefineByMeasureParams().getMeasures(); + List fieldNameDepended = measures.stream().map(Measure::getBizName) // measure bizName = model bizName_fieldName .map(name -> name.replaceFirst(metricResp.getModelBizName() + "_", "")) .collect(Collectors.toList()); @@ -679,12 +679,11 @@ public class MetricServiceImpl extends ServiceImpl } // Measure define will get from first measure List measures = modelResp.getModelDetail().getMeasures(); - List measureParams = - metricResp.getMetricDefineByMeasureParams().getMeasures(); + List measureParams = metricResp.getMetricDefineByMeasureParams().getMeasures(); if (CollectionUtils.isEmpty(measureParams)) { return ""; } - MeasureParam firstMeasure = measureParams.get(0); + Measure firstMeasure = measureParams.get(0); for (Measure measure : measures) { if (measure.getBizName().equalsIgnoreCase(firstMeasure.getBizName())) { @@ -727,7 +726,7 @@ public class MetricServiceImpl extends ServiceImpl QueryStructReq queryStructReq = new QueryStructReq(); DateConf dateInfo = queryMetricReq.getDateInfo(); if (Objects.nonNull(dateInfo) && dateInfo.isGroupByDate()) { - queryStructReq.getGroups().add(dateInfo.getGroupByTimeDimension()); + queryStructReq.getGroups().add(dateInfo.getDateField()); } if (!CollectionUtils.isEmpty(dimensionBizNames)) { queryStructReq.getGroups().addAll(dimensionBizNames); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java index a40309f69..0b32bace2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagQueryServiceImpl.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.User; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.ValueDistribution; @@ -26,12 +25,7 @@ import org.springframework.util.StringUtils; import java.time.LocalDate; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; @Service @Slf4j @@ -121,8 +115,8 @@ public class TagQueryServiceImpl implements TagQueryService { User user) { String sqlPattern = "select max(%s) as %s from tbl where %s is not null"; - String sql = String.format(sqlPattern, TimeDimensionEnum.DAY.getName(), maxDateAlias, - tag.getBizName()); + String sql = String.format(sqlPattern, itemValueReq.getDateConf().getDateField(), + maxDateAlias, tag.getBizName()); // 添加时间过滤信息 log.info("[queryTagDateFromDbBySql] calculate the maximum time start"); @@ -140,8 +134,8 @@ public class TagQueryServiceImpl implements TagQueryService { String end = LocalDate.now().minusDays(0) .format(DateTimeFormatter.ofPattern(dateFormat)); sql = sql + String.format(" and ( %s > '%s' and %s <= '%s' )", - TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(), - end); + itemValueReq.getDateConf().getDateField(), start, + itemValueReq.getDateConf().getDateField(), end); } } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java index 4263abe40..2d900161f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java @@ -206,12 +206,6 @@ public class DataSetSchemaBuilder { private static void setDefaultTimeFormat(SchemaElement dimToAdd, DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) { - if (null != dimensionTimeTypeParams && TimeDimensionEnum.DAY.name() - .equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) { - dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, - DateUtils.DEFAULT_DATE_FORMAT); - } else { - dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat); - } + dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT, timeFormat); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 464372467..c906ad098 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -1,17 +1,7 @@ package com.tencent.supersonic.headless.server.utils; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.Filter; -import com.tencent.supersonic.common.pojo.Order; -import com.tencent.supersonic.common.pojo.User; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.pojo.*; +import com.tencent.supersonic.common.pojo.enums.*; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.headless.api.pojo.Dimension; @@ -20,13 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; -import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.response.TagResp; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; @@ -45,22 +29,9 @@ import org.springframework.util.CollectionUtils; import java.time.LocalDate; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.StringJoiner; +import java.util.*; -import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.POUND; -import static com.tencent.supersonic.common.pojo.Constants.SPACE; +import static com.tencent.supersonic.common.pojo.Constants.*; @Slf4j @Component @@ -456,14 +427,14 @@ public class DictUtils { return joiner.toString(); } - public String defaultDateFilter() { + public String defaultDateFilter(DateConf dateConf) { String format = itemValueDateFormat; String start = LocalDate.now().minusDays(itemValueDateStart) .format(DateTimeFormatter.ofPattern(format)); String end = LocalDate.now().minusDays(itemValueDateEnd) .format(DateTimeFormatter.ofPattern(format)); - return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), - start, TimeDimensionEnum.DAY.getName(), end); + return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start, + dateConf.getDateField(), end); } private String generateDictDateFilter(DictItemResp dictItemResp) { @@ -473,7 +444,7 @@ public class DictUtils { } // 未进行设置 if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) { - return defaultDateFilter(); + return defaultDateFilter(config.getDateConf()); } // 全表扫描 if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) { @@ -481,9 +452,9 @@ public class DictUtils { } // 静态日期 if (DateConf.DateMode.BETWEEN.equals(config.getDateConf().getDateMode())) { - return String.format("( %s >= '%s' and %s <= '%s' )", TimeDimensionEnum.DAY.getName(), - config.getDateConf().getStartDate(), TimeDimensionEnum.DAY.getName(), - config.getDateConf().getEndDate()); + return String.format("( %s >= '%s' and %s <= '%s' )", + config.getDateConf().getDateField(), config.getDateConf().getStartDate(), + config.getDateConf().getDateField(), config.getDateConf().getEndDate()); } // 动态日期 if (DateConf.DateMode.RECENT.equals(config.getDateConf().getDateMode())) { @@ -519,8 +490,8 @@ public class DictUtils { String end = LocalDate.now().minusDays(0) .format(DateTimeFormatter.ofPattern(dateFormat)); return String.format("( %s > '%s' and %s <= '%s' )", - TimeDimensionEnum.DAY.getName(), start, TimeDimensionEnum.DAY.getName(), - end); + dictItemResp.getConfig().getDateConf().getDateField(), start, + dictItemResp.getConfig().getDateConf().getDateField(), end); } } return ""; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java index ec5672f50..0cb952b41 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricCheckUtils.java @@ -46,9 +46,9 @@ public class MetricCheckUtils { throw new InvalidArgumentException("指标定义参数不可为空"); } expr = typeParams.getExpr(); - if (CollectionUtils.isEmpty(typeParams.getFields())) { - throw new InvalidArgumentException("定义指标的字段列表参数不可为空"); - } + // if (CollectionUtils.isEmpty(typeParams.getFields())) { + // throw new InvalidArgumentException("定义指标的字段列表参数不可为空"); + // } if (!hasAggregateFunction(expr)) { throw new InvalidArgumentException("基于字段来创建指标,表达式中必须包含聚合函数"); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java index 8489df62a..5fc8e2185 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java @@ -2,14 +2,9 @@ package com.tencent.supersonic.headless.server.utils; import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.server.service.MetricService; import lombok.extern.slf4j.Slf4j; @@ -61,9 +56,6 @@ public class MetricDrillDownChecker { } } for (String dimensionBizName : groupByFields) { - if (TimeDimensionEnum.containsTimeDimension(dimensionBizName)) { - continue; - } List metricResps = getMetrics(metricFields, semanticSchemaResp); if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) { DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index dcdfe42dc..751f28265 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -7,21 +7,8 @@ import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.util.BeanMapper; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.headless.api.pojo.ColumnSchema; -import com.tencent.supersonic.headless.api.pojo.Dimension; -import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.MeasureParam; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; -import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.ModelSchema; -import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; -import com.tencent.supersonic.headless.api.pojo.enums.FieldType; -import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; -import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; +import com.tencent.supersonic.headless.api.pojo.*; +import com.tencent.supersonic.headless.api.pojo.enums.*; import com.tencent.supersonic.headless.api.pojo.request.DimensionReq; import com.tencent.supersonic.headless.api.pojo.request.MetricReq; import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq; @@ -34,12 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.util.CollectionUtils; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; public class ModelConverter { @@ -141,7 +123,7 @@ public class ModelConverter { metricReq.setModelId(modelDO.getId()); MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams(); exprTypeParams.setExpr(measure.getBizName()); - MeasureParam measureParam = new MeasureParam(); + Measure measureParam = new Measure(); BeanMapper.mapper(measure, measureParam); exprTypeParams.setMeasures(Lists.newArrayList(measureParam)); metricReq.setMetricDefineByMeasureParams(exprTypeParams); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index ead019ffb..3e46b4278 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -2,13 +2,10 @@ package com.tencent.supersonic.headless.server.utils; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.supersonic.common.jsqlparser.FieldExpression; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; -import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.ItemDateResp; -import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; @@ -31,16 +28,7 @@ import org.springframework.util.CollectionUtils; import java.time.LocalDate; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; @@ -274,55 +262,4 @@ public class QueryStructUtils { return Triple.of("", "", ""); } - public DateConf getDateConfBySql(String sql) { - List fieldExpressions = SqlSelectHelper.getFilterExpression(sql); - if (!CollectionUtils.isEmpty(fieldExpressions)) { - Set dateList = new HashSet<>(); - String startDate = ""; - String endDate = ""; - DatePeriodEnum period = null; - for (FieldExpression f : fieldExpressions) { - if (Objects.isNull(f.getFieldName()) - || !internalCols.contains(f.getFieldName().toLowerCase())) { - continue; - } - if (Objects.isNull(f.getFieldValue()) - || !dateModeUtils.isDateStr(f.getFieldValue().toString())) { - continue; - } - period = dateModeUtils.getPeriodByCol(f.getFieldName().toLowerCase()); - if (period == null) { - continue; - } - if ("=".equals(f.getOperator())) { - dateList.add(f.getFieldValue().toString()); - } else if ("<".equals(f.getOperator()) || "<=".equals(f.getOperator())) { - if (startDate.isEmpty() - || startDate.compareTo(f.getFieldValue().toString()) > 0) { - startDate = f.getFieldValue().toString(); - } - } else if (">".equals(f.getOperator()) || ">=".equals(f.getOperator())) { - if (endDate.isEmpty() || endDate.compareTo(f.getFieldValue().toString()) < 0) { - endDate = f.getFieldValue().toString(); - } - } - } - if (period != null) { - DateConf dateConf = new DateConf(); - dateConf.setPeriod(period); - if (!CollectionUtils.isEmpty(dateList)) { - dateConf.setDateList(new ArrayList<>(dateList)); - dateConf.setDateMode(DateMode.LIST); - return dateConf; - } - if (!"".equals(startDate) && !"".equals(endDate)) { - dateConf.setStartDate(startDate); - dateConf.setEndDate(endDate); - dateConf.setDateMode(DateMode.BETWEEN); - return dateConf; - } - } - } - return null; - } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 150bb3cda..e3e7fefb8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.utils; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; @@ -55,10 +54,6 @@ public class QueryUtils { private void populateNamePairs(SemanticSchemaResp semanticSchemaResp, Map namePair, Map nameTypePair) { - for (TimeDimensionEnum timeDimensionEnum : TimeDimensionEnum.values()) { - namePair.put(timeDimensionEnum.getName(), "date"); - nameTypePair.put(timeDimensionEnum.getName(), "DATE"); - } semanticSchemaResp.getMetrics().forEach(metricDesc -> { namePair.put(metricDesc.getBizName(), metricDesc.getName()); nameTypePair.put(metricDesc.getBizName(), SemanticType.NUMBER.name()); diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index f4ea82d10..750b907ef 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.calcite; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.enums.EngineType; +import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.QueryStatement; @@ -76,7 +77,7 @@ class HeadlessParserServiceTest { dimension.setName("imp_date"); dimension.setExpr("imp_date"); dimension.setType("time"); - DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl(); + DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams(); dimensionTimeTypeParams.setIsPrimary("true"); dimensionTimeTypeParams.setTimeGranularity("day"); dimension.setTypeParams(dimensionTimeTypeParams); @@ -87,7 +88,7 @@ class HeadlessParserServiceTest { dimension2.setName("sys_imp_date"); dimension2.setExpr("imp_date"); dimension2.setType("time"); - DimensionTimeTypeParamsTpl dimensionTimeTypeParams2 = new DimensionTimeTypeParamsTpl(); + DimensionTimeTypeParams dimensionTimeTypeParams2 = new DimensionTimeTypeParams(); dimensionTimeTypeParams2.setIsPrimary("true"); dimensionTimeTypeParams2.setTimeGranularity("day"); dimension2.setTypeParams(dimensionTimeTypeParams2); @@ -97,7 +98,7 @@ class HeadlessParserServiceTest { dimension3.setName("sys_imp_week"); dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))"); dimension3.setType("time"); - DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl(); + DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams(); dimensionTimeTypeParams3.setIsPrimary("true"); dimensionTimeTypeParams3.setTimeGranularity("day"); dimension3.setTypeParams(dimensionTimeTypeParams3); @@ -201,7 +202,7 @@ class HeadlessParserServiceTest { dimension.setName("sys_imp_date"); dimension.setExpr("imp_date"); dimension.setType("time"); - DimensionTimeTypeParamsTpl dimensionTimeTypeParams = new DimensionTimeTypeParamsTpl(); + DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams(); dimensionTimeTypeParams.setIsPrimary("true"); dimensionTimeTypeParams.setTimeGranularity("day"); dimension.setTypeParams(dimensionTimeTypeParams); @@ -212,7 +213,7 @@ class HeadlessParserServiceTest { dimension3.setName("sys_imp_week"); dimension3.setExpr("to_monday(from_unixtime(unix_timestamp(imp_date), 'yyyy-MM-dd'))"); dimension3.setType("time"); - DimensionTimeTypeParamsTpl dimensionTimeTypeParams3 = new DimensionTimeTypeParamsTpl(); + DimensionTimeTypeParams dimensionTimeTypeParams3 = new DimensionTimeTypeParams(); dimensionTimeTypeParams3.setIsPrimary("true"); dimensionTimeTypeParams3.setTimeGranularity("week"); dimension3.setTypeParams(dimensionTimeTypeParams3); diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java deleted file mode 100644 index 8fe98ec0d..000000000 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/MetricServiceImplTest.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.tencent.supersonic.headless.server.service; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.DataFormat; -import com.tencent.supersonic.common.pojo.User; -import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum; -import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; -import com.tencent.supersonic.common.pojo.enums.TypeEnums; -import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.MeasureParam; -import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; -import com.tencent.supersonic.headless.api.pojo.RelateDimension; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.enums.MetricType; -import com.tencent.supersonic.headless.api.pojo.request.MetricReq; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.server.facade.service.ChatLayerService; -import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO; -import com.tencent.supersonic.headless.server.persistence.repository.MetricRepository; -import com.tencent.supersonic.headless.server.service.impl.DataSetServiceImpl; -import com.tencent.supersonic.headless.server.service.impl.MetricServiceImpl; -import com.tencent.supersonic.headless.server.utils.AliasGenerateHelper; -import com.tencent.supersonic.headless.server.utils.MetricConverter; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.springframework.context.ApplicationEventPublisher; - -import java.util.HashMap; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -public class MetricServiceImplTest { - - @Test - void createMetric() throws Exception { - MetricRepository metricRepository = Mockito.mock(MetricRepository.class); - ModelService modelService = Mockito.mock(ModelService.class); - MetricService metricService = mockMetricService(metricRepository, modelService); - MetricReq metricReq = buildMetricReq(); - when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp()); - when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList()); - MetricResp actualMetricResp = metricService.createMetric(metricReq, User.getDefaultUser()); - MetricResp expectedMetricResp = buildExpectedMetricResp(); - Assertions.assertEquals(expectedMetricResp, actualMetricResp); - } - - @Test - void updateMetric() throws Exception { - MetricRepository metricRepository = Mockito.mock(MetricRepository.class); - ModelService modelService = Mockito.mock(ModelService.class); - MetricService metricService = mockMetricService(metricRepository, modelService); - MetricReq metricReq = buildMetricUpdateReq(); - when(modelService.getModel(metricReq.getModelId())).thenReturn(mockModelResp()); - when(modelService.getModelByDomainIds(any())).thenReturn(Lists.newArrayList()); - MetricDO metricDO = MetricConverter.convert2MetricDO(buildMetricReq()); - when(metricRepository.getMetricById(metricDO.getId())).thenReturn(metricDO); - MetricResp actualMetricResp = metricService.updateMetric(metricReq, User.getDefaultUser()); - MetricResp expectedMetricResp = buildExpectedMetricResp(); - Assertions.assertEquals(expectedMetricResp, actualMetricResp); - } - - private MetricService mockMetricService(MetricRepository metricRepository, - ModelService modelService) { - AliasGenerateHelper aliasGenerateHelper = Mockito.mock(AliasGenerateHelper.class); - CollectService collectService = Mockito.mock(CollectService.class); - ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class); - DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class); - DimensionService dimensionService = Mockito.mock(DimensionService.class); - TagMetaService tagMetaService = Mockito.mock(TagMetaService.class); - ChatLayerService chatLayerService = Mockito.mock(ChatLayerService.class); - return new MetricServiceImpl(metricRepository, modelService, aliasGenerateHelper, - collectService, dataSetService, eventPublisher, dimensionService, tagMetaService, - chatLayerService); - } - - private MetricReq buildMetricReq() { - MetricReq metricReq = new MetricReq(); - metricReq.setId(1L); - metricReq.setName("hr部门的访问次数"); - metricReq.setBizName("pv"); - metricReq.setDescription("SuperSonic的访问情况"); - metricReq.setAlias("pv"); - metricReq.setMetricDefineType(MetricDefineType.MEASURE); - metricReq.setModelId(2L); - metricReq.setDataFormatType(DataFormatTypeEnum.PERCENT.getName()); - DataFormat dataFormat = new DataFormat(); - dataFormat.setDecimalPlaces(3); - dataFormat.setNeedMultiply100(false); - metricReq.setDataFormat(dataFormat); - MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams(); - typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"), - new MeasureParam("s2_uv", "department='hr'"))); - typeParams.setExpr("s2_pv/s2_uv"); - metricReq.setMetricDefineByMeasureParams(typeParams); - metricReq.setClassifications(Lists.newArrayList("核心指标")); - metricReq.setRelateDimension(RelateDimension.builder().drillDownDimensions( - Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false))) - .build()); - metricReq.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode()); - metricReq.setExt(new HashMap<>()); - return metricReq; - } - - private MetricResp buildExpectedMetricResp() { - MetricResp metricResp = new MetricResp(); - metricResp.setId(1L); - metricResp.setName("hr部门的访问次数"); - metricResp.setBizName("pv"); - metricResp.setDescription("SuperSonic的访问情况"); - metricResp.setAlias("pv"); - metricResp.setMetricDefineType(MetricDefineType.MEASURE); - metricResp.setModelId(2L); - metricResp.setDataFormatType(DataFormatTypeEnum.PERCENT.getName()); - DataFormat dataFormat = new DataFormat(); - dataFormat.setDecimalPlaces(3); - dataFormat.setNeedMultiply100(false); - metricResp.setDataFormat(dataFormat); - MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams(); - typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"), - new MeasureParam("s2_uv", "department='hr'"))); - typeParams.setExpr("s2_pv/s2_uv"); - metricResp.setMetricDefineByMeasureParams(typeParams); - metricResp.setClassifications("核心指标"); - metricResp.setRelateDimension(RelateDimension.builder().drillDownDimensions( - Lists.newArrayList(new DrillDownDimension(1L), new DrillDownDimension(1L, false))) - .build()); - metricResp.setSensitiveLevel(SensitiveLevelEnum.LOW.getCode()); - metricResp.setExt(new HashMap<>()); - metricResp.setTypeEnum(TypeEnums.METRIC); - metricResp.setIsCollect(false); - metricResp.setType(MetricType.DERIVED.name()); - metricResp.setStatus(StatusEnum.ONLINE.getCode()); - return metricResp; - } - - private MetricReq buildMetricUpdateReq() { - MetricReq metricReq = new MetricReq(); - metricReq.setId(1L); - metricReq.setName("hr部门的访问次数"); - metricReq.setBizName("pv"); - metricReq.setMetricDefineType(MetricDefineType.MEASURE); - MetricDefineByMeasureParams typeParams = new MetricDefineByMeasureParams(); - typeParams.setMeasures(Lists.newArrayList(new MeasureParam("s2_pv", "department='hr'"), - new MeasureParam("s2_uv", "department='hr'"))); - typeParams.setExpr("s2_pv/s2_uv"); - metricReq.setMetricDefineByMeasureParams(typeParams); - return metricReq; - } - - private ModelResp mockModelResp() { - ModelResp modelResp = new ModelResp(); - modelResp.setId(2L); - modelResp.setDomainId(1L); - return modelResp; - } -} diff --git a/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java b/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java index 097939a65..53ff6fecc 100644 --- a/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java +++ b/launchers/common/src/main/java/com/tencent/supersonic/web/WebConfig.java @@ -19,6 +19,7 @@ public class WebConfig implements WebMvcConfigurer { public void addViewControllers(ViewControllerRegistry registry) { registry.addViewController("/").setViewName("redirect:/webapp/"); registry.addViewController("/webapp/").setViewName("forward:/webapp/index.html"); - registry.addViewController("/webapp/**/{path:[^\\.]*}").setViewName("forward:/webapp/index.html"); + registry.addViewController("/webapp/**/{path:[^\\.]*}") + .setViewName("forward:/webapp/index.html"); } } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java index 381afd6ab..52aaa3fe4 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java @@ -60,6 +60,7 @@ public class S2VisitsDemo extends S2BaseDemo { DimensionResp userDimension = getDimension("user_name", userModel); updateMetric(stayTimeModel, departmentDimension, userDimension); updateMetric_pv(pvUvModel, departmentDimension, userDimension, metricPv); + addMetric_uv(pvUvModel, departmentDimension); // create dict conf for dimensions enableDimensionValue(departmentDimension); @@ -307,9 +308,9 @@ public class S2VisitsDemo extends S2BaseDemo { metricReq.setClassifications(Collections.singletonList("核心指标")); MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams(); metricTypeParams.setExpr("s2_stay_time_statis_stay_hours"); - List measures = new ArrayList<>(); - MeasureParam measure = new MeasureParam("s2_stay_time_statis_stay_hours", "", - AggOperatorEnum.SUM.getOperator()); + List measures = new ArrayList<>(); + Measure measure = new Measure("停留时长", "s2_stay_time_statis_stay_hours", + AggOperatorEnum.SUM.getOperator(), 0); measures.add(measure); metricTypeParams.setMeasures(measures); metricReq.setMetricDefineByMeasureParams(metricTypeParams); @@ -329,9 +330,9 @@ public class S2VisitsDemo extends S2BaseDemo { metricReq.setDescription("一段时间内用户的访问次数"); MetricDefineByMeasureParams metricTypeParams = new MetricDefineByMeasureParams(); metricTypeParams.setExpr("s2_pv_uv_statis_pv"); - List measures = new ArrayList<>(); - MeasureParam measure = - new MeasureParam("s2_pv_uv_statis_pv", "", AggOperatorEnum.SUM.getOperator()); + List measures = new ArrayList<>(); + Measure measure = + new Measure("访问次数", "s2_pv_uv_statis_pv", AggOperatorEnum.SUM.getOperator(), 0); measures.add(measure); metricTypeParams.setMeasures(measures); metricReq.setMetricDefineByMeasureParams(metricTypeParams); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java index 2a8c7738a..32cd62f8f 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java @@ -39,6 +39,7 @@ public class MetricTest extends BaseTest { } @Test + @SetSystemProperty(key = "s2.test", value = "true") public void testMetricModel() throws Exception { QueryResult actualResult = submitNewChat("超音数 访问次数", agent.getId()); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java index 263a0225e..da2787d8a 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java @@ -106,6 +106,7 @@ public class BaseTest extends BaseApplication { dateConf.setDateMode(DateMode.BETWEEN); dateConf.setEndDate(now().plusDays(0).toString()); dateConf.setStartDate(now().plusDays(-365).toString()); + dateConf.setDateField("imp_date"); queryStructReq.setDateInfo(dateConf); List orders = new ArrayList<>(); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 1bc6bebc9..cbbebabd8 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -79,6 +79,7 @@ public class DataUtils { dateInfo.setPeriod(period); dateInfo.setStartDate(startDate); dateInfo.setEndDate(endDate); + dateInfo.setDateField("imp_date"); return dateInfo; }