(improvement)(project) The Corrector is integrated into the Semantic module as an intermediate step for translating S2QL into SQL (#306)

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java

@@ -15,7 +15,7 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
 import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
 import com.tencent.supersonic.semantic.query.parser.SemanticConverter;
 import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
-import com.tencent.supersonic.semantic.query.utils.DateUtils;
+import com.tencent.supersonic.common.util.DateModeUtils;
 import com.tencent.supersonic.semantic.query.utils.QueryStructUtils;
 import com.tencent.supersonic.semantic.query.utils.SqlGenerateUtils;
 import java.util.ArrayList;
@@ -410,8 +410,8 @@ public class CalculateAggConverter implements SemanticConverter {
     }
 
     private static String getTimeDim(QueryStructReq queryStructCmd) {
-        DateUtils dateUtils = ContextUtils.getContext().getBean(DateUtils.class);
-        return dateUtils.getSysDateCol(queryStructCmd.getDateInfo());
+        DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class);
+        return dateModeUtils.getSysDateCol(queryStructCmd.getDateInfo());
     }
 
     private static String getLimit(QueryStructReq queryStructCmd) {

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/DefaultDimValueConverter.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.semantic.query.parser.convert;
 
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
-import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
+import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.semantic.api.query.request.MetricReq;
 import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.semantic.query.parser.convert;
 import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.semantic.api.query.request.MetricReq;
 import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MultiSourceJoin.java

@@ -8,7 +8,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.Measure;
 import com.tencent.supersonic.semantic.api.model.response.DatasourceResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.semantic.api.query.request.MetricReq;
 import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.semantic.query.parser.convert;
 
 
 import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.util.DateUtils;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -46,7 +47,6 @@ import org.springframework.util.CollectionUtils;
 @Slf4j
 public class QueryReqConverter {
 
-    public static final String TABLE_PREFIX = "t_";
     @Autowired
     private ModelService domainService;
     @Autowired
@@ -135,7 +135,7 @@ public class QueryReqConverter {
         Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResp);
         String sql = databaseReq.getSql();
         log.info("convert name to bizName before:{}", sql);
-        String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, false);
+        String replaceFields = SqlParserReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true);
         log.info("convert name to bizName after:{}", replaceFields);
         databaseReq.setSql(replaceFields);
     }
@@ -202,7 +202,8 @@ public class QueryReqConverter {
     }
 
     public void correctTableName(QueryS2QLReq databaseReq) {
-        String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(), TABLE_PREFIX + databaseReq.getModelId());
+        String sql = SqlParserReplaceHelper.replaceTable(databaseReq.getSql(),
+                Constants.TABLE_PREFIX + databaseReq.getModelId());
         databaseReq.setSql(sql);
     }
 

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java

@@ -17,9 +17,9 @@ import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
 import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
-import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
+import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
 import com.tencent.supersonic.semantic.api.query.pojo.Cache;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
 import com.tencent.supersonic.semantic.api.query.request.MetricReq;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DataPermissionAOP.java

@@ -20,8 +20,8 @@ import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaR
 import com.tencent.supersonic.semantic.api.model.response.ModelResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
-import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
+import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DateUtils.java

@@ -1,405 +0,0 @@
-package com.tencent.supersonic.semantic.query.utils;
-
-import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
-import static com.tencent.supersonic.common.pojo.Constants.COMMA;
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT;
-import static com.tencent.supersonic.common.pojo.Constants.WEEK;
-
-import com.google.common.base.Strings;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
-import java.time.temporal.ChronoUnit;
-import java.time.temporal.TemporalAdjusters;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-import java.util.StringJoiner;
-import java.util.regex.Pattern;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-
-@Slf4j
-@Component
-public class DateUtils {
-
-    @Value("${query.parameter.sys.date:sys_imp_date}")
-    private String sysDateCol;
-    @Value("${query.parameter.sys.month:sys_imp_month}")
-    private String sysDateMonthCol;
-    @Value("${query.parameter.sys.month:sys_imp_week}")
-    private String sysDateWeekCol;
-
-    @Value("${query.parameter.sys.zipper.begin:start_}")
-    private String sysZipperDateColBegin;
-    @Value("${query.parameter.sys.zipper.end:end_}")
-    private String sysZipperDateColEnd;
-
-    public Boolean recentMode(DateConf dateInfo) {
-        if (Objects.nonNull(dateInfo) && DateConf.DateMode.RECENT == dateInfo.getDateMode()
-                && DAY.equalsIgnoreCase(dateInfo.getPeriod()) && Objects.nonNull(dateInfo.getUnit())) {
-            return true;
-        }
-        return false;
-    }
-
-    public boolean hasAvailableDataMode(DateConf dateInfo) {
-        if (Objects.nonNull(dateInfo) && DateConf.DateMode.AVAILABLE == dateInfo.getDateMode()) {
-            return true;
-        }
-        return false;
-    }
-
-    /**
-     * dateMode = 4, advance time until data is available
-     *
-     * @param dateDate
-     * @param dateInfo
-     * @return
-     */
-    public String hasDataModeStr(ItemDateResp dateDate, DateConf dateInfo) {
-        if (Objects.isNull(dateDate)
-                || Strings.isNullOrEmpty(dateDate.getStartDate())
-                || Strings.isNullOrEmpty(dateDate.getStartDate())
-        ) {
-            return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(), sysDateCol,
-                    dateInfo.getEndDate());
-        } else {
-            log.info("dateDate:{}", dateDate);
-        }
-        String dateFormatStr = dateDate.getDateFormat();
-        if (Strings.isNullOrEmpty(dateFormatStr)) {
-            dateFormatStr = DAY_FORMAT;
-        }
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
-        LocalDate endData = LocalDate.parse(dateDate.getEndDate(), formatter);
-        LocalDate endReq = LocalDate.parse(dateInfo.getEndDate(), formatter);
-
-        if (endReq.isAfter(endData)) {
-            if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) {
-                Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateFormatStr, ChronoUnit.DAYS);
-                LocalDate dateMax = endData;
-                LocalDate dateMin = dateMax.minusDays(unit - 1);
-                return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, dateMax);
-            }
-
-            if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
-                Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateFormatStr,
-                        ChronoUnit.MONTHS);
-                return generateMonthSql(endData, unit, dateFormatStr);
-            }
-
-        }
-        return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateInfo.getStartDate(), sysDateCol,
-                dateInfo.getEndDate());
-    }
-
-
-    public String generateMonthSql(LocalDate endData, Long unit, String dateFormatStr) {
-        LocalDate dateMax = endData;
-        List<String> months = generateMonthStr(dateMax, unit, dateFormatStr);
-        if (!CollectionUtils.isEmpty(months)) {
-            StringJoiner joiner = new StringJoiner(",");
-            months.stream().forEach(month -> joiner.add("'" + month + "'"));
-            return String.format("(%s in (%s))", sysDateCol, joiner.toString());
-        }
-        return "";
-    }
-
-    private List<String> generateMonthStr(LocalDate dateMax, Long unit, String formatStr) {
-        DateTimeFormatter format = DateTimeFormatter.ofPattern(formatStr);
-        List<String> months = new ArrayList<>();
-        for (int i = unit.intValue() - 1; i >= 0; i--) {
-            LocalDate tmp = dateMax.minusMonths(i);
-            months.add(tmp.with(TemporalAdjusters.firstDayOfMonth()).format(format));
-        }
-        return months;
-    }
-
-    public String recentDayStr(ItemDateResp dateDate, DateConf dateInfo) {
-        ImmutablePair<String, String> dayRange = recentDay(dateDate, dateInfo);
-        return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dayRange.left, sysDateCol, dayRange.right);
-    }
-
-    public ImmutablePair<String, String> recentDay(ItemDateResp dateDate, DateConf dateInfo) {
-        String dateFormatStr = dateDate.getDateFormat();
-        if (Strings.isNullOrEmpty(dateFormatStr)) {
-            dateFormatStr = DAY_FORMAT;
-        }
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
-        LocalDate end = LocalDate.parse(dateDate.getEndDate(), formatter);
-        // todo  unavailableDateList logic
-
-        Integer unit = dateInfo.getUnit() - 1;
-        String start = end.minusDays(unit).format(formatter);
-        return ImmutablePair.of(start, dateDate.getEndDate());
-    }
-
-    public String recentMonthStr(LocalDate endData, Long unit, String dateFormatStr) {
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
-        String endStr = endData.format(formatter);
-        String start = endData.minusMonths(unit).format(formatter);
-        return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, start, sysDateMonthCol, endStr);
-    }
-
-    public String recentMonthStr(ItemDateResp dateDate, DateConf dateInfo) {
-        List<ImmutablePair<String, String>> range = recentMonth(dateDate, dateInfo);
-        if (range.size() == 1) {
-            return String.format("(%s >= '%s' and %s <= '%s')", sysDateMonthCol, range.get(0).left, sysDateMonthCol,
-                    range.get(0).right);
-        }
-        if (range.size() > 0) {
-            StringJoiner joiner = new StringJoiner(",");
-            range.stream().forEach(month -> joiner.add("'" + month.left + "'"));
-            return String.format("(%s in (%s))", sysDateCol, joiner.toString());
-        }
-        return "";
-    }
-
-    public List<ImmutablePair<String, String>> recentMonth(ItemDateResp dateDate, DateConf dateInfo) {
-        LocalDate endData = LocalDate.parse(dateDate.getEndDate(),
-                DateTimeFormatter.ofPattern(dateDate.getDateFormat()));
-        List<ImmutablePair<String, String>> ret = new ArrayList<>();
-        if (dateDate.getDatePeriod() != null && MONTH.equalsIgnoreCase(dateDate.getDatePeriod())) {
-            Long unit = getInterval(dateInfo.getStartDate(), dateInfo.getEndDate(), dateDate.getDateFormat(),
-                    ChronoUnit.MONTHS);
-            LocalDate dateMax = endData;
-            List<String> months = generateMonthStr(dateMax, unit, dateDate.getDateFormat());
-            if (!CollectionUtils.isEmpty(months)) {
-                months.stream().forEach(m -> ret.add(ImmutablePair.of(m, m)));
-                return ret;
-            }
-        }
-        String dateFormatStr = MONTH_FORMAT;
-        Integer unit = dateInfo.getUnit() - 1;
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
-        String endStr = endData.format(formatter);
-        String start = endData.minusMonths(unit).format(formatter);
-        ret.add(ImmutablePair.of(start, endStr));
-        return ret;
-    }
-
-    public String recentWeekStr(LocalDate endData, Long unit) {
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DAY_FORMAT);
-        String start = endData.minusDays(unit * 7).format(formatter);
-        return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, start, sysDateWeekCol,
-                endData.format(formatter));
-    }
-
-    public String recentWeekStr(ItemDateResp dateDate, DateConf dateInfo) {
-        ImmutablePair<String, String> dayRange = recentWeek(dateDate, dateInfo);
-        return String.format("(%s >= '%s' and %s <= '%s')", sysDateWeekCol, dayRange.left, sysDateWeekCol,
-                dayRange.right);
-    }
-
-    public ImmutablePair<String, String> recentWeek(ItemDateResp dateDate, DateConf dateInfo) {
-        String dateFormatStr = dateDate.getDateFormat();
-        if (Strings.isNullOrEmpty(dateFormatStr)) {
-            dateFormatStr = DAY_FORMAT;
-        }
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormatStr);
-        LocalDate end = LocalDate.parse(dateDate.getEndDate(), formatter);
-        Integer unit = dateInfo.getUnit() - 1;
-        String start = end.minusDays(unit * 7).format(formatter);
-        return ImmutablePair.of(start, end.format(formatter));
-    }
-
-    private Long getInterval(String startDate, String endDate, String dateFormat, ChronoUnit chronoUnit) {
-        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormat);
-        try {
-            LocalDate start = LocalDate.parse(startDate, formatter);
-            LocalDate end = LocalDate.parse(endDate, formatter);
-            return start.until(end, chronoUnit) + 1;
-        } catch (Exception e) {
-            log.warn("e:{}", e);
-        }
-        return -1L;
-    }
-
-
-    public String recentDateStr(ItemDateResp dateDate, DateConf dateInfo) {
-        if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) {
-            return recentDayStr(dateDate, dateInfo);
-        }
-        if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
-            return recentMonthStr(dateDate, dateInfo);
-        }
-        if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
-            return recentWeekStr(dateDate, dateInfo);
-        }
-        return "";
-    }
-
-    /**
-     * dateMode = 1; between, continuous value
-     *
-     * @param dateInfo
-     * @return
-     */
-    public String betweenDateStr(ItemDateResp dateDate, DateConf dateInfo) {
-        if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
-            LocalDate endData = LocalDate.parse(dateInfo.getEndDate(),
-                    DateTimeFormatter.ofPattern(DAY_FORMAT));
-            LocalDate startData = LocalDate.parse(dateInfo.getStartDate(),
-                    DateTimeFormatter.ofPattern(DAY_FORMAT));
-            DateTimeFormatter formatter = DateTimeFormatter.ofPattern(MONTH_FORMAT);
-            return String.format("%s >= '%s' and %s <= '%s'",
-                    sysDateMonthCol, startData.format(formatter), sysDateMonthCol, endData.format(formatter));
-        }
-        if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
-            return String.format("%s >= '%s' and %s <= '%s'",
-                    sysDateWeekCol, dateInfo.getStartDate(), sysDateWeekCol, dateInfo.getEndDate());
-        }
-        return String.format("%s >= '%s' and %s <= '%s'",
-                sysDateCol, dateInfo.getStartDate(), sysDateCol, dateInfo.getEndDate());
-    }
-
-    /**
-     * dateMode = 2; list discrete value
-     *
-     * @param dateInfo
-     * @return
-     */
-    public String listDateStr(ItemDateResp dateDate, DateConf dateInfo) {
-        StringJoiner joiner = new StringJoiner(COMMA);
-        dateInfo.getDateList().stream().forEach(date -> joiner.add(APOSTROPHE + date + APOSTROPHE));
-        String dateCol = sysDateCol;
-        if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
-            dateCol = sysDateMonthCol;
-        }
-        if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
-            dateCol = sysDateWeekCol;
-        }
-        return String.format("(%s in (%s))", dateCol, joiner.toString());
-    }
-
-    /**
-     * dateMode = 3; - recent time units
-     *
-     * @param dateInfo
-     * @return
-     */
-    public String defaultRecentDateInfo(DateConf dateInfo) {
-        if (Objects.isNull(dateInfo)) {
-            return "";
-        }
-
-        Integer unit = dateInfo.getUnit();
-
-        if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) {
-            LocalDate dateMax = LocalDate.now().minusDays(1);
-            LocalDate dateMin = dateMax.minusDays(unit - 1);
-            return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, dateMin, sysDateCol, dateMax);
-        }
-
-        if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
-            LocalDate dateMax = LocalDate.now().minusDays(1);
-            //return generateMonthSql(dateMax, unit.longValue(), DAY_FORMAT);
-            return recentMonthStr(dateMax, unit.longValue(), MONTH_FORMAT);
-        }
-        if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
-            LocalDate dateMax = LocalDate.now().minusDays(1);
-            return recentWeekStr(dateMax, unit.longValue());
-        }
-
-        return String.format("(%s >= '%s' and %s <= '%s')", sysDateCol, LocalDate.now().minusDays(2), sysDateCol,
-                LocalDate.now().minusDays(1));
-    }
-
-    public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) {
-        String dateStr = "";
-        switch (dateInfo.getDateMode()) {
-            case BETWEEN:
-                dateStr = betweenDateStr(dateDate, dateInfo);
-                break;
-            case LIST:
-                dateStr = listDateStr(dateDate, dateInfo);
-                break;
-            case RECENT:
-                dateStr = recentDateStr(dateDate, dateInfo);
-                break;
-            case AVAILABLE:
-                dateStr = hasDataModeStr(dateDate, dateInfo);
-                break;
-            default:
-                break;
-
-        }
-
-        return dateStr;
-    }
-
-    public String getDateWhereStr(DateConf dateConf, ImmutablePair<String, String> range) {
-        if (DAY.equalsIgnoreCase(dateConf.getPeriod()) || WEEK.equalsIgnoreCase(dateConf.getPeriod())) {
-            if (range.left.equals(range.right)) {
-                return String.format("(%s <= '%s' and %s > '%s')", sysZipperDateColBegin + sysDateCol, range.left,
-                        sysZipperDateColEnd + sysDateCol, range.left);
-            }
-            return String.format("( '%s' <= %s and '%s' >= %s)", range.left, sysZipperDateColEnd + sysDateCol,
-                    range.right, sysZipperDateColBegin + sysDateCol);
-        }
-
-        if (MONTH.equalsIgnoreCase(dateConf.getPeriod())) {
-            if (range.left.equals(range.right)) {
-                return String.format("(%s <= '%s' and %s > '%s')", sysZipperDateColBegin + sysDateMonthCol, range.left,
-                        sysZipperDateColEnd + sysDateMonthCol, range.left);
-            }
-            return String.format("( '%s' <= %s and '%s' >= %s)", range.left, sysZipperDateColEnd + sysDateMonthCol,
-                    range.right, sysZipperDateColBegin + sysDateMonthCol);
-
-        }
-        return "";
-    }
-
-    public String getSysDateCol(DateConf dateInfo) {
-        if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) {
-            return sysDateCol;
-        }
-        if (WEEK.equalsIgnoreCase(dateInfo.getPeriod())) {
-            return sysDateWeekCol;
-        }
-        if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
-            return sysDateMonthCol;
-        }
-        return "";
-    }
-
-    public boolean isDateStr(String date) {
-        return Pattern.matches("[\\d\\s-:]+", date);
-    }
-
-    public String getPeriodByCol(String col) {
-        if (sysDateCol.equalsIgnoreCase(col)) {
-            return DAY;
-        }
-        if (sysDateWeekCol.equalsIgnoreCase(col)) {
-            return WEEK;
-        }
-        if (sysDateMonthCol.equalsIgnoreCase(col)) {
-            return MONTH;
-        }
-        return "";
-    }
-
-    public String getDateColBegin(DateConf dateInfo) {
-        return sysZipperDateColBegin + getSysDateCol(dateInfo);
-    }
-
-    public String getDateColEnd(DateConf dateInfo) {
-        return sysZipperDateColEnd + getSysDateCol(dateInfo);
-    }
-
-    public List<String> getDateCol() {
-        return Arrays.asList(sysDateCol, sysDateMonthCol, sysDateWeekCol);
-    }
-
-}

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/DimValueAspect.java

@@ -8,8 +8,8 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.semantic.api.model.pojo.DimValueMap;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
-import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
+import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.semantic.api.query.request.QueryS2QLReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.model.domain.DimensionService;

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java

@@ -11,6 +11,8 @@ import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.DateConf.DateMode;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
+import com.tencent.supersonic.common.util.DateModeUtils;
+import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.StringUtil;
 import com.tencent.supersonic.common.util.jsqlparser.FilterExpression;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
@@ -21,7 +23,7 @@ import com.tencent.supersonic.semantic.api.model.pojo.SchemaItem;
 import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
 import com.tencent.supersonic.semantic.api.model.response.DimSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
-import com.tencent.supersonic.semantic.api.model.response.ItemDateResp;
+import com.tencent.supersonic.common.pojo.ItemDateResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
@@ -67,7 +69,7 @@ public class QueryStructUtils {
         internalCols.addAll(internalTimeCols);
     }
 
-    private final DateUtils dateUtils;
+    private final DateModeUtils dateModeUtils;
     private final SqlFilterUtils sqlFilterUtils;
     private final Catalog catalog;
     @Value("${internal.metric.cnt.suffix:internal_cnt}")
@@ -82,10 +84,10 @@ public class QueryStructUtils {
     private String variablePrefix = "'${";
 
     public QueryStructUtils(
-            DateUtils dateUtils,
+            DateModeUtils dateModeUtils,
             SqlFilterUtils sqlFilterUtils, Catalog catalog) {
 
-        this.dateUtils = dateUtils;
+        this.dateModeUtils = dateModeUtils;
         this.sqlFilterUtils = sqlFilterUtils;
         this.catalog = catalog;
     }
@@ -145,19 +147,19 @@ public class QueryStructUtils {
                 || Strings.isEmpty(dateDate.getStartDate())
                 && Strings.isEmpty(dateDate.getEndDate())) {
             if (dateInfo.getDateMode().equals(DateMode.LIST)) {
-                return dateUtils.listDateStr(dateDate, dateInfo);
+                return dateModeUtils.listDateStr(dateDate, dateInfo);
             }
             if (dateInfo.getDateMode().equals(DateMode.BETWEEN)) {
-                return dateUtils.betweenDateStr(dateDate, dateInfo);
+                return dateModeUtils.betweenDateStr(dateDate, dateInfo);
             }
-            if (dateUtils.hasAvailableDataMode(dateInfo)) {
-                return dateUtils.hasDataModeStr(dateDate, dateInfo);
+            if (dateModeUtils.hasAvailableDataMode(dateInfo)) {
+                return dateModeUtils.hasDataModeStr(dateDate, dateInfo);
             }
 
-            return dateUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
+            return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
         }
         log.info("dateDate:{}", dateDate);
-        return dateUtils.getDateWhereStr(dateInfo, dateDate);
+        return dateModeUtils.getDateWhereStr(dateInfo, dateDate);
     }
 
 
@@ -177,7 +179,7 @@ public class QueryStructUtils {
             return whereFromDate;
         } else if (Strings.isEmpty(whereFromDate) && Strings.isEmpty(whereClauseFromFilter)) {
             log.info("the current date information is empty, enter the date initialization logic");
-            return dateUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
+            return dateModeUtils.defaultRecentDateInfo(queryStructCmd.getDateInfo());
         }
         return whereClauseFromFilter;
     }
@@ -283,19 +285,19 @@ public class QueryStructUtils {
     public String generateZipperWhere(QueryStatement queryStatement, QueryStructReq queryStructReq) {
         if (Objects.nonNull(queryStatement.getParseSqlReq().getSql())) {
             String sql = SqlParserRemoveHelper.removeWhere(queryStatement.getParseSqlReq().getSql(),
-                    dateUtils.getDateCol());
+                    dateModeUtils.getDateCol());
             if (!CollectionUtils.isEmpty(queryStatement.getMetricReq().getDimensions())) {
                 List<String> dimension = queryStatement.getMetricReq().getDimensions().stream()
-                        .filter(d -> !dateUtils.getDateCol().contains(d.toLowerCase())).collect(
+                        .filter(d -> !dateModeUtils.getDateCol().contains(d.toLowerCase())).collect(
                                 Collectors.toList());
-                dimension.add(dateUtils.getDateColBegin(queryStructReq.getDateInfo()));
-                dimension.add(dateUtils.getDateColEnd(queryStructReq.getDateInfo()));
+                dimension.add(dateModeUtils.getDateColBegin(queryStructReq.getDateInfo()));
+                dimension.add(dateModeUtils.getDateColEnd(queryStructReq.getDateInfo()));
                 queryStatement.getMetricReq().setDimensions(dimension);
             }
             return SqlParserAddHelper.addWhere(sql,
                     SqlParserSelectHelper.getTimeFilter(queryStatement.getTimeRanges(),
-                            dateUtils.getDateColBegin(queryStructReq.getDateInfo()),
-                            dateUtils.getDateColEnd(queryStructReq.getDateInfo())));
+                            dateModeUtils.getDateColBegin(queryStructReq.getDateInfo()),
+                            dateModeUtils.getDateColEnd(queryStructReq.getDateInfo())));
         }
         return queryStatement.getSql();
     }
@@ -306,7 +308,7 @@ public class QueryStructUtils {
         List<String> wheres = new ArrayList<>();
         if (!CollectionUtils.isEmpty(timeRanges)) {
             for (ImmutablePair<String, String> range : timeRanges) {
-                String strWhere = dateUtils.getDateWhereStr(queryStructCmd.getDateInfo(), range);
+                String strWhere = dateModeUtils.getDateWhereStr(queryStructCmd.getDateInfo(), range);
                 if (!strWhere.isEmpty()) {
                     wheres.add(strWhere);
                 }
@@ -337,11 +339,11 @@ public class QueryStructUtils {
                 }
                 switch (dateConf.getPeriod()) {
                     case DAY:
-                        return dateUtils.recentDay(dateDate, dateConf);
+                        return dateModeUtils.recentDay(dateDate, dateConf);
                     case WEEK:
-                        return dateUtils.recentWeek(dateDate, dateConf);
+                        return dateModeUtils.recentWeek(dateDate, dateConf);
                     case MONTH:
-                        List<ImmutablePair<String, String>> rets = dateUtils.recentMonth(dateDate, dateConf);
+                        List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dateDate, dateConf);
                         Optional<String> minBegins = rets.stream().map(i -> i.left).sorted().findFirst();
                         Optional<String> maxBegins = rets.stream().map(i -> i.right).sorted(Comparator.reverseOrder())
                                 .findFirst();
@@ -387,13 +389,13 @@ public class QueryStructUtils {
                 }
                 switch (dateConf.getPeriod()) {
                     case DAY:
-                        ret.add(dateUtils.recentDay(dateDate, dateConf));
+                        ret.add(dateModeUtils.recentDay(dateDate, dateConf));
                         break;
                     case WEEK:
-                        ret.add(dateUtils.recentWeek(dateDate, dateConf));
+                        ret.add(dateModeUtils.recentWeek(dateDate, dateConf));
                         break;
                     case MONTH:
-                        List<ImmutablePair<String, String>> rets = dateUtils.recentMonth(dateDate, dateConf);
+                        List<ImmutablePair<String, String>> rets = dateModeUtils.recentMonth(dateDate, dateConf);
                         ret.addAll(rets);
                         break;
                     default:
@@ -426,10 +428,10 @@ public class QueryStructUtils {
                 if (Objects.isNull(f.getFieldName()) || !internalCols.contains(f.getFieldName().toLowerCase())) {
                     continue;
                 }
-                if (Objects.isNull(f.getFieldValue()) || !dateUtils.isDateStr(f.getFieldValue().toString())) {
+                if (Objects.isNull(f.getFieldValue()) || !dateModeUtils.isDateStr(f.getFieldValue().toString())) {
                     continue;
                 }
-                period = dateUtils.getPeriodByCol(f.getFieldName().toLowerCase());
+                period = dateModeUtils.getPeriodByCol(f.getFieldName().toLowerCase());
                 if ("".equals(period)) {
                     continue;
                 }
@@ -465,7 +467,7 @@ public class QueryStructUtils {
     }
 
     public List<String> getDateCol() {
-        return dateUtils.getDateCol();
+        return dateModeUtils.getDateCol();
     }
 
     public String getVariablePrefix() {

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/SqlFilterUtils.java

@@ -1,245 +0,0 @@
-package com.tencent.supersonic.semantic.query.utils;
-
-import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_END;
-import static com.tencent.supersonic.common.pojo.Constants.PARENTHESES_START;
-import static com.tencent.supersonic.common.pojo.Constants.SPACE;
-import static com.tencent.supersonic.common.pojo.Constants.SYS_VAR;
-
-import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
-import com.tencent.supersonic.semantic.api.query.pojo.Criterion;
-import com.tencent.supersonic.semantic.api.query.pojo.Filter;
-import com.tencent.supersonic.common.pojo.Constants;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.StringJoiner;
-import java.util.regex.Pattern;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.logging.log4j.util.Strings;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-
-@Component
-@Slf4j
-public class SqlFilterUtils {
-
-    private static String pattern = "^'.*?'$";
-    private static String numericPattern = "^[0-9]+$";
-
-    public List<String> getFiltersCol(List<Filter> filters) {
-        List<String> filterCols = new ArrayList<>();
-        if (CollectionUtils.isEmpty(filters)) {
-            return filterCols;
-        }
-        for (Filter filter : filters) {
-            filterCols.addAll(getFilterCol(filter));
-        }
-        return filterCols;
-    }
-
-    private List<String> getFilterCol(Filter filter) {
-        List<String> filterCols = new ArrayList<>();
-        if (Filter.Relation.FILTER.equals(filter.getRelation())) {
-            if (Strings.isNotEmpty(filter.getBizName())) {
-                filterCols.add(filter.getBizName());
-            }
-        }
-
-        List<Filter> children = filter.getChildren();
-        if (!CollectionUtils.isEmpty(children)) {
-            for (Filter child : children) {
-                filterCols.addAll(getFilterCol(child));
-            }
-        }
-        return filterCols;
-    }
-
-
-    public String getWhereClause(List<Filter> filters) {
-        StringJoiner joiner = new StringJoiner(Constants.AND_UPPER);
-
-        if (!CollectionUtils.isEmpty(filters)) {
-            filters.stream()
-                    .forEach(filter -> {
-                        if (Strings.isNotEmpty(dealFilter(filter))) {
-                            joiner.add(SPACE + dealFilter(filter) + SPACE);
-                        }
-                    });
-            log.info("getWhereClause, where sql : {}", joiner.toString());
-            return joiner.toString();
-        }
-
-        return "";
-    }
-
-    public String dealFilter(Filter filter) {
-        if (Objects.isNull(filter)) {
-            return "";
-        }
-        if (Strings.isNotEmpty(filter.getBizName()) && filter.getBizName().endsWith(SYS_VAR)) {
-            return "";
-        }
-        StringBuilder condition = new StringBuilder();
-        if (Filter.Relation.FILTER.equals(filter.getRelation())) {
-            return dealSingleFilter(filter);
-        }
-
-        List<Filter> children = filter.getChildren();
-        condition.append(PARENTHESES_START);
-        StringJoiner joiner = new StringJoiner(SPACE + filter.getRelation().name() + SPACE);
-        for (Filter child : children) {
-            joiner.add(dealFilter(child));
-        }
-        condition.append(joiner.toString());
-        condition.append(PARENTHESES_END);
-        return condition.toString();
-    }
-
-    // todo deal metric filter
-    private String dealSingleFilter(Filter filter) {
-        String name = filter.getBizName();
-        Object value = filter.getValue();
-        FilterOperatorEnum operator = filter.getOperator();
-
-        String dataType = Criterion.StringDataType.STRING.name();
-
-        Criterion criterion = new Criterion(name, operator, value, dataType);
-        return generator(criterion);
-    }
-
-    private String generator(Criterion criterion) {
-        log.info("criterion :{}", criterion);
-        String sqlPart;
-        switch (criterion.getOperator()) {
-            case SQL_PART:
-                sqlPart = sqlPartLogic(criterion);
-                break;
-
-            case IS_NULL:
-            case IS_NOT_NULL:
-                sqlPart = judgeNullLogic(criterion);
-                break;
-
-            case EQUALS:
-            case NOT_EQUALS:
-            case GREATER_THAN:
-            case GREATER_THAN_EQUALS:
-            case MINOR_THAN:
-            case MINOR_THAN_EQUALS:
-                sqlPart = singleValueLogic(criterion);
-                break;
-
-            case BETWEEN:
-                sqlPart = betweenLogic(criterion);
-                break;
-
-            case IN:
-            case NOT_IN:
-                sqlPart = inLogic(criterion);
-                break;
-            case LIKE:
-                sqlPart = likeLogic(criterion);
-
-                break;
-            default:
-                throw new IllegalStateException("Unexpected value: " + criterion.getOperator());
-        }
-        return sqlPart;
-    }
-
-    private String likeLogic(Criterion criterion) {
-        if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
-            throw new RuntimeException("criterion.getValue() can not be null");
-        }
-        StringBuilder whereClause = new StringBuilder();
-        whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE);
-        String value = criterion.getValue().toString();
-        if (criterion.isNeedApostrophe() && !Pattern.matches(pattern, value)) {
-            // like click => 'like%'
-            whereClause.append(Constants.APOSTROPHE + value + Constants.PERCENT_SIGN + Constants.APOSTROPHE);
-
-        } else {
-            // like 'click' => 'like%'
-            whereClause.append(Constants.APOSTROPHE + value.replaceAll(Constants.APOSTROPHE, Constants.PERCENT_SIGN)
-                    + Constants.APOSTROPHE);
-        }
-        return whereClause.toString();
-    }
-
-    private String inLogic(Criterion criterion) {
-        if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
-            throw new RuntimeException("criterion.getValue() can not be null");
-        }
-
-        StringBuilder whereClause = new StringBuilder();
-        whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE);
-        List values = (List) criterion.getValue();
-        whereClause.append(PARENTHESES_START);
-        StringJoiner joiner = new StringJoiner(",");
-        if (criterion.isNeedApostrophe()) {
-            values.stream().forEach(value -> joiner.add(valueApostropheLogic(value.toString())));
-        } else {
-            values.stream().forEach(value -> joiner.add(value.toString()));
-        }
-        whereClause.append(joiner);
-        whereClause.append(PARENTHESES_END);
-        return whereClause.toString();
-    }
-
-    private String betweenLogic(Criterion criterion) {
-        if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
-            throw new RuntimeException("criterion.getValue() can not be null");
-        }
-        List values = (List) criterion.getValue();
-        if (values.size() != 2) {
-            throw new RuntimeException("between value size should be 2");
-        }
-
-        if (criterion.isNeedApostrophe()) {
-            return String.format("(%s >= %s and %s <= %s)", criterion.getColumn(),
-                    valueApostropheLogic(values.get(0).toString()),
-                    criterion.getColumn(), valueApostropheLogic(values.get(1).toString()));
-        }
-        return String.format("(%s >= %s and %s <= %s)", criterion.getColumn(), values.get(0).toString(),
-                criterion.getColumn(), values.get(1).toString());
-    }
-
-    private String singleValueLogic(Criterion criterion) {
-        if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
-            throw new RuntimeException("criterion.getValue() can not be null");
-        }
-        StringBuilder whereClause = new StringBuilder();
-        whereClause.append(criterion.getColumn() + SPACE + criterion.getOperator().getValue() + SPACE);
-        String value = criterion.getValue().toString();
-        if (criterion.isNeedApostrophe()) {
-            value = valueApostropheLogic(value);
-        }
-        whereClause.append(value);
-        return whereClause.toString();
-    }
-
-    private String valueApostropheLogic(String value) {
-        if (Pattern.matches(pattern, value) || Pattern.matches(numericPattern, value)) {
-            return value;
-        }
-        return Constants.APOSTROPHE + value + Constants.APOSTROPHE;
-    }
-
-    private String judgeNullLogic(Criterion criterion) {
-
-        if (Objects.isNull(criterion) || Objects.isNull(criterion.getColumn())) {
-            throw new RuntimeException("criterion.getColumn() can not be null");
-        }
-        return String.format("( %s %s)", criterion.getColumn(), criterion.getOperator().getValue());
-    }
-
-    private String sqlPartLogic(Criterion criterion) {
-        if (Objects.isNull(criterion) || Objects.isNull(criterion.getValue())) {
-            throw new RuntimeException("criterion.getValue() can not be null");
-        }
-        return PARENTHESES_START + SPACE + criterion.getValue().toString() + SPACE + PARENTHESES_END;
-    }
-
-
-}

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.semantic.api.model.enums.QueryTypeBackEnum;
 import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;