(improvement)(chat) The rules will uniformly use 'data date' and exclude data partition dimensions for dimension pushdown. (#1598)

common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java

@@ -16,7 +16,6 @@ import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
 import java.time.temporal.TemporalAdjusters;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.StringJoiner;
@@ -251,7 +250,7 @@ public class DateModeUtils {
      * @param dateInfo
      * @return
      */
-    public String betweenDateStr(ItemDateResp dateDate, DateConf dateInfo) {
+    public String betweenDateStr(DateConf dateInfo) {
         if (MONTH.equalsIgnoreCase(dateInfo.getPeriod())) {
             // startDate YYYYMM
             if (!dateInfo.getStartDate().contains(Constants.MINUS)) {
@@ -280,7 +279,7 @@ public class DateModeUtils {
      * @param dateInfo
      * @return
      */
-    public String listDateStr(ItemDateResp dateDate, DateConf dateInfo) {
+    public String listDateStr(DateConf dateInfo) {
         StringJoiner joiner = new StringJoiner(COMMA);
         dateInfo.getDateList().stream().forEach(date -> joiner.add(APOSTROPHE + date + APOSTROPHE));
         String dateCol = sysDateCol;
@@ -330,8 +329,7 @@ public class DateModeUtils {
     }
 
     public String getDateWhereStr(DateConf dateInfo) {
-        ItemDateResp dateDate = null;
+        return getDateWhereStr(dateInfo, null);
-        return getDateWhereStr(dateInfo, dateDate);
     }
 
     public String getDateWhereStr(DateConf dateInfo, ItemDateResp dateDate) {
@@ -341,10 +339,10 @@ public class DateModeUtils {
         String dateStr = "";
         switch (dateInfo.getDateMode()) {
             case BETWEEN:
-                dateStr = betweenDateStr(dateDate, dateInfo);
+                dateStr = betweenDateStr(dateInfo);
                 break;
             case LIST:
-                dateStr = listDateStr(dateDate, dateInfo);
+                dateStr = listDateStr(dateInfo);
                 break;
             case RECENT:
                 dateStr = recentDateStr(dateDate, dateInfo);
@@ -356,32 +354,9 @@ public class DateModeUtils {
                 break;
 
         }
-
         return dateStr;
     }
 
-    public String getDateWhereStr(DateConf dateConf, ImmutablePair<String, String> range) {
-        if (DAY.equalsIgnoreCase(dateConf.getPeriod()) || WEEK.equalsIgnoreCase(dateConf.getPeriod())) {
-            if (range.left.equals(range.right)) {
-                return String.format("(%s <= '%s' and %s > '%s')", sysZipperDateColBegin + sysDateCol, range.left,
-                        sysZipperDateColEnd + sysDateCol, range.left);
-            }
-            return String.format("( '%s' <= %s and '%s' >= %s)", range.left, sysZipperDateColEnd + sysDateCol,
-                    range.right, sysZipperDateColBegin + sysDateCol);
-        }
-
-        if (MONTH.equalsIgnoreCase(dateConf.getPeriod())) {
-            if (range.left.equals(range.right)) {
-                return String.format("(%s <= '%s' and %s > '%s')", sysZipperDateColBegin + sysDateMonthCol, range.left,
-                        sysZipperDateColEnd + sysDateMonthCol, range.left);
-            }
-            return String.format("( '%s' <= %s and '%s' >= %s)", range.left, sysZipperDateColEnd + sysDateMonthCol,
-                    range.right, sysZipperDateColBegin + sysDateMonthCol);
-
-        }
-        return "";
-    }
-
     public String getSysDateCol(DateConf dateInfo) {
         if (DAY.equalsIgnoreCase(dateInfo.getPeriod())) {
             return sysDateCol;
@@ -412,16 +387,4 @@ public class DateModeUtils {
         return "";
     }
 
-    public String getDateColBegin(DateConf dateInfo) {
-        return sysZipperDateColBegin + getSysDateCol(dateInfo);
-    }
-
-    public String getDateColEnd(DateConf dateInfo) {
-        return sysZipperDateColEnd + getSysDateCol(dateInfo);
-    }
-
-    public List<String> getDateCol() {
-        return Arrays.asList(sysDateCol, sysDateMonthCol, sysDateWeekCol);
-    }
-
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimensionResp.java

@@ -47,4 +47,8 @@ public class DimensionResp extends SchemaItem {
         return DimensionType.isTimeDimension(type);
     }
 
+    public boolean isPartitionTime() {
+        return DimensionType.isPartitionTime(type);
+    }
+
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java

@@ -53,9 +53,12 @@ public class QueryReqBuilder {
         queryStructReq.setMetricFilters(metricFilters);
 
         addDateDimension(parseInfo);
-        List<String> dimensions = parseInfo.getDimensions().stream().map(SchemaElement::getBizName)
+
-                .collect(Collectors.toList());
+        if (isDateFieldAlreadyPresent(parseInfo, getDateField(parseInfo.getDateInfo()))) {
-        queryStructReq.setGroups(dimensions);
+            parseInfo.getDimensions().removeIf(schemaElement -> schemaElement.containsPartitionTime());
+        }
+        queryStructReq.setGroups(parseInfo.getDimensions().stream().map(SchemaElement::getBizName)
+                .collect(Collectors.toList()));
         queryStructReq.setLimit(parseInfo.getLimit());
         // only one metric is queried at once
         Set<SchemaElement> metrics = parseInfo.getMetrics();
@@ -212,8 +215,7 @@ public class QueryReqBuilder {
 
     private static boolean isDateFieldAlreadyPresent(SemanticParseInfo parseInfo, String dateField) {
         return parseInfo.getDimensions().stream()
-                .anyMatch(dimension -> dimension.getBizName().equalsIgnoreCase(dateField)
+                .anyMatch(dimension -> dimension.getBizName().equalsIgnoreCase(dateField));
-                        || dimension.containsPartitionTime());
     }
 
     private static void addDimension(SemanticParseInfo parseInfo, SchemaElement dimension) {

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java

@@ -62,7 +62,7 @@ public class SqlGenerateUtils {
     private final ExecutorConfig executorConfig;
 
     public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils,
-            DateModeUtils dateModeUtils, ExecutorConfig executorConfig) {
+                            DateModeUtils dateModeUtils, ExecutorConfig executorConfig) {
         this.sqlFilterUtils = sqlFilterUtils;
         this.dateModeUtils = dateModeUtils;
         this.executorConfig = executorConfig;
@@ -168,7 +168,7 @@ public class SqlGenerateUtils {
     }
 
     private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter,
-            String whereFromDate) {
+                                        String whereFromDate) {
         if (StringUtils.isNotEmpty(whereFromDate) && StringUtils.isNotEmpty(whereClauseFromFilter)) {
             return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
         } else if (StringUtils.isEmpty(whereFromDate) && StringUtils.isNotEmpty(whereClauseFromFilter)) {
@@ -187,10 +187,10 @@ public class SqlGenerateUtils {
                 || StringUtils.isEmpty(dateDate.getStartDate())
                 && StringUtils.isEmpty(dateDate.getEndDate())) {
             if (dateInfo.getDateMode().equals(DateConf.DateMode.LIST)) {
-                return dateModeUtils.listDateStr(dateDate, dateInfo);
+                return dateModeUtils.listDateStr(dateInfo);
             }
             if (dateInfo.getDateMode().equals(DateConf.DateMode.BETWEEN)) {
-                return dateModeUtils.betweenDateStr(dateDate, dateInfo);
+                return dateModeUtils.betweenDateStr(dateInfo);
             }
             if (dateModeUtils.hasAvailableDataMode(dateInfo)) {
                 return dateModeUtils.hasDataModeStr(dateDate, dateInfo);
@@ -270,13 +270,13 @@ public class SqlGenerateUtils {
     }
 
     public String generateDerivedMetric(final List<MetricSchemaResp> metricResps, final Set<String> allFields,
-            final Map<String, Measure> allMeasures,
+                                        final Map<String, Measure> allMeasures,
-            final List<DimSchemaResp> dimensionResps,
+                                        final List<DimSchemaResp> dimensionResps,
-            final String expression, final MetricDefineType metricDefineType,
+                                        final String expression, final MetricDefineType metricDefineType,
-            AggOption aggOption,
+                                        AggOption aggOption,
-            Set<String> visitedMetric,
+                                        Set<String> visitedMetric,
-            Set<String> measures,
+                                        Set<String> measures,
-            Set<String> dimensions) {
+                                        Set<String> dimensions) {
         Set<String> fields = SqlSelectHelper.getColumnFromExpr(expression);
         if (!CollectionUtils.isEmpty(fields)) {
             Map<String, String> replace = new HashMap<>();
@@ -331,8 +331,8 @@ public class SqlGenerateUtils {
         if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) {
             return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
                     : AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " "
-                            + measure.getBizName()
+                    + measure.getBizName()
-                            + " ) ";
+                    + " ) ";
         }
         return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
                 : measure.getAgg() + " ( " + measure.getBizName() + " ) ";

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java

@@ -17,6 +17,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
+
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
@@ -63,6 +64,9 @@ public class MetricDrillDownChecker {
             List<MetricResp> metricResps = getMetrics(metricFields, semanticSchemaResp);
             if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) {
                 DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName);
+                if (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime()) {
+                    continue;
+                }
                 String errMsg = String.format("维度:%s, 不在当前查询指标的下钻维度配置中, 请检查", dimSchemaResp.getName());
                 throw new InvalidArgumentException(errMsg);
             }