(opt)(headless)Fetching dimension values should use partition time field to do filtering.

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java

@@ -33,6 +33,7 @@ import java.util.Objects;
 import java.util.PriorityQueue;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.PriorityBlockingQueue;
 
 import static com.hankcs.hanlp.utility.Predefine.logger;
 
@@ -40,7 +41,7 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
 
     public static int MAX_SIZE = 10;
     public static Boolean removeDuplicates = true;
-    public static ConcurrentHashMap<String, PriorityQueue<Term>> NATURE_TO_VALUES =
+    public static ConcurrentHashMap<String, PriorityBlockingQueue<Term>> NATURE_TO_VALUES =
             new ConcurrentHashMap<>();
     private static boolean addToSuggesterTrie = true;
 
@@ -146,9 +147,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
                 }
                 for (int i = 0; i < attribute.nature.length; i++) {
                     Nature nature = attribute.nature[i];
-                    PriorityQueue<Term> priorityQueue = NATURE_TO_VALUES.get(nature.toString());
+                    PriorityBlockingQueue<Term> priorityQueue =
+                            NATURE_TO_VALUES.get(nature.toString());
                     if (Objects.isNull(priorityQueue)) {
-                        priorityQueue = new PriorityQueue<>(MAX_SIZE,
+                        priorityQueue = new PriorityBlockingQueue<>(MAX_SIZE,
                                 Comparator.comparingInt(Term::getFrequency).reversed());
                         NATURE_TO_VALUES.put(nature.toString(), priorityQueue);
                     }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java

@@ -24,6 +24,7 @@ import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.PriorityBlockingQueue;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -200,7 +201,7 @@ public class SearchService {
     public static List<String> getDimensionValue(DimensionValueReq dimensionValueReq) {
         String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId()
                 + DictWordType.NATURE_SPILT + dimensionValueReq.getElementID();
-        PriorityQueue<Term> terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature);
+        PriorityBlockingQueue<Term> terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature);
         if (CollectionUtils.isEmpty(terms)) {
             return new ArrayList<>();
         }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java

@@ -144,7 +144,8 @@ public class PromptHelper {
                 dimensionStr.append(" ALIAS '").append(alias).append("'");
             }
             if (Objects.nonNull(dimension.getExtInfo().get(DIMENSION_DATA_TYPE))) {
-                dimensionStr.append(" DATATYPE '").append(dimension.getExtInfo().get(DIMENSION_DATA_TYPE)).append("'");
+                dimensionStr.append(" DATATYPE '")
+                        .append(dimension.getExtInfo().get(DIMENSION_DATA_TYPE)).append("'");
             }
             if (StringUtils.isNotEmpty(dimension.getTimeFormat())) {
                 dimensionStr.append(" FORMAT '").append(dimension.getTimeFormat()).append("'");

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java

@@ -47,7 +47,8 @@ public class SqlQueryParser implements QueryParser {
         SqlQuery sqlQuery = queryStatement.getSqlQuery();
         List<String> queryFields = SqlSelectHelper.getAllSelectFields(sqlQuery.getSql());
         Set<String> queryAliases = SqlSelectHelper.getAliasFields(sqlQuery.getSql());
-        List<Pair<String, String>> ontologyMetricsDimensionsAndBizName = Collections.synchronizedList(new ArrayList<>());
+        List<Pair<String, String>> ontologyMetricsDimensionsAndBizName =
+                Collections.synchronizedList(new ArrayList<>());
         queryFields.removeAll(queryAliases);
         Ontology ontology = queryStatement.getOntology();
         OntologyQuery ontologyQuery = buildOntologyQuery(ontology, queryFields);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java

@@ -519,7 +519,7 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
 
     private DataItem getDataItem(DimensionDO dimensionDO) {
         ModelResp modelResp = modelService.getModel(dimensionDO.getModelId());
-        if(modelResp == null) {
+        if (modelResp == null) {
             return null;
         }
         DimensionResp dimensionResp = DimensionConverter.convert2DimensionResp(dimensionDO,
@@ -530,10 +530,8 @@ public class DimensionServiceImpl extends ServiceImpl<DimensionDOMapper, Dimensi
     }
 
     private DataEvent getDataEvent(List<DimensionDO> dimensionDOS, EventType eventType) {
-        List<DataItem> dataItems =
+        List<DataItem> dataItems = dimensionDOS.stream().map(this::getDataItem)
-                dimensionDOS.stream().map(this::getDataItem)
+                .filter(Objects::nonNull).collect(Collectors.toList());
-                        .filter(Objects::nonNull)
-                        .collect(Collectors.toList());
         return new DataEvent(this, dataItems, eventType);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java

@@ -682,10 +682,8 @@ public class MetricServiceImpl extends ServiceImpl<MetricDOMapper, MetricDO>
     }
 
     private DataEvent getDataEvent(List<MetricDO> metricDOS, EventType eventType) {
-        List<DataItem> dataItems =
+        List<DataItem> dataItems = metricDOS.stream().map(this::getDataItem)
-                metricDOS.stream().map(this::getDataItem)
+                .filter(Objects::nonNull).collect(Collectors.toList());
-                        .filter(Objects::nonNull)
-                        .collect(Collectors.toList());
         return new DataEvent(this, dataItems, eventType);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java

@@ -96,8 +96,9 @@ public class DataSetSchemaBuilder {
         Set<SchemaElement> dimensions = new HashSet<>();
         Map<Long, Map<String, String>> dataTypeMap = Maps.newHashMap();
         for (ModelResp modelResp : resp.getModelResps()) {
-            dataTypeMap.put(modelResp.getId(), modelResp.getModelDetail().getFields().stream()
+            dataTypeMap.put(modelResp.getId(),
-                    .collect(Collectors.toMap(Field::getFieldName, Field::getDataType, (k1, k2) -> k2)));
+                    modelResp.getModelDetail().getFields().stream().collect(Collectors
+                            .toMap(Field::getFieldName, Field::getDataType, (k1, k2) -> k2)));
         }
 
         for (DimSchemaResp dim : resp.getDimensions()) {
@@ -119,7 +120,8 @@ public class DataSetSchemaBuilder {
             dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TYPE, dim.getType());
             // data type
             if (dim.getDataType() != null) {
-                dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE, dim.getDataType());
+                dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE,
+                        dim.getDataType());
             } else {
                 dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE,
                         dataTypeMap.get(dim.getModelId()).get(dim.getBizName()));

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -422,33 +422,25 @@ public class DictUtils {
         return joiner.toString();
     }
 
-    public String defaultDateFilter(DateConf dateConf) {
-        String format = itemValueDateFormat;
-        String start = LocalDate.now().minusDays(itemValueDateStart)
-                .format(DateTimeFormatter.ofPattern(format));
-        String end = LocalDate.now().minusDays(itemValueDateEnd)
-                .format(DateTimeFormatter.ofPattern(format));
-        if (Objects.nonNull(dateConf)) {
-            return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start,
-                    dateConf.getDateField(), end);
-        } else {
-            return String.format("( %s >= '%s' and %s <= '%s' )", "dt", start, "dt", end);
-        }
-    }
-
     private String generateDictDateFilter(DictItemResp dictItemResp) {
-        ItemValueConfig config = dictItemResp.getConfig();
+        Dimension partitionTimeDimension = getPartitionTimeDimension(dictItemResp.getModelId());
-        if (config == null) {
+        // 如果没有设置数据时间维度，则无法做时间分区过滤
+        if (partitionTimeDimension == null) {
             return "";
         }
 
-        if (!partitionedModel(dictItemResp.getModelId())) {
+        ItemValueConfig config = dictItemResp.getConfig();
-            return "";
+        // 默认使用数据时间维度进行时间分区过滤
-        }
-        // 未进行设置
         if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) {
-            return defaultDateFilter(null);
+            String startDate = LocalDate.now().minusDays(itemValueDateStart)
+                    .format(DateTimeFormatter.ofPattern(partitionTimeDimension.getDateFormat()));
+            String endDate = LocalDate.now().minusDays(itemValueDateEnd)
+                    .format(DateTimeFormatter.ofPattern(partitionTimeDimension.getDateFormat()));
+            return String.format("( %s >= '%s' and %s <= '%s' )",
+                    partitionTimeDimension.getBizName(), startDate,
+                    partitionTimeDimension.getBizName(), endDate);
         }
+
         // 全表扫描
         if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) {
             return "";
@@ -467,15 +459,15 @@ public class DictUtils {
         return "";
     }
 
-    private boolean partitionedModel(Long modelId) {
+    private Dimension getPartitionTimeDimension(Long modelId) {
         ModelResp model = modelService.getModel(modelId);
         if (Objects.nonNull(model)) {
             List<Dimension> timeDims = model.getTimeDimension();
             if (!CollectionUtils.isEmpty(timeDims)) {
-                return true;
+                return timeDims.get(0);
             }
         }
-        return false;
+        return null;
     }
 
     private String generateDictDateFilterRecent(DictItemResp dictItemResp) {