From fe3f4c36b5ce61084091e75b05d5501184886ba6 Mon Sep 17 00:00:00 2001
From: daikon <1059907724@qq.com>
Date: Thu, 10 Oct 2024 14:29:05 +0800
Subject: [PATCH] Opt knowledge data (#1769)

---
 .../chat/knowledge/file/FileHandlerImpl.java       | 14 +++++++++-----
 .../headless/server/utils/DictUtils.java           | 14 ++++++++++++++
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
index 417cb6b0f..f92afc531 100644
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
@@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
@@ -26,6 +27,10 @@ import java.util.stream.Stream;
 @Slf4j
 @Component
 public class FileHandlerImpl implements FileHandler {
+
+    @Value("${dict.value.max.count.page:1000}")
+    private int dictValueMaxCountPage;
+
     public static final String FILE_SPILT = File.separator;
 
     private final LocalFileConfig localFileConfig;
@@ -92,10 +97,9 @@ public class FileHandlerImpl implements FileHandler {
         Long fileLineNum = getFileLineNum(filePath);
         Integer startLine = 1;
         List<DictValueResp> dictValueRespList =
-                getFileData(filePath, startLine, fileLineNum.intValue());
-        dictValueRespList = dictValueRespList.stream()
-                .filter(dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue()))
-                .collect(Collectors.toList());
+                getFileData(filePath, startLine, fileLineNum.intValue()).stream().filter(
+                        dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue()))
+                        .collect(Collectors.toList());
         if (CollectionUtils.isEmpty(dictValueRespList)) {
             dictValueRespPageInfo.setList(new ArrayList<>());
             return dictValueRespPageInfo;
@@ -120,7 +124,7 @@ public class FileHandlerImpl implements FileHandler {
             DictValueReq dictValueReq) {
         PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
         String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
-        Long fileLineNum = getFileLineNum(filePath);
+        Long fileLineNum = Math.min(dictValueMaxCountPage, getFileLineNum(filePath));
         Integer startLine = 1;
         Integer endLine = Integer.valueOf(
                 Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + "");
diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
index c3f68c052..5dad427c7 100644
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
@@ -467,6 +467,9 @@ public class DictUtils {
 
     private String generateDictDateFilter(DictItemResp dictItemResp) {
         ItemValueConfig config = dictItemResp.getConfig();
+        if (!partitionedModel(dictItemResp.getModelId())) {
+            return "";
+        }
         // 未进行设置
         if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) {
             return defaultDateFilter();
@@ -489,6 +492,17 @@ public class DictUtils {
         return "";
     }
 
+    private boolean partitionedModel(Long modelId) {
+        ModelResp model = modelService.getModel(modelId);
+        if (Objects.nonNull(model)) {
+            List<Dim> timeDims = model.getTimeDimension();
+            if (!CollectionUtils.isEmpty(timeDims)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private String generateDictDateFilterRecent(DictItemResp dictItemResp) {
         ModelResp model = modelService.getModel(dictItemResp.getModelId());
         if (Objects.nonNull(model)) {