From 1b53226645f3b421b821694de1d92acb9132e411 Mon Sep 17 00:00:00 2001
From: lexluo09 <39718951+lexluo09@users.noreply.github.com>
Date: Thu, 10 Oct 2024 20:53:55 +0800
Subject: [PATCH] [improvement][headless] Handle compatibility issues when
 loading all files under the window directory (#1776)

---
 .../headless/chat/knowledge/helper/FileHelper.java |  5 ++++-
 .../chat/knowledge/helper/HdfsFileHelper.java      | 14 +++++++++-----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java
index b42669056..3af9fb836 100644
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java
@@ -6,6 +6,8 @@ import lombok.extern.slf4j.Slf4j;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -45,7 +47,8 @@ public class FileHelper {
     }
 
     private static String getCustomPath(String[] path) {
-        return path[0].substring(0, path[0].lastIndexOf(FILE_SPILT)) + FILE_SPILT;
+        Path firstPath = Paths.get(path[0]).normalize();
+        return firstPath.getParent().toString() + File.separator;
     }
 
     /**
diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java
index 0e6d7bad8..15ea42e36 100644
--- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java
+++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java
@@ -11,6 +11,7 @@ import org.apache.hadoop.fs.Path;
 
 import java.io.IOException;
 import java.net.URI;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -59,11 +60,14 @@ public class HdfsFileHelper {
         String[] path = Config.CustomDictionaryPath;
         FileSystem fs = FileSystem.get(URI.create(path[0]), new Configuration());
         String cacheFilePath = path[0] + Predefine.BIN_EXT;
-        int customBase = cacheFilePath.lastIndexOf(FileHelper.FILE_SPILT);
-        String customPath =
-                cacheFilePath.substring(0, customBase) + FileHelper.FILE_SPILT + "*.txt";
-        log.info("customPath:{}", customPath);
-        List<String> fileList = getFileList(fs, new Path(customPath));
+
+        java.nio.file.Path normalizedPath = Paths.get(cacheFilePath).normalize();
+        int customBase = normalizedPath.toString().lastIndexOf(FileHelper.FILE_SPILT);
+        String customPathStr = normalizedPath.toString().substring(0, customBase)
+                + FileHelper.FILE_SPILT + "*.txt";
+
+        log.info("customPath:{}", customPathStr);
+        List<String> fileList = getFileList(fs, new Path(customPathStr));
         log.info("CustomDictionaryPath:{}", fileList);
         Config.CustomDictionaryPath = fileList.toArray(new String[0]);
         customDictionary.path =