topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless] Handle compatibility issues when loading all files under the window directory (#1776)

Browse Source
This commit is contained in:
lexluo09
2024-10-10 20:53:55 +08:00
committed by GitHub
parent 8c4db38e5f
commit 1b53226645
2 changed files with 13 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/FileHelper.java
View File

@@ -6,6 +6,8 @@ import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
@@ -45,7 +47,8 @@ public class FileHelper {
}
private static String getCustomPath(String[] path) {
return path[0].substring(0, path[0].lastIndexOf(FILE_SPILT)) + FILE_SPILT;
Path firstPath = Paths.get(path[0]).normalize();
return firstPath.getParent().toString() + File.separator;
}
/**

14
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java
View File

@@ -11,6 +11,7 @@ import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
@@ -59,11 +60,14 @@ public class HdfsFileHelper {
String[] path = Config.CustomDictionaryPath;
FileSystem fs = FileSystem.get(URI.create(path[0]), new Configuration());
String cacheFilePath = path[0] + Predefine.BIN_EXT;
int customBase = cacheFilePath.lastIndexOf(FileHelper.FILE_SPILT);
String customPath =
cacheFilePath.substring(0, customBase) + FileHelper.FILE_SPILT + "*.txt";
log.info("customPath:{}", customPath);
List<String> fileList = getFileList(fs, new Path(customPath));
java.nio.file.Path normalizedPath = Paths.get(cacheFilePath).normalize();
int customBase = normalizedPath.toString().lastIndexOf(FileHelper.FILE_SPILT);
String customPathStr = normalizedPath.toString().substring(0, customBase)
+ FileHelper.FILE_SPILT + "*.txt";
log.info("customPath:{}", customPathStr);
List<String> fileList = getFileList(fs, new Path(customPathStr));
log.info("CustomDictionaryPath:{}", fileList);
Config.CustomDictionaryPath = fileList.toArray(new String[0]);
customDictionary.path =