topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][chat] Fix the issue of HDFS path concatenation (#1820)

Browse Source
This commit is contained in:
lexluo09
2024-10-17 14:12:12 +08:00
committed by GitHub
parent c8d2d75ad5
commit ee4b218a1e
3 changed files with 10 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java
View File

@@ -11,7 +11,6 @@ import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
@@ -61,13 +60,11 @@ public class HdfsFileHelper {
FileSystem fs = FileSystem.get(URI.create(path[0]), new Configuration());
String cacheFilePath = path[0] + Predefine.BIN_EXT;
java.nio.file.Path normalizedPath = Paths.get(cacheFilePath).normalize();
int customBase = normalizedPath.toString().lastIndexOf(FileHelper.FILE_SPILT);
String customPathStr = normalizedPath.toString().substring(0, customBase)
+ FileHelper.FILE_SPILT + "*.txt";
log.info("customPath:{}", customPathStr);
List<String> fileList = getFileList(fs, new Path(customPathStr));
Path hdfsPath = new Path(cacheFilePath);
String parentPath = hdfsPath.getParent().toString();
Path customPath = new Path(parentPath, "*.txt");
log.info("customPath:{}", customPath);
List<String> fileList = getFileList(fs, customPath);
log.info("CustomDictionaryPath:{}", fileList);
Config.CustomDictionaryPath = fileList.toArray(new String[0]);
customDictionary.path =

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java
View File

@@ -5,11 +5,12 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
import lombok.extern.slf4j.Slf4j;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class H2Adaptor extends BaseDbAdaptor {