topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 04:27:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][chat] Fix the issue of HDFS path concatenation (#1820)

Browse Source
This commit is contained in:
lexluo09
2024-10-17 14:12:12 +08:00
committed by GitHub
parent c8d2d75ad5
commit ee4b218a1e
3 changed files with 10 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java
View File

@@ -38,8 +38,9 @@ public class MemoryServiceImpl implements MemoryService {
@Override @Override
public void createMemory(ChatMemoryDO memory) { public void createMemory(ChatMemoryDO memory) {
// if an existing enabled memory has the same question, just skip // if an existing enabled memory has the same question, just skip
List<ChatMemoryDO> memories = getMemories(ChatMemoryFilter.builder() List<ChatMemoryDO> memories =
.agentId(memory.getAgentId()).question(memory.getQuestion()).status(MemoryStatus.ENABLED).build()); getMemories(ChatMemoryFilter.builder().agentId(memory.getAgentId())
.question(memory.getQuestion()).status(MemoryStatus.ENABLED).build());
if (memories.size() == 0) { if (memories.size() == 0) {
chatMemoryRepository.createMemory(memory); chatMemoryRepository.createMemory(memory);
} }

13
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HdfsFileHelper.java
View File

@@ -11,7 +11,6 @@ import org.apache.hadoop.fs.Path;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -61,13 +60,11 @@ public class HdfsFileHelper {
FileSystem fs = FileSystem.get(URI.create(path[0]), new Configuration()); FileSystem fs = FileSystem.get(URI.create(path[0]), new Configuration());
String cacheFilePath = path[0] + Predefine.BIN_EXT; String cacheFilePath = path[0] + Predefine.BIN_EXT;
java.nio.file.Path normalizedPath = Paths.get(cacheFilePath).normalize(); Path hdfsPath = new Path(cacheFilePath);
int customBase = normalizedPath.toString().lastIndexOf(FileHelper.FILE_SPILT); String parentPath = hdfsPath.getParent().toString();
String customPathStr = normalizedPath.toString().substring(0, customBase) Path customPath = new Path(parentPath, "*.txt");
+ FileHelper.FILE_SPILT + "*.txt"; log.info("customPath:{}", customPath);
List<String> fileList = getFileList(fs, customPath);
log.info("customPath:{}", customPathStr);
List<String> fileList = getFileList(fs, new Path(customPathStr));
log.info("CustomDictionaryPath:{}", fileList); log.info("CustomDictionaryPath:{}", fileList);
Config.CustomDictionaryPath = fileList.toArray(new String[0]); Config.CustomDictionaryPath = fileList.toArray(new String[0]);
customDictionary.path = customDictionary.path =

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/H2Adaptor.java
View File

@@ -5,11 +5,12 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DBColumn; import com.tencent.supersonic.headless.api.pojo.DBColumn;
import com.tencent.supersonic.headless.core.pojo.ConnectInfo; import com.tencent.supersonic.headless.core.pojo.ConnectInfo;
import lombok.extern.slf4j.Slf4j;
import java.sql.DatabaseMetaData; import java.sql.DatabaseMetaData;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class H2Adaptor extends BaseDbAdaptor { public class H2Adaptor extends BaseDbAdaptor {