topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(dict) support queryDictValue (#1196)

Browse Source
This commit is contained in:
daikon
2024-06-23 11:56:26 +08:00
committed by GitHub
parent a658b9c45f
commit 2ae94fb38c
7 changed files with 177 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java
View File

@@ -1,5 +1,9 @@
package com.tencent.supersonic.headless.chat.knowledge.file;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import java.util.List;
public interface FileHandler {
@@ -54,4 +58,7 @@ public interface FileHandler {
*/
void deleteFile(String fileName);
PageInfo<DictValueResp> queryDictValue(String fileName, DictValueReq dictValueReq);
String queryDictFilePath(String fileName);
}

79
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
View File

@@ -1,6 +1,10 @@
package com.tencent.supersonic.headless.chat.knowledge.file;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -14,7 +18,11 @@ import java.nio.file.StandardOpenOption;
import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Component
@@ -22,6 +30,7 @@ public class FileHandlerImpl implements FileHandler {
public static final String FILE_SPILT = File.separator;
private final LocalFileConfig localFileConfig;
public FileHandlerImpl(LocalFileConfig localFileConfig) {
this.localFileConfig = localFileConfig;
}
@@ -68,6 +77,76 @@ public class FileHandlerImpl implements FileHandler {
}
}
@Override
public PageInfo<DictValueResp> queryDictValue(String fileName, DictValueReq dictValueReq) {
PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
Long fileLineNum = getFileLineNum(filePath);
Integer startLine = (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1;
Integer endLine = Integer.valueOf(
Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + "");
List<DictValueResp> dictValueRespList = getFileData(filePath, startLine, endLine);
dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize());
dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent());
dictValueRespPageInfo.setTotal(fileLineNum);
dictValueRespPageInfo.setList(dictValueRespList);
dictValueRespPageInfo.setHasNextPage(endLine >= fileLineNum ? false : true);
dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true);
return dictValueRespPageInfo;
}
@Override
public String queryDictFilePath(String fileName) {
String path = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
if (existPath(path)) {
return path;
}
log.info("dict file:{} is not exist", path);
return null;
}
private List<DictValueResp> getFileData(String filePath, Integer startLine, Integer endLine) {
List<DictValueResp> fileData = new ArrayList<>();
try (Stream<String> lines = Files.lines(Paths.get(filePath))) {
fileData = lines
.skip(startLine - 1)
.limit(endLine - startLine + 1)
.map(lineStr -> convert2Resp(lineStr))
.filter(line -> Objects.nonNull(line))
.collect(Collectors.toList());
} catch (IOException e) {
log.warn("[getFileData] e:{}", e);
}
return fileData;
}
private DictValueResp convert2Resp(String lineStr) {
DictValueResp dictValueResp = new DictValueResp();
if (Strings.isNotEmpty(lineStr)) {
String[] itemArray = lineStr.split("\\s+");
if (Objects.nonNull(itemArray) && itemArray.length >= 3) {
dictValueResp.setValue(itemArray[0].replace("#", " "));
dictValueResp.setNature(itemArray[1]);
dictValueResp.setFrequency(Long.parseLong(itemArray[2]));
}
}
return dictValueResp;
}
private Long getFileLineNum(String filePath) {
try (Stream<String> lines = Files.lines(Paths.get(filePath))) {
Long lineCount = lines
.count();
return lineCount;
} catch (IOException e) {
e.printStackTrace();
}
return 0L;
}
@Override
public Boolean existPath(String pathStr) {
Path path = Paths.get(pathStr);