From 2ae94fb38c0e54462d09d5f27c5dd223c6d8e83d Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Sun, 23 Jun 2024 11:56:26 +0800 Subject: [PATCH] (improvement)(dict) support queryDictValue (#1196) --- .../api/pojo/request/DictValueReq.java | 16 ++++ .../api/pojo/response/DictValueResp.java | 18 +++++ .../chat/knowledge/file/FileHandler.java | 7 ++ .../chat/knowledge/file/FileHandlerImpl.java | 79 +++++++++++++++++++ .../server/rest/KnowledgeController.java | 30 +++++++ .../server/service/DictTaskService.java | 7 ++ .../service/impl/DictTaskServiceImpl.java | 20 +++++ 7 files changed, 177 insertions(+) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java new file mode 100644 index 000000000..ec47dd586 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.tencent.supersonic.common.pojo.PageBaseReq; +import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class DictValueReq extends PageBaseReq { + private Long modelId; + + private Long itemId; + + private TypeEnums type = TypeEnums.DIMENSION; +} \ No newline at end of file diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java new file mode 100644 index 000000000..e7f55d6cc --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueResp.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import lombok.Data; +import lombok.ToString; + +/** + * @author: kanedai + * @date: 2024/6/22 + */ +@ToString +@Data +public class DictValueResp { + private String value; + + private String nature; + + private Long frequency; +} \ No newline at end of file diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java index bd2d71840..fa0cd70b0 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandler.java @@ -1,5 +1,9 @@ package com.tencent.supersonic.headless.chat.knowledge.file; +import com.github.pagehelper.PageInfo; +import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; +import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; + import java.util.List; public interface FileHandler { @@ -54,4 +58,7 @@ public interface FileHandler { */ void deleteFile(String fileName); + PageInfo queryDictValue(String fileName, DictValueReq dictValueReq); + + String queryDictFilePath(String fileName); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java index 8f7640c44..56807f933 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java @@ -1,6 +1,10 @@ package com.tencent.supersonic.headless.chat.knowledge.file; +import com.github.pagehelper.PageInfo; +import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; +import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.util.Strings; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -14,7 +18,11 @@ import java.nio.file.StandardOpenOption; import java.io.BufferedWriter; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Component @@ -22,6 +30,7 @@ public class FileHandlerImpl implements FileHandler { public static final String FILE_SPILT = File.separator; private final LocalFileConfig localFileConfig; + public FileHandlerImpl(LocalFileConfig localFileConfig) { this.localFileConfig = localFileConfig; } @@ -68,6 +77,76 @@ public class FileHandlerImpl implements FileHandler { } } + @Override + public PageInfo queryDictValue(String fileName, DictValueReq dictValueReq) { + PageInfo dictValueRespPageInfo = new PageInfo<>(); + String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; + Long fileLineNum = getFileLineNum(filePath); + Integer startLine = (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1; + Integer endLine = Integer.valueOf( + Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + ""); + List dictValueRespList = getFileData(filePath, startLine, endLine); + + dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize()); + dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent()); + dictValueRespPageInfo.setTotal(fileLineNum); + dictValueRespPageInfo.setList(dictValueRespList); + dictValueRespPageInfo.setHasNextPage(endLine >= fileLineNum ? false : true); + dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true); + return dictValueRespPageInfo; + } + + @Override + public String queryDictFilePath(String fileName) { + String path = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; + if (existPath(path)) { + return path; + } + log.info("dict file:{} is not exist", path); + return null; + } + + private List getFileData(String filePath, Integer startLine, Integer endLine) { + List fileData = new ArrayList<>(); + + try (Stream lines = Files.lines(Paths.get(filePath))) { + fileData = lines + .skip(startLine - 1) + .limit(endLine - startLine + 1) + .map(lineStr -> convert2Resp(lineStr)) + .filter(line -> Objects.nonNull(line)) + .collect(Collectors.toList()); + } catch (IOException e) { + log.warn("[getFileData] e:{}", e); + } + return fileData; + + } + + private DictValueResp convert2Resp(String lineStr) { + DictValueResp dictValueResp = new DictValueResp(); + if (Strings.isNotEmpty(lineStr)) { + String[] itemArray = lineStr.split("\\s+"); + if (Objects.nonNull(itemArray) && itemArray.length >= 3) { + dictValueResp.setValue(itemArray[0].replace("#", " ")); + dictValueResp.setNature(itemArray[1]); + dictValueResp.setFrequency(Long.parseLong(itemArray[2])); + } + } + return dictValueResp; + } + + private Long getFileLineNum(String filePath) { + try (Stream lines = Files.lines(Paths.get(filePath))) { + Long lineCount = lines + .count(); + return lineCount; + } catch (IOException e) { + e.printStackTrace(); + } + return 0L; + } + @Override public Boolean existPath(String pathStr) { Path path = Paths.get(pathStr); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java index 01ea4a6c2..83a112b75 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java @@ -1,13 +1,16 @@ package com.tencent.supersonic.headless.server.rest; +import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; import com.tencent.supersonic.headless.server.schedule.EmbeddingTask; import com.tencent.supersonic.headless.server.service.DictConfService; import com.tencent.supersonic.headless.server.service.DictTaskService; @@ -137,4 +140,31 @@ public class KnowledgeController { embeddingTask.reloadMetaEmbedding(); return true; } + + /** + * queryDictValue-返回字典的数据 + * + * @param dictValueReq + */ + @PostMapping("/dict/data") + public PageInfo queryDictValue(@RequestBody @Valid DictValueReq dictValueReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return taskService.queryDictValue(dictValueReq, user); + } + + /** + * queryDictFilePath-返回字典文件的路径 + * + * @param dictValueReq + */ + @PostMapping("/dict/file") + public String queryDictFilePath(@RequestBody @Valid DictValueReq dictValueReq, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return taskService.queryDictFilePath(dictValueReq, user); + } + } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java index e7e7c65e5..f62f740c2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java @@ -1,8 +1,11 @@ package com.tencent.supersonic.headless.server.service; +import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; /** * Manage dictionary tasks @@ -15,4 +18,8 @@ public interface DictTaskService { Boolean dailyDictTask(); DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq, User user); + + PageInfo queryDictValue(DictValueReq dictValueReq, User user); + + String queryDictFilePath(DictValueReq dictValueReq, User user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java index 92d0eac77..6d9237a55 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -1,13 +1,16 @@ package com.tencent.supersonic.headless.server.service.impl; +import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; +import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService; import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler; import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; @@ -148,4 +151,21 @@ public class DictTaskServiceImpl implements DictTaskService { return dictRepository.queryLatestDictTask(taskReq); } + @Override + public PageInfo queryDictValue(DictValueReq dictValueReq, User user) { + String fileName = String.format("dic_value_%d_%s_%s", + dictValueReq.getModelId(), dictValueReq.getType().name(), dictValueReq.getItemId()) + + Constants.DOT + dictFileType; + PageInfo dictValueRespList = fileHandler.queryDictValue(fileName, dictValueReq); + return dictValueRespList; + } + + @Override + public String queryDictFilePath(DictValueReq dictValueReq, User user) { + String fileName = String.format("dic_value_%d_%s_%s", + dictValueReq.getModelId(), dictValueReq.getType().name(), dictValueReq.getItemId()) + + Constants.DOT + dictFileType; + return fileHandler.queryDictFilePath(fileName); + } + } \ No newline at end of file