From ee6b8a1044668d2809e8432720a822925c6d4fcf Mon Sep 17 00:00:00 2001 From: daikon <1059907724@qq.com> Date: Tue, 8 Oct 2024 00:10:34 +0800 Subject: [PATCH] (improvement)(knowledge) add keyValue parameter for queryDictValue (#1749) --- .../api/pojo/request/DictValueReq.java | 2 + .../api/pojo/response/DictValueDimResp.java | 21 ++++ .../chat/knowledge/file/FileHandlerImpl.java | 43 +++++++- .../server/rest/KnowledgeController.java | 8 +- .../server/service/DictTaskService.java | 4 +- .../service/impl/DictTaskServiceImpl.java | 103 +++++++++++++++++- .../server/service/impl/DictWordService.java | 10 ++ .../headless/server/utils/DictUtils.java | 4 + 8 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java index dca5e5870..dff926202 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictValueReq.java @@ -13,4 +13,6 @@ public class DictValueReq extends PageBaseReq { private Long itemId; private TypeEnums type = TypeEnums.DIMENSION; + + private String keyValue; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java new file mode 100644 index 000000000..f2cb1ce11 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DictValueDimResp.java @@ -0,0 +1,21 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import lombok.Data; +import lombok.ToString; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author: kanedai + * @date: 2024/9/29 + */ +@Data +@ToString +public class DictValueDimResp extends DictValueResp { + /** dimension value for result show */ + private String bizName; + + /** dimension value for user query */ + private List alias = new ArrayList<>(); +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java index 7ea1212bd..417cb6b0f 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java @@ -77,10 +77,51 @@ public class FileHandlerImpl implements FileHandler { @Override public PageInfo queryDictValue(String fileName, DictValueReq dictValueReq) { + if (StringUtils.isEmpty(dictValueReq.getKeyValue())) { + return getDictValueRespPagWithoutKey(fileName, dictValueReq); + } + return getDictValueRespPagWithKey(fileName, dictValueReq); + } + + private PageInfo getDictValueRespPagWithKey(String fileName, + DictValueReq dictValueReq) { + PageInfo dictValueRespPageInfo = new PageInfo<>(); + dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize()); + dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent()); + String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; + Long fileLineNum = getFileLineNum(filePath); + Integer startLine = 1; + List dictValueRespList = + getFileData(filePath, startLine, fileLineNum.intValue()); + dictValueRespList = dictValueRespList.stream() + .filter(dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue())) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(dictValueRespList)) { + dictValueRespPageInfo.setList(new ArrayList<>()); + return dictValueRespPageInfo; + } + + Integer startIndex = + Math.max((dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize(), 0); + Integer endIndex = + Integer.valueOf(Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), + dictValueRespList.size()) + ""); + List list = dictValueRespList.subList(startIndex, endIndex); + dictValueRespPageInfo.setPageSize(dictValueReq.getPageSize()); + dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent()); + dictValueRespPageInfo.setTotal(dictValueRespList.size()); + dictValueRespPageInfo.setList(list); + dictValueRespPageInfo.setHasNextPage(endIndex >= dictValueRespList.size() ? false : true); + dictValueRespPageInfo.setHasPreviousPage(startLine <= 0 ? false : true); + return dictValueRespPageInfo; + } + + private PageInfo getDictValueRespPagWithoutKey(String fileName, + DictValueReq dictValueReq) { PageInfo dictValueRespPageInfo = new PageInfo<>(); String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName; Long fileLineNum = getFileLineNum(filePath); - Integer startLine = (dictValueReq.getCurrent() - 1) * dictValueReq.getPageSize() + 1; + Integer startLine = 1; Integer endLine = Integer.valueOf( Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + ""); List dictValueRespList = getFileData(filePath, startLine, endLine); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java index 696f2ebcb..50c3dafd1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java @@ -15,7 +15,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; -import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; +import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp; import com.tencent.supersonic.headless.server.service.DictConfService; import com.tencent.supersonic.headless.server.service.DictTaskService; import com.tencent.supersonic.headless.server.task.DictionaryReloadTask; @@ -112,7 +112,9 @@ public class KnowledgeController { return taskService.deleteDictTask(taskReq, user); } - /** dailyDictTask-手动离线更新所有字典 */ + /** + * dailyDictTask-手动离线更新所有字典 + */ @PutMapping("/task/all") public Boolean dailyDictTask() { return taskService.dailyDictTask(); @@ -155,7 +157,7 @@ public class KnowledgeController { * @param dictValueReq */ @PostMapping("/dict/data") - public PageInfo queryDictValue(@RequestBody @Valid DictValueReq dictValueReq, + public PageInfo queryDictValue(@RequestBody @Valid DictValueReq dictValueReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); return taskService.queryDictValue(dictValueReq, user); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java index 3f306cbd3..0b72edfa3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictTaskService.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; -import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; +import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp; /** Manage dictionary tasks */ public interface DictTaskService { @@ -17,7 +17,7 @@ public interface DictTaskService { DictTaskResp queryLatestDictTask(DictSingleTaskReq taskReq, User user); - PageInfo queryDictValue(DictValueReq dictValueReq, User user); + PageInfo queryDictValue(DictValueReq dictValueReq, User user); String queryDictFilePath(DictValueReq dictValueReq, User user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java index 599560b06..43c50321a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -5,25 +5,36 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.util.BeanMapper; +import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; +import com.tencent.supersonic.headless.api.pojo.response.DictValueDimResp; import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; +import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; +import com.tencent.supersonic.headless.chat.knowledge.DictWord; import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler; import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; import com.tencent.supersonic.headless.server.service.DictTaskService; +import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.utils.DictUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; @Service @Slf4j @@ -45,14 +56,17 @@ public class DictTaskServiceImpl implements DictTaskService { private final DictUtils dictUtils; private final FileHandler fileHandler; private final DictWordService dictWordService; + private final DimensionService dimensionService; public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter, - DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService) { + DictUtils dictUtils, FileHandler fileHandler, DictWordService dictWordService, + DimensionService dimensionService) { this.dictRepository = dictRepository; this.dictConverter = dictConverter; this.dictUtils = dictUtils; this.fileHandler = fileHandler; this.dictWordService = dictWordService; + this.dimensionService = dimensionService; } @Override @@ -151,13 +165,96 @@ public class DictTaskServiceImpl implements DictTaskService { } @Override - public PageInfo queryDictValue(DictValueReq dictValueReq, User user) { + public PageInfo queryDictValue(DictValueReq dictValueReq, User user) { + // todo 优化读取内存结构 + // return getDictValuePageFromMemory(dictValueReq); + return getDictValuePageFromFile(dictValueReq); + } + + private PageInfo getDictValuePageFromFile(DictValueReq dictValueReq) { String fileName = String.format("dic_value_%d_%s_%s", dictValueReq.getModelId(), dictValueReq.getType().name(), dictValueReq.getItemId()) + Constants.DOT + dictFileType; PageInfo dictValueRespList = fileHandler.queryDictValue(fileName, dictValueReq); - return dictValueRespList; + PageInfo result = convert2DictValueDimRespPage(dictValueRespList); + return result; + } + + private PageInfo convert2DictValueDimRespPage( + PageInfo dictValueRespPage) { + PageInfo result = new PageInfo<>(); + BeanMapper.mapper(dictValueRespPage, result); + if (CollectionUtils.isEmpty(dictValueRespPage.getList())) { + return result; + } + + List list = getDictValueDimRespList(dictValueRespPage.getList()); + result.setList(list); + return result; + } + + private List getDictValueDimRespList(List dictValueRespList) { + List list = + dictValueRespList.stream().map(dictValue -> convert2DictValueInternal(dictValue)) + .collect(Collectors.toList()); + return list; + } + + private List getDictValueDimRespList(List dictWords, Long dimId) { + DimensionResp dimResp = dimensionService.getDimension(dimId); + List list = + dictWords.stream().map(dictWord -> convert2DictValueInternal(dictWord, dimResp)) + .collect(Collectors.toList()); + return list; + } + + private DictValueDimResp convert2DictValueInternal(DictWord dictWord, DimensionResp dimResp) { + DictValueDimResp dictValueDimResp = new DictValueDimResp(); + BeanMapper.mapper(dictWord, dictValueDimResp); + if (Objects.nonNull(dimResp.getDimValueMaps())) { + Map techAndAliasMap = dimResp.getDimValueMaps().stream().collect( + Collectors.toMap(dimValue -> dimValue.getTechName(), v -> v, (v1, v2) -> v2)); + if (techAndAliasMap.containsKey(dictWord.getWord())) { + DimValueMap dimValueMap = techAndAliasMap.get(dictWord.getWord()); + BeanMapper.mapper(dimValueMap, dictValueDimResp); + } + } + return dictValueDimResp; + } + + private DictValueDimResp convert2DictValueInternal(DictValueResp dictValue) { + DictValueDimResp dictValueDimResp = new DictValueDimResp(); + BeanMapper.mapper(dictValue, dictValueDimResp); + return dictValueDimResp; + } + + private PageInfo getDictValuePageFromMemory(DictValueReq dictValueReq) { + PageInfo dictValueRespPageInfo = new PageInfo<>(); + Set dimSet = new HashSet<>(); + dimSet.add(dictValueReq.getItemId()); + List dimDictWords = dictWordService.getDimDictWords(dimSet); + if (CollectionUtils.isEmpty(dimDictWords)) { + return dictValueRespPageInfo; + } + if (StringUtils.isNotEmpty(dictValueReq.getKeyValue())) { + dimDictWords = dimDictWords.stream() + .filter(dimValue -> dimValue.getWord().contains(dictValueReq.getKeyValue())) + .collect(Collectors.toList()); + } + + Integer pageSize = dictValueReq.getPageSize(); + Integer current = dictValueReq.getCurrent(); + dictValueRespPageInfo.setTotal(dimDictWords.size()); + dictValueRespPageInfo.setPageSize(pageSize); + dictValueRespPageInfo.setPageNum(dictValueReq.getCurrent()); + + // 分页 + int startIndex = (current - 1) * pageSize; + int endIndex = Math.min(startIndex + pageSize, dimDictWords.size()); + List data = dimDictWords.subList(startIndex, endIndex); + dictValueRespPageInfo.setList(getDictValueDimRespList(data, dictValueReq.getItemId())); + return dictValueRespPageInfo; } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java index 624a44bc9..a7d73fade 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictWordService.java @@ -15,6 +15,7 @@ import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -63,6 +64,15 @@ public class DictWordService { return words; } + public List getDimDictWords(Set dimIds) { + SemanticSchema semanticSchema = schemaService.getSemanticSchema(); + List requiredDims = semanticSchema.getDimensionValues().stream() + .filter(dim -> dimIds.contains(dim.getId())).collect(Collectors.toList()); + List words = new ArrayList<>(); + addWordsByType(DictWordType.VALUE, requiredDims, words); + return words; + } + private void addWordsByType(DictWordType value, List metas, List natures) { metas = distinct(metas); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index ec9abcdc2..601302f7d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -320,6 +320,10 @@ public class DictUtils { long limit = (Objects.isNull(config) || Objects.isNull(config.getLimit())) ? itemValueMaxCount : dictItemResp.getConfig().getLimit(); + if (limit <= 0) { + limit = Integer.MAX_VALUE; + } + String sql = String.format(sqlPattern, bizName, where, bizName, limit); Set modelIds = new HashSet<>(); modelIds.add(dictItemResp.getModelId());