diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java index 1f560b335..6bf2ecc98 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java @@ -56,23 +56,25 @@ public class KnowledgeBaseService { return HanlpHelper.getTerms(text, modelIdToDataSetIds); } - public List prefixSearch(String key, int limit, Map> modelIdToDataSetIds, - Set detectDataSetIds) { + public List prefixSearch(String key, int limit, + Map> modelIdToDataSetIds, + Set detectDataSetIds) { return prefixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds); } public List prefixSearchByModel(String key, int limit, - Map> modelIdToDataSetIds, Set detectDataSetIds) { + Map> modelIdToDataSetIds, + Set detectDataSetIds) { return SearchService.prefixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds); } - public List suffixSearch(String key, int limit, Map> modelIdToDataSetIds, - Set detectDataSetIds) { + public List suffixSearch(String key, int limit, Map> modelIdToDataSetIds, Set detectDataSetIds) { return suffixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds); } - public List suffixSearchByModel(String key, int limit, Map> modelIdToDataSetIds, - Set detectDataSetIds) { + public List suffixSearchByModel(String key, int limit, Map> modelIdToDataSetIds, Set detectDataSetIds) { return SearchService.suffixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java index 3af36af5f..a9759e9ee 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java @@ -17,6 +17,12 @@ import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult; import com.tencent.supersonic.headless.chat.knowledge.MapResult; import com.tencent.supersonic.headless.chat.knowledge.MultiCustomDictionary; import com.tencent.supersonic.headless.chat.knowledge.SearchService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ResourceUtils; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -26,11 +32,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; -import org.springframework.util.ResourceUtils; /** * HanLP helper @@ -203,18 +204,20 @@ public class HanlpHelper { if (CollectionUtils.isEmpty(mapResults)) { return; } + List newResults = new ArrayList<>(); + for (T mapResult : mapResults) { - boolean isAdd = false; - if (MultiCustomDictionary.isLowerLetter(mapResult.getName())) { - if (CustomDictionary.contains(mapResult.getName())) { - CoreDictionary.Attribute attribute = CustomDictionary.get(mapResult.getName()); - if (attribute != null) { - isAdd = addLetterOriginal(newResults, mapResult, attribute); - } + String name = mapResult.getName(); + boolean isAdded = false; + if (MultiCustomDictionary.isLowerLetter(name) && CustomDictionary.contains(name)) { + CoreDictionary.Attribute attribute = CustomDictionary.get(name); + if (attribute != null) { + isAdded = addLetterOriginal(newResults, mapResult, attribute); } } - if (!isAdd) { + + if (!isAdded) { newResults.add(mapResult); } } @@ -223,48 +226,54 @@ public class HanlpHelper { } public static boolean addLetterOriginal(List mapResults, T mapResult, - CoreDictionary.Attribute attribute) { + CoreDictionary.Attribute attribute) { + if (attribute == null) { + return false; + } boolean isAdd = false; - if (attribute != null) { - if (mapResult instanceof HanlpMapResult) { - HanlpMapResult hanlpMapResult = (HanlpMapResult) mapResult; - for (String nature : hanlpMapResult.getNatures()) { - String orig = attribute.getOriginal(Nature.fromString(nature)); - if (orig != null) { - MapResult addMapResult = new HanlpMapResult(orig, Arrays.asList(nature), - hanlpMapResult.getDetectWord()); - mapResults.add((T) addMapResult); - isAdd = true; - } - } - } else if (mapResult instanceof DatabaseMapResult) { - List originals = attribute.getOriginals(); - if (!CollectionUtils.isEmpty(originals)) { - for (String orig : originals) { - DatabaseMapResult addMapResult = new DatabaseMapResult(); - addMapResult.setName(orig); - addMapResult.setSchemaElement(((DatabaseMapResult) mapResult).getSchemaElement()); - addMapResult.setDetectWord(mapResult.getDetectWord()); - mapResults.add((T) addMapResult); - isAdd = true; - } - } - } else if (mapResult instanceof EmbeddingResult) { - List originals = attribute.getOriginals(); - if (!CollectionUtils.isEmpty(originals)) { - for (String orig : originals) { - EmbeddingResult addMapResult = new EmbeddingResult(); - addMapResult.setName(orig); - addMapResult.setDetectWord(mapResult.getDetectWord()); - addMapResult.setId(((EmbeddingResult) mapResult).getId()); - addMapResult.setMetadata(((EmbeddingResult) mapResult).getMetadata()); - addMapResult.setDistance(((EmbeddingResult) mapResult).getDistance()); - mapResults.add((T) addMapResult); - isAdd = true; - } + if (mapResult instanceof HanlpMapResult) { + HanlpMapResult hanlpMapResult = (HanlpMapResult) mapResult; + for (String nature : hanlpMapResult.getNatures()) { + String orig = attribute.getOriginal(Nature.fromString(nature)); + if (orig != null) { + MapResult addMapResult = new HanlpMapResult( + orig, Arrays.asList(nature), hanlpMapResult.getDetectWord()); + mapResults.add((T) addMapResult); + isAdd = true; } } + return isAdd; } + + List originals = attribute.getOriginals(); + if (CollectionUtils.isEmpty(originals)) { + return false; + } + + if (mapResult instanceof DatabaseMapResult) { + DatabaseMapResult dbMapResult = (DatabaseMapResult) mapResult; + for (String orig : originals) { + DatabaseMapResult addMapResult = new DatabaseMapResult(); + addMapResult.setName(orig); + addMapResult.setSchemaElement(dbMapResult.getSchemaElement()); + addMapResult.setDetectWord(dbMapResult.getDetectWord()); + mapResults.add((T) addMapResult); + isAdd = true; + } + } else if (mapResult instanceof EmbeddingResult) { + EmbeddingResult embeddingResult = (EmbeddingResult) mapResult; + for (String orig : originals) { + EmbeddingResult addMapResult = new EmbeddingResult(); + addMapResult.setName(orig); + addMapResult.setDetectWord(embeddingResult.getDetectWord()); + addMapResult.setId(embeddingResult.getId()); + addMapResult.setMetadata(embeddingResult.getMetadata()); + addMapResult.setDistance(embeddingResult.getDistance()); + mapResults.add((T) addMapResult); + isAdd = true; + } + } + return isAdd; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java index 6d9237a55..a2e997d44 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -11,9 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; -import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService; import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler; -import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper; import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; import com.tencent.supersonic.headless.server.service.DictTaskService; @@ -24,7 +22,6 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.io.IOException; import java.util.List; import java.util.Objects; @@ -44,16 +41,18 @@ public class DictTaskServiceImpl implements DictTaskService { private final DictUtils dictConverter; private final DictUtils dictUtils; private final FileHandler fileHandler; + private final DictWordService dictWordService; public DictTaskServiceImpl(DictRepository dictRepository, DictUtils dictConverter, DictUtils dictUtils, FileHandler fileHandler, - KnowledgeBaseService knowledgeBaseService) { + DictWordService dictWordService) { this.dictRepository = dictRepository; this.dictConverter = dictConverter; this.dictUtils = dictUtils; this.fileHandler = fileHandler; + this.dictWordService = dictWordService; } @Override @@ -105,10 +104,11 @@ public class DictTaskServiceImpl implements DictTaskService { // 3.Change in-memory dictionary data in real time try { - HanlpHelper.reloadCustomDictionary(); + dictWordService.loadDictWord(); + dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); dictRepository.editDictTask(dictTaskDO); - } catch (IOException e) { + } catch (Exception e) { log.error("reloadCustomDictionary error", e); } @@ -121,7 +121,7 @@ public class DictTaskServiceImpl implements DictTaskService { fileHandler.deleteDictFile(fileName); try { - HanlpHelper.reloadCustomDictionary(); + dictWordService.loadDictWord(); } catch (Exception e) { log.error("reloadCustomDictionary error", e); }