topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) After fixing the dictionary execution task, it is necessary to reload the metadata. (#1579)

Browse Source
This commit is contained in:
lexluo09
2024-08-17 23:49:56 +08:00
committed by GitHub
parent 115cf19078
commit 07e0ba24bc
3 changed files with 76 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java
View File

@@ -56,23 +56,25 @@ public class KnowledgeBaseService {
return HanlpHelper.getTerms(text, modelIdToDataSetIds); return HanlpHelper.getTerms(text, modelIdToDataSetIds);
} }
public List<HanlpMapResult> prefixSearch(String key, int limit, Map<Long, List<Long>> modelIdToDataSetIds, public List<HanlpMapResult> prefixSearch(String key, int limit,
Set<Long> detectDataSetIds) { Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) {
return prefixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds); return prefixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
} }
public List<HanlpMapResult> prefixSearchByModel(String key, int limit, public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) { Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) {
return SearchService.prefixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds); return SearchService.prefixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
} }
public List<HanlpMapResult> suffixSearch(String key, int limit, Map<Long, List<Long>> modelIdToDataSetIds, public List<HanlpMapResult> suffixSearch(String key, int limit, Map<Long,
Set<Long> detectDataSetIds) { List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
return suffixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds); return suffixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
} }
public List<HanlpMapResult> suffixSearchByModel(String key, int limit, Map<Long, List<Long>> modelIdToDataSetIds, public List<HanlpMapResult> suffixSearchByModel(String key, int limit, Map<Long,
Set<Long> detectDataSetIds) { List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
return SearchService.suffixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds); return SearchService.suffixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
} }

111
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java
View File

@@ -17,6 +17,12 @@ import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.chat.knowledge.MapResult; import com.tencent.supersonic.headless.chat.knowledge.MapResult;
import com.tencent.supersonic.headless.chat.knowledge.MultiCustomDictionary; import com.tencent.supersonic.headless.chat.knowledge.MultiCustomDictionary;
import com.tencent.supersonic.headless.chat.knowledge.SearchService; import com.tencent.supersonic.headless.chat.knowledge.SearchService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ResourceUtils;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
@@ -26,11 +32,6 @@ import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ResourceUtils;
/** /**
* HanLP helper * HanLP helper
@@ -203,18 +204,20 @@ public class HanlpHelper {
if (CollectionUtils.isEmpty(mapResults)) { if (CollectionUtils.isEmpty(mapResults)) {
return; return;
} }
List<T> newResults = new ArrayList<>(); List<T> newResults = new ArrayList<>();
for (T mapResult : mapResults) { for (T mapResult : mapResults) {
boolean isAdd = false; String name = mapResult.getName();
if (MultiCustomDictionary.isLowerLetter(mapResult.getName())) { boolean isAdded = false;
if (CustomDictionary.contains(mapResult.getName())) { if (MultiCustomDictionary.isLowerLetter(name) && CustomDictionary.contains(name)) {
CoreDictionary.Attribute attribute = CustomDictionary.get(mapResult.getName()); CoreDictionary.Attribute attribute = CustomDictionary.get(name);
if (attribute != null) { if (attribute != null) {
isAdd = addLetterOriginal(newResults, mapResult, attribute); isAdded = addLetterOriginal(newResults, mapResult, attribute);
}
} }
} }
if (!isAdd) {
if (!isAdded) {
newResults.add(mapResult); newResults.add(mapResult);
} }
} }
@@ -223,48 +226,54 @@ public class HanlpHelper {
} }
public static <T extends MapResult> boolean addLetterOriginal(List<T> mapResults, T mapResult, public static <T extends MapResult> boolean addLetterOriginal(List<T> mapResults, T mapResult,
CoreDictionary.Attribute attribute) { CoreDictionary.Attribute attribute) {
if (attribute == null) {
return false;
}
boolean isAdd = false; boolean isAdd = false;
if (attribute != null) { if (mapResult instanceof HanlpMapResult) {
if (mapResult instanceof HanlpMapResult) { HanlpMapResult hanlpMapResult = (HanlpMapResult) mapResult;
HanlpMapResult hanlpMapResult = (HanlpMapResult) mapResult; for (String nature : hanlpMapResult.getNatures()) {
for (String nature : hanlpMapResult.getNatures()) { String orig = attribute.getOriginal(Nature.fromString(nature));
String orig = attribute.getOriginal(Nature.fromString(nature)); if (orig != null) {
if (orig != null) { MapResult addMapResult = new HanlpMapResult(
MapResult addMapResult = new HanlpMapResult(orig, Arrays.asList(nature), orig, Arrays.asList(nature), hanlpMapResult.getDetectWord());
hanlpMapResult.getDetectWord()); mapResults.add((T) addMapResult);
mapResults.add((T) addMapResult); isAdd = true;
isAdd = true;
}
}
} else if (mapResult instanceof DatabaseMapResult) {
List<String> originals = attribute.getOriginals();
if (!CollectionUtils.isEmpty(originals)) {
for (String orig : originals) {
DatabaseMapResult addMapResult = new DatabaseMapResult();
addMapResult.setName(orig);
addMapResult.setSchemaElement(((DatabaseMapResult) mapResult).getSchemaElement());
addMapResult.setDetectWord(mapResult.getDetectWord());
mapResults.add((T) addMapResult);
isAdd = true;
}
}
} else if (mapResult instanceof EmbeddingResult) {
List<String> originals = attribute.getOriginals();
if (!CollectionUtils.isEmpty(originals)) {
for (String orig : originals) {
EmbeddingResult addMapResult = new EmbeddingResult();
addMapResult.setName(orig);
addMapResult.setDetectWord(mapResult.getDetectWord());
addMapResult.setId(((EmbeddingResult) mapResult).getId());
addMapResult.setMetadata(((EmbeddingResult) mapResult).getMetadata());
addMapResult.setDistance(((EmbeddingResult) mapResult).getDistance());
mapResults.add((T) addMapResult);
isAdd = true;
}
} }
} }
return isAdd;
} }
List<String> originals = attribute.getOriginals();
if (CollectionUtils.isEmpty(originals)) {
return false;
}
if (mapResult instanceof DatabaseMapResult) {
DatabaseMapResult dbMapResult = (DatabaseMapResult) mapResult;
for (String orig : originals) {
DatabaseMapResult addMapResult = new DatabaseMapResult();
addMapResult.setName(orig);
addMapResult.setSchemaElement(dbMapResult.getSchemaElement());
addMapResult.setDetectWord(dbMapResult.getDetectWord());
mapResults.add((T) addMapResult);
isAdd = true;
}
} else if (mapResult instanceof EmbeddingResult) {
EmbeddingResult embeddingResult = (EmbeddingResult) mapResult;
for (String orig : originals) {
EmbeddingResult addMapResult = new EmbeddingResult();
addMapResult.setName(orig);
addMapResult.setDetectWord(embeddingResult.getDetectWord());
addMapResult.setId(embeddingResult.getId());
addMapResult.setMetadata(embeddingResult.getMetadata());
addMapResult.setDistance(embeddingResult.getDistance());
mapResults.add((T) addMapResult);
isAdd = true;
}
}
return isAdd; return isAdd;
} }

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
View File

@@ -11,9 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler; import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler;
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
import com.tencent.supersonic.headless.server.service.DictTaskService; import com.tencent.supersonic.headless.server.service.DictTaskService;
@@ -24,7 +22,6 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
@@ -44,16 +41,18 @@ public class DictTaskServiceImpl implements DictTaskService {
private final DictUtils dictConverter; private final DictUtils dictConverter;
private final DictUtils dictUtils; private final DictUtils dictUtils;
private final FileHandler fileHandler; private final FileHandler fileHandler;
private final DictWordService dictWordService;
public DictTaskServiceImpl(DictRepository dictRepository, public DictTaskServiceImpl(DictRepository dictRepository,
DictUtils dictConverter, DictUtils dictConverter,
DictUtils dictUtils, DictUtils dictUtils,
FileHandler fileHandler, FileHandler fileHandler,
KnowledgeBaseService knowledgeBaseService) { DictWordService dictWordService) {
this.dictRepository = dictRepository; this.dictRepository = dictRepository;
this.dictConverter = dictConverter; this.dictConverter = dictConverter;
this.dictUtils = dictUtils; this.dictUtils = dictUtils;
this.fileHandler = fileHandler; this.fileHandler = fileHandler;
this.dictWordService = dictWordService;
} }
@Override @Override
@@ -105,10 +104,11 @@ public class DictTaskServiceImpl implements DictTaskService {
// 3.Change in-memory dictionary data in real time // 3.Change in-memory dictionary data in real time
try { try {
HanlpHelper.reloadCustomDictionary(); dictWordService.loadDictWord();
dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus());
dictRepository.editDictTask(dictTaskDO); dictRepository.editDictTask(dictTaskDO);
} catch (IOException e) { } catch (Exception e) {
log.error("reloadCustomDictionary error", e); log.error("reloadCustomDictionary error", e);
} }
@@ -121,7 +121,7 @@ public class DictTaskServiceImpl implements DictTaskService {
fileHandler.deleteDictFile(fileName); fileHandler.deleteDictFile(fileName);
try { try {
HanlpHelper.reloadCustomDictionary(); dictWordService.loadDictWord();
} catch (Exception e) { } catch (Exception e) {
log.error("reloadCustomDictionary error", e); log.error("reloadCustomDictionary error", e);
} }