topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(chat) After fixing the dictionary execution task, it is necessary to reload the metadata. (#1579)

Browse Source
This commit is contained in:
lexluo09
2024-08-17 23:49:56 +08:00
committed by GitHub
parent 115cf19078
commit 07e0ba24bc
3 changed files with 76 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/KnowledgeBaseService.java
View File

@@ -56,23 +56,25 @@ public class KnowledgeBaseService {
return HanlpHelper.getTerms(text, modelIdToDataSetIds);
}
public List<HanlpMapResult> prefixSearch(String key, int limit, Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) {
public List<HanlpMapResult> prefixSearch(String key, int limit,
Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) {
return prefixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
}
public List<HanlpMapResult> prefixSearchByModel(String key, int limit,
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) {
return SearchService.prefixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
}
public List<HanlpMapResult> suffixSearch(String key, int limit, Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) {
public List<HanlpMapResult> suffixSearch(String key, int limit, Map<Long,
List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
return suffixSearchByModel(key, limit, modelIdToDataSetIds, detectDataSetIds);
}
public List<HanlpMapResult> suffixSearchByModel(String key, int limit, Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) {
public List<HanlpMapResult> suffixSearchByModel(String key, int limit, Map<Long,
List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
return SearchService.suffixSearch(key, limit, modelIdToDataSetIds, detectDataSetIds);
}

111
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/helper/HanlpHelper.java
View File

@@ -17,6 +17,12 @@ import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.chat.knowledge.MapResult;
import com.tencent.supersonic.headless.chat.knowledge.MultiCustomDictionary;
import com.tencent.supersonic.headless.chat.knowledge.SearchService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ResourceUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -26,11 +32,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ResourceUtils;
/**
* HanLP helper
@@ -203,18 +204,20 @@ public class HanlpHelper {
if (CollectionUtils.isEmpty(mapResults)) {
return;
}
List<T> newResults = new ArrayList<>();
for (T mapResult : mapResults) {
boolean isAdd = false;
if (MultiCustomDictionary.isLowerLetter(mapResult.getName())) {
if (CustomDictionary.contains(mapResult.getName())) {
CoreDictionary.Attribute attribute = CustomDictionary.get(mapResult.getName());
if (attribute != null) {
isAdd = addLetterOriginal(newResults, mapResult, attribute);
}
String name = mapResult.getName();
boolean isAdded = false;
if (MultiCustomDictionary.isLowerLetter(name) && CustomDictionary.contains(name)) {
CoreDictionary.Attribute attribute = CustomDictionary.get(name);
if (attribute != null) {
isAdded = addLetterOriginal(newResults, mapResult, attribute);
}
}
if (!isAdd) {
if (!isAdded) {
newResults.add(mapResult);
}
}
@@ -223,48 +226,54 @@ public class HanlpHelper {
}
public static <T extends MapResult> boolean addLetterOriginal(List<T> mapResults, T mapResult,
CoreDictionary.Attribute attribute) {
CoreDictionary.Attribute attribute) {
if (attribute == null) {
return false;
}
boolean isAdd = false;
if (attribute != null) {
if (mapResult instanceof HanlpMapResult) {
HanlpMapResult hanlpMapResult = (HanlpMapResult) mapResult;
for (String nature : hanlpMapResult.getNatures()) {
String orig = attribute.getOriginal(Nature.fromString(nature));
if (orig != null) {
MapResult addMapResult = new HanlpMapResult(orig, Arrays.asList(nature),
hanlpMapResult.getDetectWord());
mapResults.add((T) addMapResult);
isAdd = true;
}
}
} else if (mapResult instanceof DatabaseMapResult) {
List<String> originals = attribute.getOriginals();
if (!CollectionUtils.isEmpty(originals)) {
for (String orig : originals) {
DatabaseMapResult addMapResult = new DatabaseMapResult();
addMapResult.setName(orig);
addMapResult.setSchemaElement(((DatabaseMapResult) mapResult).getSchemaElement());
addMapResult.setDetectWord(mapResult.getDetectWord());
mapResults.add((T) addMapResult);
isAdd = true;
}
}
} else if (mapResult instanceof EmbeddingResult) {
List<String> originals = attribute.getOriginals();
if (!CollectionUtils.isEmpty(originals)) {
for (String orig : originals) {
EmbeddingResult addMapResult = new EmbeddingResult();
addMapResult.setName(orig);
addMapResult.setDetectWord(mapResult.getDetectWord());
addMapResult.setId(((EmbeddingResult) mapResult).getId());
addMapResult.setMetadata(((EmbeddingResult) mapResult).getMetadata());
addMapResult.setDistance(((EmbeddingResult) mapResult).getDistance());
mapResults.add((T) addMapResult);
isAdd = true;
}
if (mapResult instanceof HanlpMapResult) {
HanlpMapResult hanlpMapResult = (HanlpMapResult) mapResult;
for (String nature : hanlpMapResult.getNatures()) {
String orig = attribute.getOriginal(Nature.fromString(nature));
if (orig != null) {
MapResult addMapResult = new HanlpMapResult(
orig, Arrays.asList(nature), hanlpMapResult.getDetectWord());
mapResults.add((T) addMapResult);
isAdd = true;
}
}
return isAdd;
}
List<String> originals = attribute.getOriginals();
if (CollectionUtils.isEmpty(originals)) {
return false;
}
if (mapResult instanceof DatabaseMapResult) {
DatabaseMapResult dbMapResult = (DatabaseMapResult) mapResult;
for (String orig : originals) {
DatabaseMapResult addMapResult = new DatabaseMapResult();
addMapResult.setName(orig);
addMapResult.setSchemaElement(dbMapResult.getSchemaElement());
addMapResult.setDetectWord(dbMapResult.getDetectWord());
mapResults.add((T) addMapResult);
isAdd = true;
}
} else if (mapResult instanceof EmbeddingResult) {
EmbeddingResult embeddingResult = (EmbeddingResult) mapResult;
for (String orig : originals) {
EmbeddingResult addMapResult = new EmbeddingResult();
addMapResult.setName(orig);
addMapResult.setDetectWord(embeddingResult.getDetectWord());
addMapResult.setId(embeddingResult.getId());
addMapResult.setMetadata(embeddingResult.getMetadata());
addMapResult.setDistance(embeddingResult.getDistance());
mapResults.add((T) addMapResult);
isAdd = true;
}
}
return isAdd;
}

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
View File

@@ -11,9 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.chat.knowledge.file.FileHandler;
import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
import com.tencent.supersonic.headless.server.service.DictTaskService;
@@ -24,7 +22,6 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
@@ -44,16 +41,18 @@ public class DictTaskServiceImpl implements DictTaskService {
private final DictUtils dictConverter;
private final DictUtils dictUtils;
private final FileHandler fileHandler;
private final DictWordService dictWordService;
public DictTaskServiceImpl(DictRepository dictRepository,
DictUtils dictConverter,
DictUtils dictUtils,
FileHandler fileHandler,
KnowledgeBaseService knowledgeBaseService) {
DictWordService dictWordService) {
this.dictRepository = dictRepository;
this.dictConverter = dictConverter;
this.dictUtils = dictUtils;
this.fileHandler = fileHandler;
this.dictWordService = dictWordService;
}
@Override
@@ -105,10 +104,11 @@ public class DictTaskServiceImpl implements DictTaskService {
// 3.Change in-memory dictionary data in real time
try {
HanlpHelper.reloadCustomDictionary();
dictWordService.loadDictWord();
dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus());
dictRepository.editDictTask(dictTaskDO);
} catch (IOException e) {
} catch (Exception e) {
log.error("reloadCustomDictionary error", e);
}
@@ -121,7 +121,7 @@ public class DictTaskServiceImpl implements DictTaskService {
fileHandler.deleteDictFile(fileName);
try {
HanlpHelper.reloadCustomDictionary();
dictWordService.loadDictWord();
} catch (Exception e) {
log.error("reloadCustomDictionary error", e);
}