topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](Headless) Optimize the issue of integrating the Headless knowledge dictionary. (#731)

Browse Source
This commit is contained in:
lexluo09
2024-02-20 20:36:57 +08:00
committed by GitHub
parent 33240cc382
commit d10801ef38
12 changed files with 136 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.chat.core.mapper;
import com.tencent.supersonic.chat.core.config.OptimizationConfig;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.server.service.KnowledgeService;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -34,6 +34,9 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
@Autowired
private OptimizationConfig optimizationConfig;
@Autowired
private KnowledgeService knowledgeService;
@Override
public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<S2Term> terms,
Set<Long> detectViewIds) {
@@ -61,10 +64,10 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
String detectSegment, int offset) {
// step1. pre search
Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize();
LinkedHashSet<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize,
detectViewIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
LinkedHashSet<HanlpMapResult> hanlpMapResults = knowledgeService.prefixSearch(detectSegment,
oneDetectionMaxSize, detectViewIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
// step2. suffix search
LinkedHashSet<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(detectSegment,
LinkedHashSet<HanlpMapResult> suffixHanlpMapResults = knowledgeService.suffixSearch(detectSegment,
oneDetectionMaxSize, detectViewIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
hanlpMapResults.addAll(suffixHanlpMapResults);

9
chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.server.service.KnowledgeService;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -14,6 +15,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
@@ -25,6 +27,9 @@ public class SearchMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
private static final int SEARCH_SIZE = 3;
@Autowired
private KnowledgeService knowledgeService;
@Override
public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<S2Term> originals,
Set<Long> detectViewIds) {
@@ -51,9 +56,9 @@ public class SearchMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
String detectSegment = text.substring(detectIndex);
if (StringUtils.isNotEmpty(detectSegment)) {
List<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment,
List<HanlpMapResult> hanlpMapResults = knowledgeService.prefixSearch(detectSegment,
SearchService.SEARCH_SIZE, detectViewIds);
List<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(
List<HanlpMapResult> suffixHanlpMapResults = knowledgeService.suffixSearch(
detectSegment, SEARCH_SIZE, detectViewIds);
hanlpMapResults.addAll(suffixHanlpMapResults);
// remove entity name where search