topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 20:25:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Filter out terms whose detect word length is 1 (#1041)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-05-28 21:37:19 +08:00
committed by GitHub
parent c42f3477a4
commit bd9cc8f88a
3 changed files with 21 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryMapReq.java
View File

@@ -14,7 +14,7 @@ public class QueryMapReq {
private String queryText; private String queryText;
private List<String> dataSetNames; private List<String> dataSetNames;
private User user; private User user;
private Integer topN = 10; private Integer topN = 0;
private MapModeEnum mapModeEnum = MapModeEnum.STRICT; private MapModeEnum mapModeEnum = MapModeEnum.STRICT;
private QueryDataType queryDataType = QueryDataType.ALL; private QueryDataType queryDataType = QueryDataType.ALL;
} }

17
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMapper.java
View File

@@ -46,6 +46,7 @@ public abstract class BaseMapper implements SchemaMapper {
private void filter(QueryContext queryContext) { private void filter(QueryContext queryContext) {
filterByDataSetId(queryContext); filterByDataSetId(queryContext);
filterTermByDetectWordLen(queryContext);
switch (queryContext.getQueryDataType()) { switch (queryContext.getQueryDataType()) {
case TAG: case TAG:
filterByQueryDataType(queryContext, element -> !(element.getIsTag() > 0)); filterByQueryDataType(queryContext, element -> !(element.getIsTag() > 0));
@@ -79,6 +80,22 @@ public abstract class BaseMapper implements SchemaMapper {
} }
} }
private static void filterTermByDetectWordLen(QueryContext queryContext) {
Map<Long, List<SchemaElementMatch>> dataSetElementMatches =
queryContext.getMapInfo().getDataSetElementMatches();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
List<SchemaElementMatch> value = entry.getValue();
if (!CollectionUtils.isEmpty(value)) {
value.removeIf(schemaElementMatch -> {
if (!SchemaElementType.TERM.equals(schemaElementMatch.getElement().getType())) {
return false;
}
return StringUtils.length(schemaElementMatch.getDetectWord()) <= 1;
});
}
}
}
private static void filterByQueryDataType(QueryContext queryContext, Predicate<SchemaElement> needRemovePredicate) { private static void filterByQueryDataType(QueryContext queryContext, Predicate<SchemaElement> needRemovePredicate) {
queryContext.getMapInfo().getDataSetElementMatches().values().stream().forEach( queryContext.getMapInfo().getDataSetElementMatches().values().stream().forEach(
schemaElementMatches -> schemaElementMatches.removeIf( schemaElementMatches -> schemaElementMatches.removeIf(

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetaDiscoveryServiceImpl.java
View File

@@ -119,7 +119,9 @@ public class MetaDiscoveryServiceImpl implements MetaDiscoveryService {
SchemaMapInfo mapInfo, SchemaMapInfo mapInfo,
Map<Long, DataSetResp> dataSetMap) { Map<Long, DataSetResp> dataSetMap) {
Map<Long, List<SchemaElementMatch>> result = new HashMap<>(); Map<Long, List<SchemaElementMatch>> result = new HashMap<>();
if (0 == topN) {
return result;
}
SemanticSchema semanticSchema = semanticService.getSemanticSchema(); SemanticSchema semanticSchema = semanticService.getSemanticSchema();
for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) { for (Map.Entry<Long, List<SchemaElementMatch>> entry : mapInfo.getDataSetElementMatches().entrySet()) {
Long dataSetId = entry.getKey(); Long dataSetId = entry.getKey();