topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 20:25:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(fix)(chat) Fix the issue of mapping in the map (#1825)

Browse Source
This commit is contained in:
lexluo09
2024-10-19 13:22:39 +08:00
committed by GitHub
parent bb07fd77cf
commit 2250289c94
5 changed files with 25 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java
View File

@@ -58,9 +58,8 @@ public class SearchService {
return new HanlpMapResult(name, entry.getValue(), key, similarity); return new HanlpMapResult(name, entry.getValue(), key, similarity);
}).sorted((a, b) -> -(b.getName().length() - a.getName().length())) }).sorted((a, b) -> -(b.getName().length() - a.getName().length()))
.collect(Collectors.toList()); .collect(Collectors.toList());
hanlpMapResults = transformAndFilterByDataSet(hanlpMapResults, modelIdToDataSetIds, return transformAndFilterByDataSet(hanlpMapResults, modelIdToDataSetIds, detectDataSetIds,
detectDataSetIds, limit); limit);
return hanlpMapResults;
} }
/** /**
@@ -71,15 +70,14 @@ public class SearchService {
*/ */
public static List<HanlpMapResult> suffixSearch(String key, int limit, public static List<HanlpMapResult> suffixSearch(String key, int limit,
Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) { Map<Long, List<Long>> modelIdToDataSetIds, Set<Long> detectDataSetIds) {
String reverseDetectSegment = StringUtils.reverse(key); return suffixSearch(key, limit, suffixTrie, modelIdToDataSetIds, detectDataSetIds);
return suffixSearch(reverseDetectSegment, limit, suffixTrie, modelIdToDataSetIds,
detectDataSetIds);
} }
public static List<HanlpMapResult> suffixSearch(String key, int limit, public static List<HanlpMapResult> suffixSearch(String key, int limit,
BinTrie<List<String>> binTrie, Map<Long, List<Long>> modelIdToDataSetIds, BinTrie<List<String>> binTrie, Map<Long, List<Long>> modelIdToDataSetIds,
Set<Long> detectDataSetIds) { Set<Long> detectDataSetIds) {
Set<Map.Entry<String, List<String>>> result = search(key, binTrie); String reverseDetectSegment = StringUtils.reverse(key);
Set<Map.Entry<String, List<String>>> result = search(reverseDetectSegment, binTrie);
List<HanlpMapResult> hanlpMapResults = result.stream().map(entry -> { List<HanlpMapResult> hanlpMapResults = result.stream().map(entry -> {
String name = entry.getKey().replace("#", " "); String name = entry.getKey().replace("#", " ");
List<String> natures = entry.getValue().stream() List<String> natures = entry.getValue().stream()

6
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java
View File

@@ -77,9 +77,9 @@ public class MapFilter {
for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) { for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {
List<SchemaElementMatch> value = entry.getValue(); List<SchemaElementMatch> value = entry.getValue();
if (!CollectionUtils.isEmpty(value)) { if (!CollectionUtils.isEmpty(value)) {
value.removeIf(schemaElementMatch -> StringUtils value.removeIf(
.length(schemaElementMatch.getDetectWord()) <= 2 schemaElementMatch -> StringUtils.length(schemaElementMatch.getWord()) <= 2
&& schemaElementMatch.getSimilarity() < 1); && schemaElementMatch.getSimilarity() < 1);
} }
} }
} }

2
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java
View File

@@ -15,7 +15,7 @@ public class MapperConfig extends ParameterConfig {
"number", "Mapper相关配置"); "number", "Mapper相关配置");
public static final Parameter MAPPER_NAME_THRESHOLD = public static final Parameter MAPPER_NAME_THRESHOLD =
new Parameter("s2.mapper.name.threshold", "0.5", "指标名、维度名文本相似度阈值", new Parameter("s2.mapper.name.threshold", "0.3", "指标名、维度名文本相似度阈值",
"文本片段和匹配到的指标、维度名计算出来的编辑距离阈值, 若超出该阈值, 则舍弃", "number", "Mapper相关配置"); "文本片段和匹配到的指标、维度名计算出来的编辑距离阈值, 若超出该阈值, 则舍弃", "number", "Mapper相关配置");
public static final Parameter MAPPER_NAME_THRESHOLD_MIN = public static final Parameter MAPPER_NAME_THRESHOLD_MIN =

11
launchers/standalone/src/main/resources/config.update/sql-update.sql
View File

@@ -385,13 +385,12 @@ CREATE TABLE IF NOT EXISTS `s2_chat_model` (
`viewer` varchar(500) DEFAULT NULL, `viewer` varchar(500) DEFAULT NULL,
PRIMARY KEY (`id`) PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='对话大模型实例表'; ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='对话大模型实例表';
ALTER TABLE s2_agent RENAME COLUMN config TO tool_config; ALTER TABLE s2_agent CHANGE COLUMN config tool_config text;
ALTER TABLE s2_agent RENAME COLUMN model_config TO chat_model_config; ALTER TABLE s2_agent CHANGE COLUMN model_config chat_model_config text;
--20241011 --20241011
ALTER TABLE s2_agent DROP COLUMN IF EXISTS `prompt_config`; ALTER TABLE s2_agent DROP COLUMN `prompt_config`;
ALTER TABLE s2_agent DROP COLUMN IF EXISTS `multi_turn_config`; ALTER TABLE s2_agent DROP COLUMN `multi_turn_config`;
ALTER TABLE s2_agent DROP COLUMN IF EXISTS `enable_memory_review`; ALTER TABLE s2_agent DROP COLUMN `enable_memory_review`;
--20241012 --20241012
alter table s2_agent add column `enable_feedback` tinyint DEFAULT 1; alter table s2_agent add column `enable_feedback` tinyint DEFAULT 1;

12
launchers/standalone/src/main/resources/s2-config.yaml
View File

@@ -31,4 +31,14 @@ s2:
token: token:
http: http:
header: header:
key: Authorization key: Authorization
mapper:
name:
threshold: 0.30
min:
threshold:value: 0.25
value:
threshold: 0.5
min:
threshold: 0.3