From 6dda8eed45930a9879df4ae2be692d55c8dafe38 Mon Sep 17 00:00:00 2001 From: QJ_wonder <38885395+BigdataQIJI@users.noreply.github.com> Date: Wed, 21 May 2025 11:16:28 +0800 Subject: [PATCH] =?UTF-8?q?(feature)(headless)=20=E6=9B=B4=E6=94=B9?= =?UTF-8?q?=E5=8E=9F=E6=9C=89=E6=95=B0=E6=8D=AE=E9=9B=86=E9=80=89=E6=8B=A9?= =?UTF-8?q?=E6=96=B9=E6=B3=95=EF=BC=8C=E6=9B=BF=E6=8D=A2=E4=B8=BA=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E4=BD=99=E5=BC=A6=E7=9B=B8=E4=BC=BC=E5=BA=A6=E8=AE=A1?= =?UTF-8?q?=E7=AE=97=E6=9F=A5=E8=AF=A2=E6=96=87=E6=9C=AC=E5=92=8C=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E9=9B=86=E5=90=8D=E7=A7=B0=E7=9A=84=E7=9B=B8=E4=BC=BC?= =?UTF-8?q?=E5=BA=A6=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更改原有数据集选择方法，替换为使用余弦相似度计算查询文本和数据集名称的相似度。 --- .../parser/llm/HeuristicDataSetResolver.java | 95 +++---------------- 1 file changed, 14 insertions(+), 81 deletions(-) diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java index c1f6591a1..33866d8c3 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/HeuristicDataSetResolver.java @@ -1,98 +1,31 @@ package com.tencent.supersonic.headless.chat.parser.llm; -import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch; -import com.tencent.supersonic.headless.api.pojo.SchemaElementType; -import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo; -import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.chat.ChatQueryContext; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; + +import static com.tencent.supersonic.headless.chat.parser.llm.TextSimilarityCalculation.getDataSetSimilarity; /** - * HeuristicDataSetResolver select ONE most suitable data set out of matched data sets. The - * selection is based on similarity comparison rule and the priority is like: 1. - * maxSimilarity(matched dataset) 2. maxSimilarity(all matched metrics) 3. totalSimilarity(all - * matched elements) + * HeuristicDataSetResolver select ONE most suitable data set out of data sets. The + * selection is based on the cosine similarity directly between the question text and the dataset name */ @Slf4j public class HeuristicDataSetResolver implements DataSetResolver { public Long resolve(ChatQueryContext chatQueryContext, Set agentDataSetIds) { - SchemaMapInfo mapInfo = chatQueryContext.getMapInfo(); - Set matchedDataSets = mapInfo.getMatchedDataSetInfos(); - if (CollectionUtils.isNotEmpty(agentDataSetIds)) { - matchedDataSets.retainAll(agentDataSetIds); + String queryText = chatQueryContext.getRequest().getQueryText(); + List dataSets = chatQueryContext.getSemanticSchema().getDataSets(); + if(dataSets.size() == 1){ + return dataSets.get(0).getDataSetId(); } - if (matchedDataSets.size() == 1) { - return matchedDataSets.stream().findFirst().get(); + Map dataSetSimilarity = new LinkedHashMap<>(); + for (SchemaElement dataSet : dataSets){ + dataSetSimilarity.put(dataSet.getDataSetId(),getDataSetSimilarity(queryText,dataSet.getDataSetName())); } - return selectDataSetByMatchSimilarity(mapInfo); - } - - protected Long selectDataSetByMatchSimilarity(SchemaMapInfo schemaMap) { - Map dataSetMatchRet = - getDataSetMatchResult(schemaMap); - Entry selectedDataset = - dataSetMatchRet.entrySet().stream().sorted((o1, o2) -> { - double difference = o1.getValue().getMaxDatesetSimilarity() - - o2.getValue().getMaxDatesetSimilarity(); - if (difference == 0) { - difference = o1.getValue().getMaxMetricSimilarity() - - o2.getValue().getMaxMetricSimilarity(); - if (difference == 0) { - difference = o1.getValue().getTotalSimilarity() - - o2.getValue().getTotalSimilarity(); - } - if (difference == 0) { - difference = o1.getValue().getMaxMetricUseCnt() - - o2.getValue().getMaxMetricUseCnt(); - } - } - return difference >= 0 ? -1 : 1; - }).findFirst().orElse(null); - if (selectedDataset != null) { - log.info("selectDataSet with multiple DataSets [{}]", selectedDataset.getKey()); - return selectedDataset.getKey(); - } - - return null; - } - - protected Map getDataSetMatchResult( - SchemaMapInfo schemaMap) { - Map dateSetMatchRet = new HashMap<>(); - for (Entry> entry : schemaMap.getDataSetElementMatches() - .entrySet()) { - double maxMetricSimilarity = 0; - double maxDatasetSimilarity = 0; - double totalSimilarity = 0; - long maxMetricUseCnt = 0L; - for (SchemaElementMatch match : entry.getValue()) { - if (SchemaElementType.DATASET.equals(match.getElement().getType())) { - maxDatasetSimilarity = Math.max(maxDatasetSimilarity, match.getSimilarity()); - } - if (SchemaElementType.METRIC.equals(match.getElement().getType())) { - maxMetricSimilarity = Math.max(maxMetricSimilarity, match.getSimilarity()); - if (Objects.nonNull(match.getElement().getUseCnt())) { - maxMetricUseCnt = Math.max(maxMetricUseCnt, match.getElement().getUseCnt()); - } - } - totalSimilarity += match.getSimilarity(); - } - dateSetMatchRet.put(entry.getKey(), - SemanticParseInfo.DataSetMatchResult.builder() - .maxMetricSimilarity(maxMetricSimilarity) - .maxDatesetSimilarity(maxDatasetSimilarity) - .totalSimilarity(totalSimilarity).build()); - } - - return dateSetMatchRet; + return dataSetSimilarity.entrySet().stream().max(Map.Entry.comparingByValue()).get().getKey(); } }