Merge aaf2d46a56 into 94e853f57e

[improvement][headless]Support dataSetNames that contain dash.

[improvement][headless]Support dataSetNames that contain dash.

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ErrorMsgRewriteProcessor.java

@@ -27,7 +27,7 @@ public class ErrorMsgRewriteProcessor implements ParseResultProcessor {
 
     private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
 
-    public static final String APP_KEY_ERROR_MESSAGE = "REWRITE_ERROR_MESSAGE";
+    public static final String APP_KEY = "REWRITE_ERROR_MESSAGE";
     private static final String REWRITE_ERROR_MESSAGE_INSTRUCTION = ""
             + "#Role: You are a data business partner who closely interacts with business people.\n"
             + "#Task: Your will be provided with user input, system output and some examples, "
@@ -38,7 +38,7 @@ public class ErrorMsgRewriteProcessor implements ParseResultProcessor {
             + "#Examples: {{examples}}\n" + "#Response: ";
 
     public ErrorMsgRewriteProcessor() {
-        ChatAppManager.register(APP_KEY_ERROR_MESSAGE,
+        ChatAppManager.register(APP_KEY,
                 ChatApp.builder().prompt(REWRITE_ERROR_MESSAGE_INSTRUCTION).name("异常提示改写")
                         .appModule(AppModule.CHAT).description("通过大模型将异常信息改写为更友好和引导性的提示用语")
                         .enable(true).build());
@@ -46,7 +46,7 @@ public class ErrorMsgRewriteProcessor implements ParseResultProcessor {
 
     @Override
     public boolean accept(ParseContext parseContext) {
-        ChatApp chatApp = parseContext.getAgent().getChatAppConfig().get(APP_KEY_ERROR_MESSAGE);
+        ChatApp chatApp = parseContext.getAgent().getChatAppConfig().get(APP_KEY);
         return StringUtils.isNotBlank(parseContext.getResponse().getErrorMsg())
                 && Objects.nonNull(chatApp) && chatApp.isEnable();
     }
@@ -54,16 +54,20 @@ public class ErrorMsgRewriteProcessor implements ParseResultProcessor {
     @Override
     public void process(ParseContext parseContext) {
         String errMsg = parseContext.getResponse().getErrorMsg();
-        ChatApp chatApp = parseContext.getAgent().getChatAppConfig().get(APP_KEY_ERROR_MESSAGE);
+        ChatApp chatApp = parseContext.getAgent().getChatAppConfig().get(APP_KEY);
         Map<String, Object> variables = new HashMap<>();
         variables.put("user_question", parseContext.getRequest().getQueryText());
         variables.put("system_message", errMsg);
 
         StringBuilder exampleStr = new StringBuilder();
-        parseContext.getResponse().getUsedExemplars().forEach(e -> exampleStr.append(
-                String.format("<Question:{%s},Schema:{%s}> ", e.getQuestion(), e.getDbSchema())));
-        parseContext.getAgent().getExamples()
-                .forEach(e -> exampleStr.append(String.format("<Question:{%s}> ", e)));
+        if (parseContext.getResponse().getUsedExemplars() != null) {
+            parseContext.getResponse().getUsedExemplars().forEach(e -> exampleStr.append(String
+                    .format("<Question:{%s},Schema:{%s}> ", e.getQuestion(), e.getDbSchema())));
+        }
+        if (parseContext.getAgent().getExamples() != null) {
+            parseContext.getAgent().getExamples()
+                    .forEach(e -> exampleStr.append(String.format("<Question:{%s}> ", e)));
+        }
         variables.put("examples", exampleStr);
 
         Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variables);

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java

@@ -38,6 +38,10 @@ public class SqlReplaceHelper {
 
     private final static double replaceColumnThreshold = 0.4;
 
+    public static String escapeTableName(String table) {
+        return String.format("`%s`", table);
+    }
+
     public static String replaceAggFields(String sql,
             Map<String, Pair<String, String>> fieldNameToAggMap) {
         Select selectStatement = SqlSelectHelper.getSelect(sql);

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java

@@ -228,7 +228,7 @@ public class SqlSelectHelper {
             statement = CCJSqlParserUtil.parse(sql);
         } catch (JSQLParserException e) {
             log.error("parse error, sql:{}", sql, e);
-            return null;
+            throw new RuntimeException(e);
         }
 
         if (statement instanceof ParenthesedSelect) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElementMatch.java

@@ -1,10 +1,6 @@
 package com.tencent.supersonic.headless.api.pojo;
 
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.ToString;
+import lombok.*;
 
 import java.io.Serializable;
 
@@ -21,6 +17,7 @@ public class SchemaElementMatch implements Serializable {
     private String word;
     private Long frequency;
     private boolean isInherited;
+    private boolean llmMatched;
 
     public boolean isFullMatched() {
         return 1.0 == similarity;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
+import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DateConf;
@@ -281,7 +282,7 @@ public class QueryStructReq extends SemanticQueryReq {
 
     public String getTableName() {
         if (StringUtils.isNotBlank(dataSetName)) {
-            return dataSetName;
+            return SqlReplaceHelper.escapeTableName(dataSetName);
         }
         if (dataSetId != null) {
             return Constants.TABLE_PREFIX + dataSetId;

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/EmbeddingResult.java

@@ -13,6 +13,7 @@ public class EmbeddingResult extends MapResult {
 
     private String id;
     private Map<String, String> metadata;
+    private boolean llmMatched;
 
     @Override
     public boolean equals(Object o) {

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMapper.java

@@ -1,9 +1,12 @@
 package com.tencent.supersonic.headless.chat.mapper;
 
+import com.tencent.supersonic.common.pojo.enums.Text2SQLType;
 import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.headless.api.pojo.enums.MapModeEnum;
 import com.tencent.supersonic.headless.chat.ChatQueryContext;
 import com.tencent.supersonic.headless.chat.knowledge.EmbeddingResult;
@@ -11,6 +14,7 @@ import com.tencent.supersonic.headless.chat.knowledge.builder.BaseWordBuilder;
 import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
 import dev.langchain4j.store.embedding.Retrieval;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
 
 import java.util.List;
 import java.util.Objects;
@@ -23,10 +27,16 @@ public class EmbeddingMapper extends BaseMapper {
 
     @Override
     public boolean accept(ChatQueryContext chatQueryContext) {
-        return MapModeEnum.LOOSE.equals(chatQueryContext.getRequest().getMapModeEnum());
+        boolean b0 = MapModeEnum.LOOSE.equals(chatQueryContext.getRequest().getMapModeEnum());
+        boolean b1 = chatQueryContext.getRequest().getText2SQLType() == Text2SQLType.LLM_OR_RULE;
+        return b0 || b1;
     }
 
     public void doMap(ChatQueryContext chatQueryContext) {
+
+        // TODO: 如果是在LOOSE执行过了，那么在LLM_OR_RULE阶段可以不用执行，所以这里缺乏一个状态来传递，暂时先忽略这个浪费行为吧
+        SchemaMapInfo mappedInfo = chatQueryContext.getMapInfo();
+
         // 1. Query from embedding by queryText
         EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class);
         List<EmbeddingResult> matchResults = getMatches(chatQueryContext, matchStrategy);
@@ -53,15 +63,26 @@ public class EmbeddingMapper extends BaseMapper {
                 continue;
             }
 
+
             // Build SchemaElementMatch object
             SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
                     .element(schemaElement).frequency(BaseWordBuilder.DEFAULT_FREQUENCY)
                     .word(matchResult.getName()).similarity(matchResult.getSimilarity())
                     .detectWord(matchResult.getDetectWord()).build();
+            schemaElementMatch.setLlmMatched(matchResult.isLlmMatched());
 
             // 3. Add SchemaElementMatch to mapInfo
             addToSchemaMap(chatQueryContext.getMapInfo(), dataSetId, schemaElementMatch);
         }
+        if (CollectionUtils.isEmpty(matchResults)) {
+            log.info("embedding mapper no match");
+        } else {
+            for (EmbeddingResult matchResult : matchResults) {
+                log.info("embedding match name=[{}],detectWord=[{}],similarity=[{}],metadata=[{}]",
+                        matchResult.getName(), matchResult.getDetectWord(),
+                        matchResult.getSimilarity(), JsonUtil.toString(matchResult.getMetadata()));
+            }
+        }
     }
 
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/EmbeddingMatchStrategy.java

@@ -1,9 +1,17 @@
 package com.tencent.supersonic.headless.chat.mapper;
 
+import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.hankcs.hanlp.seg.common.Term;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
 import com.tencent.supersonic.headless.chat.ChatQueryContext;
 import com.tencent.supersonic.headless.chat.knowledge.EmbeddingResult;
 import com.tencent.supersonic.headless.chat.knowledge.MetaEmbeddingService;
+import com.tencent.supersonic.headless.chat.knowledge.helper.HanlpHelper;
+import dev.langchain4j.model.chat.ChatLanguageModel;
+import dev.langchain4j.model.input.Prompt;
+import dev.langchain4j.model.input.PromptTemplate;
+import dev.langchain4j.provider.ModelProvider;
 import dev.langchain4j.store.embedding.Retrieval;
 import dev.langchain4j.store.embedding.RetrieveQuery;
 import dev.langchain4j.store.embedding.RetrieveQueryResult;
@@ -14,18 +22,12 @@ import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.EMBEDDING_MAPPER_NUMBER;
-import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.EMBEDDING_MAPPER_ROUND_NUMBER;
-import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.EMBEDDING_MAPPER_THRESHOLD;
+import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.*;
 
 /**
  * EmbeddingMatchStrategy uses vector database to perform similarity search against the embeddings
@@ -35,37 +37,165 @@ import static com.tencent.supersonic.headless.chat.mapper.MapperConfig.EMBEDDING
 @Slf4j
 public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult> {
 
+    @Autowired
+    protected MapperConfig mapperConfig;
+
     @Autowired
     private MetaEmbeddingService metaEmbeddingService;
 
+    private static final String LLM_FILTER_PROMPT =
+            """
+                    \
+                    #Role: You are a professional data analyst specializing in metrics and dimensions.
+                    #Task: Given a user query and a list of retrieved metrics/dimensions through vector recall,
+                    please analyze which metrics/dimensions the user is most likely interested in.
+                    #Rules:
+                    1. Based on user query and retrieved info, accurately determine metrics/dimensions user truly cares about.
+                    2. Do not return all retrieved info, only select those highly relevant to user query.
+                    3. Maintain high quality output, exclude metrics/dimensions irrelevant to user intent.
+                    4. Output must be in JSON array format, only include IDs from retrieved info, e.g.: ['id1', 'id2']
+                    5. Return JSON content directly without markdown formatting
+                    #Input Example:
+                    #User Query: {{userText}}
+                    #Retrieved Metrics/Dimensions: {{retrievedInfo}}
+                    #Output:""";
+
+    @Override
+    public List<EmbeddingResult> detect(ChatQueryContext chatQueryContext, List<S2Term> terms,
+            Set<Long> detectDataSetIds) {
+        if (chatQueryContext == null || CollectionUtils.isEmpty(detectDataSetIds)) {
+            log.warn("Invalid input parameters: context={}, dataSetIds={}", chatQueryContext,
+                    detectDataSetIds);
+            return Collections.emptyList();
+        }
+
+        // 1. Base detection
+        List<EmbeddingResult> baseResults = super.detect(chatQueryContext, terms, detectDataSetIds);
+
+        boolean useLLM = Boolean.parseBoolean(mapperConfig.getParameterValue(EMBEDDING_MAPPER_USE_LLM));
+
+        // 2. LLM enhanced detection
+        if (useLLM) {
+            List<EmbeddingResult> llmResults = detectWithLLM(chatQueryContext, detectDataSetIds);
+            if (!CollectionUtils.isEmpty(llmResults)) {
+                baseResults.addAll(llmResults);
+            }
+        }
+
+        // 3. Deduplicate results
+        return baseResults.stream().distinct().collect(Collectors.toList());
+    }
+
+    /**
+     * Perform enhanced detection using LLM
+     */
+    private List<EmbeddingResult> detectWithLLM(ChatQueryContext chatQueryContext,
+            Set<Long> detectDataSetIds) {
+        try {
+            String queryText = chatQueryContext.getRequest().getQueryText();
+            if (StringUtils.isBlank(queryText)) {
+                return Collections.emptyList();
+            }
+
+            // Get segmentation results
+            Set<String> detectSegments = extractValidSegments(queryText);
+            if (CollectionUtils.isEmpty(detectSegments)) {
+                log.info("No valid segments found for text: {}", queryText);
+                return Collections.emptyList();
+            }
+
+            return detectByBatch(chatQueryContext, detectDataSetIds, detectSegments, true);
+        } catch (Exception e) {
+            log.error("Error in LLM detection for context: {}", chatQueryContext, e);
+            return Collections.emptyList();
+        }
+    }
+
+    /**
+     * Extract valid word segments by filtering out unwanted word natures
+     */
+    private Set<String> extractValidSegments(String text) {
+        List<String> natureList = Arrays.asList(StringUtils.split(mapperConfig.getParameterValue(EMBEDDING_MAPPER_ALLOWED_SEGMENT_NATURE ), ","));
+        return HanlpHelper.getSegment().seg(text).stream()
+                .filter(t -> natureList.stream().noneMatch(nature -> t.nature.startsWith(nature)))
+                .map(Term::getWord).collect(Collectors.toSet());
+    }
+
     @Override
     public List<EmbeddingResult> detectByBatch(ChatQueryContext chatQueryContext,
             Set<Long> detectDataSetIds, Set<String> detectSegments) {
+        return detectByBatch(chatQueryContext, detectDataSetIds, detectSegments, false);
+    }
+
+    /**
+     * Process detection in batches with LLM option
+     *
+     * @param chatQueryContext The context of the chat query
+     * @param detectDataSetIds Target dataset IDs for detection
+     * @param detectSegments Segments to be detected
+     * @param useLlm Whether to use LLM for filtering results
+     * @return List of embedding results
+     */
+    public List<EmbeddingResult> detectByBatch(ChatQueryContext chatQueryContext,
+            Set<Long> detectDataSetIds, Set<String> detectSegments, boolean useLlm) {
         Set<EmbeddingResult> results = ConcurrentHashMap.newKeySet();
         int embeddingMapperBatch = Integer
                 .valueOf(mapperConfig.getParameterValue(MapperConfig.EMBEDDING_MAPPER_BATCH));
 
-        List<String> queryTextsList =
-                detectSegments.stream().map(detectSegment -> detectSegment.trim())
-                        .filter(detectSegment -> StringUtils.isNotBlank(detectSegment))
-                        .collect(Collectors.toList());
+        // Process and filter query texts
+        List<String> queryTextsList = detectSegments.stream().map(String::trim)
+                .filter(StringUtils::isNotBlank).collect(Collectors.toList());
 
+        // Partition queries into sub-lists for batch processing
         List<List<String>> queryTextsSubList =
                 Lists.partition(queryTextsList, embeddingMapperBatch);
 
+        // Create and execute tasks for each batch
         List<Callable<Void>> tasks = new ArrayList<>();
         for (List<String> queryTextsSub : queryTextsSubList) {
-            tasks.add(createTask(chatQueryContext, detectDataSetIds, queryTextsSub, results));
+            tasks.add(
+                    createTask(chatQueryContext, detectDataSetIds, queryTextsSub, results, useLlm));
         }
         executeTasks(tasks);
+
+        // Apply LLM filtering if enabled
+        if (useLlm) {
+            Map<String, Object> variable = new HashMap<>();
+            variable.put("userText", chatQueryContext.getRequest().getQueryText());
+            variable.put("retrievedInfo", JSONObject.toJSONString(results));
+
+            Prompt prompt = PromptTemplate.from(LLM_FILTER_PROMPT).apply(variable);
+            ChatLanguageModel chatLanguageModel = ModelProvider.getChatModel();
+            String response = chatLanguageModel.generate(prompt.toUserMessage().singleText());
+
+            if (StringUtils.isBlank(response)) {
+                results.clear();
+            } else {
+                List<String> retrievedIds = JSONObject.parseArray(response, String.class);
+                results = results.stream().filter(t -> retrievedIds.contains(t.getId()))
+                        .collect(Collectors.toSet());
+                results.forEach(r -> r.setLlmMatched(true));
+            }
+        }
+
         return new ArrayList<>(results);
     }
 
+    /**
+     * Create a task for batch processing
+     *
+     * @param chatQueryContext The context of the chat query
+     * @param detectDataSetIds Target dataset IDs
+     * @param queryTextsSub Sub-list of query texts to process
+     * @param results Shared result set for collecting results
+     * @param useLlm Whether to use LLM
+     * @return Callable task
+     */
     private Callable<Void> createTask(ChatQueryContext chatQueryContext, Set<Long> detectDataSetIds,
-            List<String> queryTextsSub, Set<EmbeddingResult> results) {
+            List<String> queryTextsSub, Set<EmbeddingResult> results, boolean useLlm) {
         return () -> {
-            List<EmbeddingResult> oneRoundResults =
-                    detectByQueryTextsSub(detectDataSetIds, queryTextsSub, chatQueryContext);
+            List<EmbeddingResult> oneRoundResults = detectByQueryTextsSub(detectDataSetIds,
+                    queryTextsSub, chatQueryContext, useLlm);
             synchronized (results) {
                 selectResultInOneRound(results, oneRoundResults);
             }
@@ -73,57 +203,73 @@ public class EmbeddingMatchStrategy extends BatchMatchStrategy<EmbeddingResult>
         };
     }
 
+    /**
+     * Process a sub-list of query texts
+     *
+     * @param detectDataSetIds Target dataset IDs
+     * @param queryTextsSub Sub-list of query texts
+     * @param chatQueryContext Chat query context
+     * @param useLlm Whether to use LLM
+     * @return List of embedding results for this batch
+     */
     private List<EmbeddingResult> detectByQueryTextsSub(Set<Long> detectDataSetIds,
-            List<String> queryTextsSub, ChatQueryContext chatQueryContext) {
+            List<String> queryTextsSub, ChatQueryContext chatQueryContext, boolean useLlm) {
         Map<Long, List<Long>> modelIdToDataSetIds = chatQueryContext.getModelIdToDataSetIds();
+
+        // Get configuration parameters
         double threshold =
-                Double.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_THRESHOLD));
-
-        // step1. build query params
-        RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(queryTextsSub).build();
-
-        // step2. retrieveQuery by detectSegment
+                Double.parseDouble(mapperConfig.getParameterValue(EMBEDDING_MAPPER_THRESHOLD));
         int embeddingNumber =
-                Integer.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_NUMBER));
+                Integer.parseInt(mapperConfig.getParameterValue(EMBEDDING_MAPPER_NUMBER));
+        int embeddingRoundNumber =
+                Integer.parseInt(mapperConfig.getParameterValue(EMBEDDING_MAPPER_ROUND_NUMBER));
+
+        // Build and execute query
+        RetrieveQuery retrieveQuery = RetrieveQuery.builder().queryTextsList(queryTextsSub).build();
         List<RetrieveQueryResult> retrieveQueryResults = metaEmbeddingService.retrieveQuery(
                 retrieveQuery, embeddingNumber, modelIdToDataSetIds, detectDataSetIds);
 
         if (CollectionUtils.isEmpty(retrieveQueryResults)) {
-            return new ArrayList<>();
+            return Collections.emptyList();
         }
-        // step3. build EmbeddingResults
-        List<EmbeddingResult> collect = retrieveQueryResults.stream().map(retrieveQueryResult -> {
-            List<Retrieval> retrievals = retrieveQueryResult.getRetrieval();
-            if (CollectionUtils.isNotEmpty(retrievals)) {
-                retrievals.removeIf(retrieval -> {
-                    if (!retrieveQueryResult.getQuery().contains(retrieval.getQuery())) {
-                        return retrieval.getSimilarity() < threshold;
-                    }
-                    return false;
-                });
+
+        // Process results
+        List<EmbeddingResult> collect = retrieveQueryResults.stream().peek(result -> {
+            if (!useLlm && CollectionUtils.isNotEmpty(result.getRetrieval())) {
+                result.getRetrieval()
+                        .removeIf(retrieval -> !result.getQuery().contains(retrieval.getQuery())
+                                && retrieval.getSimilarity() < threshold);
             }
-            return retrieveQueryResult;
-        }).filter(retrieveQueryResult -> CollectionUtils
-                .isNotEmpty(retrieveQueryResult.getRetrieval()))
-                .flatMap(retrieveQueryResult -> retrieveQueryResult.getRetrieval().stream()
-                        .map(retrieval -> {
-                            EmbeddingResult embeddingResult = new EmbeddingResult();
-                            BeanUtils.copyProperties(retrieval, embeddingResult);
-                            embeddingResult.setDetectWord(retrieveQueryResult.getQuery());
-                            embeddingResult.setName(retrieval.getQuery());
-                            Map<String, String> convertedMap = retrieval.getMetadata().entrySet()
-                                    .stream().collect(Collectors.toMap(Map.Entry::getKey,
-                                            entry -> entry.getValue().toString()));
-                            embeddingResult.setMetadata(convertedMap);
-                            return embeddingResult;
-                        }))
+        }).filter(result -> CollectionUtils.isNotEmpty(result.getRetrieval()))
+                .flatMap(result -> result.getRetrieval().stream()
+                        .map(retrieval -> convertToEmbeddingResult(result, retrieval)))
                 .collect(Collectors.toList());
 
-        // step4. select mapResul in one round
-        int embeddingRoundNumber =
-                Integer.valueOf(mapperConfig.getParameterValue(EMBEDDING_MAPPER_ROUND_NUMBER));
-        int roundNumber = embeddingRoundNumber * queryTextsSub.size();
-        return collect.stream().sorted(Comparator.comparingDouble(EmbeddingResult::getSimilarity))
-                .limit(roundNumber).collect(Collectors.toList());
+        // Sort and limit results
+        return collect.stream()
+                .sorted(Comparator.comparingDouble(EmbeddingResult::getSimilarity).reversed())
+                .limit(embeddingRoundNumber * queryTextsSub.size()).collect(Collectors.toList());
+    }
+
+    /**
+     * Convert RetrieveQueryResult and Retrieval to EmbeddingResult
+     *
+     * @param queryResult The query result containing retrieval information
+     * @param retrieval The retrieval data to be converted
+     * @return Converted EmbeddingResult
+     */
+    private EmbeddingResult convertToEmbeddingResult(RetrieveQueryResult queryResult,
+            Retrieval retrieval) {
+        EmbeddingResult embeddingResult = new EmbeddingResult();
+        BeanUtils.copyProperties(retrieval, embeddingResult);
+        embeddingResult.setDetectWord(queryResult.getQuery());
+        embeddingResult.setName(retrieval.getQuery());
+
+        // Convert metadata to string values
+        Map<String, String> metadata = retrieval.getMetadata().entrySet().stream().collect(
+                Collectors.toMap(Map.Entry::getKey, entry -> String.valueOf(entry.getValue())));
+        embeddingResult.setMetadata(metadata);
+
+        return embeddingResult;
     }
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapFilter.java

@@ -7,12 +7,7 @@ import com.tencent.supersonic.headless.chat.ChatQueryContext;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
@@ -66,7 +61,7 @@ public class MapFilter {
             List<SchemaElementMatch> value = entry.getValue();
             if (!CollectionUtils.isEmpty(value)) {
                 value.removeIf(schemaElementMatch -> StringUtils
-                        .length(schemaElementMatch.getDetectWord()) <= 1);
+                        .length(schemaElementMatch.getDetectWord()) <= 1 && !schemaElementMatch.isLlmMatched());
             }
         }
     }
@@ -85,7 +80,7 @@ public class MapFilter {
     }
 
     public static void filterByQueryDataType(ChatQueryContext chatQueryContext,
-            Predicate<SchemaElement> needRemovePredicate) {
+                                             Predicate<SchemaElement> needRemovePredicate) {
         Map<Long, List<SchemaElementMatch>> dataSetElementMatches =
                 chatQueryContext.getMapInfo().getDataSetElementMatches();
         for (Map.Entry<Long, List<SchemaElementMatch>> entry : dataSetElementMatches.entrySet()) {

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/mapper/MapperConfig.java

@@ -57,4 +57,12 @@ public class MapperConfig extends ParameterConfig {
     public static final Parameter EMBEDDING_MAPPER_ROUND_NUMBER =
             new Parameter("s2.mapper.embedding.round.number", "10", "向量召回最小相似度阈值",
                     "向量召回相似度阈值在动态调整中的最低值", "number", "Mapper相关配置");
+
+    public static final Parameter EMBEDDING_MAPPER_USE_LLM =
+            new Parameter("s2.mapper.embedding.use-llm-enhance", "false", "使用LLM对召回的向量进行二次判断开关",
+                    "embedding的结果再通过一次LLM来筛选，这时候忽略各个向量阀值", "bool", "Mapper相关配置");
+
+    public static final Parameter EMBEDDING_MAPPER_ALLOWED_SEGMENT_NATURE =
+            new Parameter("s2.mapper.embedding.allowed-segment-nature", "['v', 'd', 'a']", "使用LLM召回二次处理时对问题分词词性的控制",
+                    "分词后允许的词性才会进行向量召回", "list", "Mapper相关配置");
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java

@@ -15,12 +15,19 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
+import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.util.CollectionUtils;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -97,15 +104,16 @@ public class QueryReqBuilder {
      * convert to QueryS2SQLReq
      *
      * @param querySql
-     * @param dataSetId
+     * @param dataSet
      * @return
      */
-    public static QuerySqlReq buildS2SQLReq(String querySql, Long dataSetId) {
+    public static QuerySqlReq buildS2SQLReq(String querySql, DataSetResp dataSet) {
         QuerySqlReq querySQLReq = new QuerySqlReq();
         if (Objects.nonNull(querySql)) {
             querySQLReq.setSql(querySql);
         }
-        querySQLReq.setDataSetId(dataSetId);
+        querySQLReq.setDataSetId(dataSet.getId());
+        querySQLReq.setDataSetName(dataSet.getName());
         return querySQLReq;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java

@@ -7,6 +7,7 @@ import com.tencent.supersonic.auth.api.authorization.request.QueryAuthResReq;
 import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResourceResp;
 import com.tencent.supersonic.auth.api.authorization.service.AuthService;
 import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
+import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
 import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.common.pojo.QueryAuthorization;
 import com.tencent.supersonic.common.pojo.User;
@@ -73,6 +74,15 @@ public class S2DataPermissionAspect {
         SemanticQueryReq queryReq = null;
         if (objects[0] instanceof SemanticQueryReq) {
             queryReq = (SemanticQueryReq) objects[0];
+            if (queryReq instanceof QuerySqlReq) {
+                QuerySqlReq sqlReq = (QuerySqlReq) queryReq;
+                if (sqlReq.getDataSetName() != null) {
+                    String escapedTable = SqlReplaceHelper.escapeTableName(sqlReq.getDataSetName());
+                    sqlReq.setSql(sqlReq.getSql().replaceAll(
+                            String.format(" %s ", sqlReq.getDataSetName()),
+                            String.format(" %s ", escapedTable)));
+                }
+            }
         }
         if (queryReq == null) {
             throw new InvalidArgumentException("queryReq is not Invalid");

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java

@@ -140,7 +140,8 @@ public class QueryUtils {
                 || type.equalsIgnoreCase("float") || type.equalsIgnoreCase("double")
                 || type.equalsIgnoreCase("real") || type.equalsIgnoreCase("numeric")
                 || type.toLowerCase().startsWith("decimal") || type.toLowerCase().startsWith("uint")
-                || type.toLowerCase().startsWith("int") || type.toLowerCase().equalsIgnoreCase("decfloat");
+                || type.toLowerCase().startsWith("int")
+                || type.toLowerCase().equalsIgnoreCase("decfloat");
     }
 
     private String getName(String nameEn) {

launchers/standalone/src/main/resources/application-postgres.yaml

@@ -1,15 +1,15 @@
 spring:
   datasource:
     driver-class-name: org.postgresql.Driver
-    url: jdbc:postgresql://${S2_DB_HOST:localhost}:${S2_DB_PORT:5432}/${S2_DB_DATABASE:postgres}?stringtype=unspecified
-    username: ${S2_DB_USER:postgres}
-    password: ${S2_DB_PASSWORD:postgres}
+    url: jdbc:postgresql://localhost:5432/postgres?stringtype=unspecified
+    username: postgres
+    password: postgres
   sql:
     init:
       continue-on-error: true
       mode: always
-      username: ${S2_DB_USER:postgres}
-      password: ${S2_DB_PASSWORD:postgres}
+      username: postgres
+      password: postgres
       schema-locations: classpath:db/schema-postgres.sql,classpath:db/schema-postgres-demo.sql
       data-locations: classpath:db/data-postgres.sql,classpath:db/data-postgres-demo.sql
 
@@ -18,9 +18,9 @@ s2:
     store:
       provider: PGVECTOR
       base:
-        url: ${S2_DB_HOST:127.0.0.1}
-      port: ${S2_DB_PORT:5432}
-      databaseName: ${S2_DB_DATABASE:postgres}
-      user: ${S2_DB_USER:postgres}
-      password: ${S2_DB_PASSWORD:postgres}
+        url: 127.0.0.1
+      port: 5432
+      databaseName: postgres
+      user: postgres
+      password: postgres
       dimension: 512

launchers/standalone/src/main/resources/s2-config.yaml

@@ -41,3 +41,5 @@ s2:
       threshold: 0.5
       min:
         threshold: 0.3
+    embedding:
+      use-llm-enhance: true

launchers/standalone/src/test/java/com/tencent/supersonic/headless/BaseTest.java

@@ -20,6 +20,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DomainDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DomainRepository;
+import com.tencent.supersonic.headless.server.service.DataSetService;
 import com.tencent.supersonic.headless.server.service.DatabaseService;
 import com.tencent.supersonic.headless.server.service.SchemaService;
 import com.tencent.supersonic.util.DataUtils;
@@ -46,6 +47,8 @@ public class BaseTest extends BaseApplication {
     private AgentService agentService;
     @Autowired
     protected DatabaseService databaseService;
+    @Autowired
+    protected DataSetService dataSetService;
 
     protected Agent agent;
     protected SemanticSchema schema;

launchers/standalone/src/test/java/com/tencent/supersonic/headless/TranslatorTest.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless;
 
 import com.tencent.supersonic.common.pojo.User;
 import com.tencent.supersonic.demo.S2VisitsDemo;
+import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp;
 import com.tencent.supersonic.headless.chat.utils.QueryReqBuilder;
 import org.junit.jupiter.api.BeforeEach;
@@ -18,14 +19,15 @@ import static org.junit.Assert.assertTrue;
 
 public class TranslatorTest extends BaseTest {
 
-    private Long dataSetId;
+    private DataSetResp dataSet;
 
     @BeforeEach
     public void init() {
         agent = getAgentByName(S2VisitsDemo.AGENT_NAME);
         schema = schemaService.getSemanticSchema(agent.getDataSetIds());
         if (Objects.nonNull(agent)) {
-            dataSetId = agent.getDataSetIds().stream().findFirst().get();
+            long dataSetId = agent.getDataSetIds().stream().findFirst().get();
+            dataSet = dataSetService.getDataSet(dataSetId);
         }
     }
 
@@ -34,7 +36,7 @@ public class TranslatorTest extends BaseTest {
         String sql =
                 "SELECT SUM(访问次数) AS _总访问次数_ FROM 超音数数据集 WHERE 数据日期 >= '2024-11-15' AND 数据日期 <= '2024-12-15'";
         SemanticTranslateResp explain = semanticLayerService
-                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSet), User.getDefaultUser());
         assertNotNull(explain);
         assertNotNull(explain.getQuerySQL());
         assertTrue(explain.getQuerySQL().contains("count(1)"));
@@ -45,7 +47,7 @@ public class TranslatorTest extends BaseTest {
     public void testSql_1() throws Exception {
         String sql = "SELECT 部门, SUM(访问次数) AS 总访问次数 FROM 超音数PVUV统计 GROUP BY 部门 ";
         SemanticTranslateResp explain = semanticLayerService
-                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSet), User.getDefaultUser());
         assertNotNull(explain);
         assertNotNull(explain.getQuerySQL());
         assertTrue(explain.getQuerySQL().contains("department"));
@@ -59,7 +61,7 @@ public class TranslatorTest extends BaseTest {
         String sql =
                 "WITH _department_visits_ AS (SELECT 部门, SUM(访问次数) AS _total_visits_ FROM 超音数数据集 WHERE 数据日期 >= '2024-11-15' AND 数据日期 <= '2024-12-15' GROUP BY 部门) SELECT 部门 FROM _department_visits_ ORDER BY _total_visits_ DESC LIMIT 2";
         SemanticTranslateResp explain = semanticLayerService
-                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSet), User.getDefaultUser());
         assertNotNull(explain);
         assertNotNull(explain.getQuerySQL());
         assertTrue(explain.getQuerySQL().toLowerCase().contains("department"));
@@ -73,7 +75,7 @@ public class TranslatorTest extends BaseTest {
         String sql =
                 "WITH recent_data AS (SELECT 用户名, 访问次数 FROM 超音数数据集 WHERE 部门 = 'marketing' AND 数据日期 >= '2024-12-01' AND 数据日期 <= '2024-12-15') SELECT 用户名 FROM recent_data ORDER BY 访问次数 DESC LIMIT 1";
         SemanticTranslateResp explain = semanticLayerService
-                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSet), User.getDefaultUser());
         assertNotNull(explain);
         assertNotNull(explain.getQuerySQL());
         assertTrue(explain.getQuerySQL().toLowerCase().contains("department"));
@@ -89,7 +91,7 @@ public class TranslatorTest extends BaseTest {
                         Paths.get(ClassLoader.getSystemResource("sql/testUnion.sql").toURI())),
                 StandardCharsets.UTF_8);
         SemanticTranslateResp explain = semanticLayerService
-                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSet), User.getDefaultUser());
         assertNotNull(explain);
         assertNotNull(explain.getQuerySQL());
         assertTrue(explain.getQuerySQL().contains("user_name"));
@@ -105,7 +107,7 @@ public class TranslatorTest extends BaseTest {
                         Paths.get(ClassLoader.getSystemResource("sql/testWith.sql").toURI())),
                 StandardCharsets.UTF_8);
         SemanticTranslateResp explain = semanticLayerService
-                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSet), User.getDefaultUser());
         assertNotNull(explain);
         assertNotNull(explain.getQuerySQL());
         executeSql(explain.getQuerySQL());
@@ -119,7 +121,7 @@ public class TranslatorTest extends BaseTest {
                         Paths.get(ClassLoader.getSystemResource("sql/testSubquery.sql").toURI())),
                 StandardCharsets.UTF_8);
         SemanticTranslateResp explain = semanticLayerService
-                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSetId), User.getDefaultUser());
+                .translate(QueryReqBuilder.buildS2SQLReq(sql, dataSet), User.getDefaultUser());
         assertNotNull(explain);
         assertNotNull(explain.getQuerySQL());
         executeSql(explain.getQuerySQL());