From 87355533b45eafd00760b3263fb54fe8dd213cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=AF=E6=85=95=E7=81=B5?= <1985312383@qq.com> Date: Wed, 11 Jun 2025 09:20:09 +0800 Subject: [PATCH 01/16] fix windows daemon.bat path configuration --- assembly/bin/supersonic-daemon.bat | 60 +++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/assembly/bin/supersonic-daemon.bat b/assembly/bin/supersonic-daemon.bat index 0b6b66789..388bb9748 100644 --- a/assembly/bin/supersonic-daemon.bat +++ b/assembly/bin/supersonic-daemon.bat @@ -20,7 +20,9 @@ if "%profile%"=="" ( set "model_name=%service%" -cd %baseDir% +REM fix path configuration - point to the correct release package directory +set "releaseDir=%buildDir%\supersonic-%service%-1.0.0-SNAPSHOT" +cd %releaseDir% if "%command%"=="restart" ( call :stop @@ -50,20 +52,58 @@ if "%command%"=="restart" ( :runJavaService echo 'java service starting, see logs in logs/' - set "libDir=%baseDir%\lib" - set "confDir=%baseDir%\conf" - set "webDir=%baseDir%\webapp" - set "logDir=%baseDir%\logs" - set "classpath=%baseDir%;%webDir%;%libDir%\*;%confDir%" - set "property=-Dfile.encoding=UTF-8 -Duser.language=Zh -Duser.region=CN -Duser.timezone=GMT+08 -Dspring.profiles.active=%profile%" - set "java-command=%property% -Xms1024m -Xmx2048m -cp %CLASSPATH% %MAIN_CLASS%" + echo 'Using release directory: %releaseDir%' + + REM use release package directory as base path + set "libDir=%releaseDir%\lib" + set "confDir=%releaseDir%\conf" + set "webDir=%releaseDir%\webapp" + set "logDir=%releaseDir%\logs" + + REM fix variable name matching problem + set "CLASSPATH=%releaseDir%;%webDir%;%libDir%\*;%confDir%" + set "MAIN_CLASS=%main_class%" + + REM add port configuration + set "property=-Dfile.encoding=UTF-8 -Duser.language=Zh -Duser.region=CN -Duser.timezone=GMT+08 -Dspring.profiles.active=%profile% -Dserver.port=9080" + set "java_command=%property% -Xms1024m -Xmx2048m -cp "%CLASSPATH%" %MAIN_CLASS%" + if not exist %logDir% mkdir %logDir% - start /B java %java-command% >nul 2>&1 - timeout /t 10 >nul + + REM check if the main jar file exists + if not exist "%libDir%\launchers-standalone-1.0.0-SNAPSHOT.jar" ( + echo "Error: Main jar file not found in %libDir%" + echo "Please make sure the application has been built and packaged correctly." + goto :EOF + ) + + echo 'Main Class: %MAIN_CLASS%' + echo 'Profile: %profile%' + echo 'Starting Java service...' + + REM start service and save logs + start /B java %java_command% > "%logDir%\supersonic.log" 2>&1 + timeout /t 15 >nul + + REM check service status + netstat -an | findstr ":9080" >nul + if errorlevel 1 ( + echo "Warning: Port 9080 is not listening" + echo "Please check the log file: %logDir%\supersonic.log" + if exist "%logDir%\supersonic.log" ( + echo "Recent log entries:" + powershell -Command "Get-Content '%logDir%\supersonic.log' | Select-Object -Last 10" + ) + ) else ( + echo "Service started successfully on port 9080" + echo "You can access the application at: http://localhost:9080" + ) + echo 'java service started' goto :EOF :stopJavaService + echo 'Stopping Java service...' for /f "tokens=2" %%i in ('tasklist ^| findstr /i "java"') do ( taskkill /PID %%i /F echo "java service (PID = %%i) is killed." From 019d737f0756f872b1724e2a6dd74af241f4b3f6 Mon Sep 17 00:00:00 2001 From: Willy-J <32266792+Willy-J@users.noreply.github.com> Date: Wed, 18 Jun 2025 15:03:40 +0800 Subject: [PATCH 02/16] (fix)(headless) add empty check for metric alias to prevent incorrect DataFormatType assignment --- .../tencent/supersonic/headless/server/utils/QueryUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 75f3ca44f..e6b89a7fd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -109,7 +109,7 @@ public class QueryUtils { column.setModelId(metric.getModelId()); } // if column nameEn contains metric alias, use metric dataFormatType - if (column.getDataFormatType() == null && metric.getAlias() != null) { + if (column.getDataFormatType() == null && StringUtils.isNotEmpty(metric.getAlias())) { for (String alias : metric.getAlias().split(",")) { if (nameEn.contains(alias)) { column.setDataFormatType(metric.getDataFormatType()); From f899d23b6320719b29c71cf1a894903ea1168e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=AF=E6=85=95=E7=81=B5?= <1985312383@qq.com> Date: Sat, 21 Jun 2025 04:57:04 +0800 Subject: [PATCH 03/16] add new chat corrector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在助理最终执行物理SQL前，加入一步LLM优化性能功能 --- .../chat/server/executor/SqlExecutor.java | 10 +- .../server/service/impl/AgentServiceImpl.java | 2 + .../service/impl/ChatQueryServiceImpl.java | 2 + .../supersonic/headless/api/pojo/SqlInfo.java | 3 + .../api/pojo/enums/ChatWorkflowState.java | 1 + .../corrector/LLMPhysicalSqlCorrector.java | 97 ++++++++++++++++++ .../chat/parser/llm/PromptHelper.java | 3 +- .../server/utils/ChatWorkflowEngine.java | 26 +++++ .../main/resources/META-INF/spring.factories | 3 +- .../main/resources/META-INF/spring.factories | 3 +- pom.xml | 10 +- webapp/package.json | 5 + webapp/packages/chat-sdk/src/common/type.ts | 1 + .../src/components/ChatItem/SqlItem.tsx | 99 ++++++++++--------- 14 files changed, 214 insertions(+), 51 deletions(-) create mode 100644 headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java create mode 100644 webapp/package.json diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java index 0871dbae8..02a5df72d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/executor/SqlExecutor.java @@ -75,8 +75,12 @@ public class SqlExecutor implements ChatQueryExecutor { return null; } - QuerySqlReq sqlReq = - QuerySqlReq.builder().sql(parseInfo.getSqlInfo().getCorrectedS2SQL()).build(); + // 使用querySQL，它已经包含了所有修正（包括物理SQL修正） + String finalSql = StringUtils.isNotBlank(parseInfo.getSqlInfo().getQuerySQL()) + ? parseInfo.getSqlInfo().getQuerySQL() + : parseInfo.getSqlInfo().getCorrectedS2SQL(); + + QuerySqlReq sqlReq = QuerySqlReq.builder().sql(finalSql).build(); sqlReq.setSqlInfo(parseInfo.getSqlInfo()); sqlReq.setDataSetId(parseInfo.getDataSetId()); @@ -90,7 +94,7 @@ public class SqlExecutor implements ChatQueryExecutor { queryResult.setQueryTimeCost(System.currentTimeMillis() - startTime); if (queryResp != null) { queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); - queryResult.setQuerySql(queryResp.getSql()); + queryResult.setQuerySql(finalSql); queryResult.setQueryResults(queryResp.getResultList()); queryResult.setQueryColumns(queryResp.getColumns()); queryResult.setQueryState(QueryState.SUCCESS); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index 2d76875bb..f09a4e30d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -39,6 +40,7 @@ public class AgentServiceImpl extends ServiceImpl implem private MemoryService memoryService; @Autowired + @Lazy private ChatQueryService chatQueryService; @Autowired diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java index 484e22305..1aca41e2d 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatQueryServiceImpl.java @@ -49,6 +49,7 @@ import net.sf.jsqlparser.schema.Column; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -66,6 +67,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { @Autowired private SemanticLayerService semanticLayerService; @Autowired + @Lazy private AgentService agentService; private final List chatQueryParsers = ComponentFactory.getChatParsers(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java index 3eab2bccf..ede33c71a 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java @@ -16,4 +16,7 @@ public class SqlInfo implements Serializable { // SQL to be executed finally private String querySQL; + + // Physical SQL corrected by LLM for performance optimization + private String correctedQuerySQL; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java index 953f1f020..79ba4d77d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ChatWorkflowState.java @@ -8,5 +8,6 @@ public enum ChatWorkflowState { VALIDATING, SQL_CORRECTING, PROCESSING, + PHYSICAL_SQL_CORRECTING, FINISHED } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java new file mode 100644 index 000000000..48a61f9ad --- /dev/null +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java @@ -0,0 +1,97 @@ +package com.tencent.supersonic.headless.chat.corrector; + +import com.tencent.supersonic.common.pojo.ChatApp; +import com.tencent.supersonic.common.pojo.enums.AppModule; +import com.tencent.supersonic.common.util.ChatAppManager; +import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.chat.ChatQueryContext; +import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.input.Prompt; +import dev.langchain4j.model.input.PromptTemplate; +import dev.langchain4j.model.output.structured.Description; +import dev.langchain4j.provider.ModelProvider; +import dev.langchain4j.service.AiServices; +import lombok.Data; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * 物理SQL修正器 - 使用LLM优化物理SQL性能 + */ +@Slf4j +public class LLMPhysicalSqlCorrector extends BaseSemanticCorrector { + + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); + + public static final String APP_KEY = "PHYSICAL_SQL_CORRECTOR"; + private static final String INSTRUCTION = "" + + "#Role: You are a senior database performance optimization expert experienced in SQL tuning." + + "\n\n#Task: You will be provided with a user question and the corresponding physical SQL query," + + " please analyze and optimize this SQL to improve query performance." + "\n\n#Rules:" + + "\n1. ALWAYS add appropriate index hints if the database supports them." + + "\n2. Optimize JOIN order by placing smaller tables first." + + "\n3. Add reasonable query limits to prevent large result sets if no LIMIT exists." + + "\n4. Optimize WHERE condition order by placing high-selectivity conditions first." + + "\n5. Ensure the optimized SQL is syntactically correct and logically equivalent." + + "\n6. If the SQL is already well-optimized, return 'positive'." + + "\n\n#Question: {{question}}" + "\n\n#OriginalSQL: {{sql}}" + "\n\n#Response:"; + + public LLMPhysicalSqlCorrector() { + ChatAppManager.register(APP_KEY, ChatApp.builder().prompt(INSTRUCTION).name("物理SQL修正") + .appModule(AppModule.CHAT).description("通过大模型对物理SQL做性能优化").enable(false).build()); + } + + @Data + @ToString + static class PhysicalSql { + @Description("either positive or negative") + private String opinion; + + @Description("optimized sql if negative") + private String sql; + } + + interface PhysicalSqlExtractor { + PhysicalSql generatePhysicalSql(String text); + } + + @Override + public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) { + ChatApp chatApp = chatQueryContext.getRequest().getChatAppConfig().get(APP_KEY); + if (!chatQueryContext.getRequest().getText2SQLType().enableLLM() || Objects.isNull(chatApp) + || !chatApp.isEnable()) { + return; + } + + ChatLanguageModel chatLanguageModel = + ModelProvider.getChatModel(chatApp.getChatModelConfig()); + PhysicalSqlExtractor extractor = + AiServices.create(PhysicalSqlExtractor.class, chatLanguageModel); + Prompt prompt = generatePrompt(chatQueryContext.getRequest().getQueryText(), + semanticParseInfo, chatApp.getPrompt()); + PhysicalSql physicalSql = + extractor.generatePhysicalSql(prompt.toUserMessage().singleText()); + keyPipelineLog.info("LLMPhysicalSqlCorrector modelReq:\n{} \nmodelResp:\n{}", prompt.text(), + physicalSql); + if ("NEGATIVE".equalsIgnoreCase(physicalSql.getOpinion()) + && StringUtils.isNotBlank(physicalSql.getSql())) { + semanticParseInfo.getSqlInfo().setCorrectedQuerySQL(physicalSql.getSql()); + } + } + + private Prompt generatePrompt(String queryText, SemanticParseInfo semanticParseInfo, + String promptTemplate) { + Map variable = new HashMap<>(); + variable.put("question", queryText); + variable.put("sql", semanticParseInfo.getSqlInfo().getQuerySQL()); + + return PromptTemplate.from(promptTemplate).apply(variable); + } +} diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java index c5d935b3b..a319b8491 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java @@ -52,7 +52,8 @@ public class PromptHelper { for (int i = 0; i < selfConsistencyNumber; i++) { List shuffledList = new ArrayList<>(exemplars); // only shuffle the exemplars from config - List subList=shuffledList.subList(llmReq.getDynamicExemplars().size(),shuffledList.size()); + List subList = + shuffledList.subList(llmReq.getDynamicExemplars().size(), shuffledList.size()); Collections.shuffle(subList); results.add(shuffledList.subList(0, Math.min(shuffledList.size(), fewShotNumber))); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java index 6fcbb983a..044fff206 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp; import com.tencent.supersonic.headless.chat.ChatQueryContext; +import com.tencent.supersonic.headless.chat.corrector.LLMPhysicalSqlCorrector; import com.tencent.supersonic.headless.chat.corrector.SemanticCorrector; import com.tencent.supersonic.headless.chat.mapper.SchemaMapper; import com.tencent.supersonic.headless.chat.parser.SemanticParser; @@ -76,6 +77,10 @@ public class ChatWorkflowEngine { long start = System.currentTimeMillis(); performTranslating(queryCtx, parseResult); parseResult.getParseTimeCost().setSqlTime(System.currentTimeMillis() - start); + queryCtx.setChatWorkflowState(ChatWorkflowState.PHYSICAL_SQL_CORRECTING); + break; + case PHYSICAL_SQL_CORRECTING: + performPhysicalSqlCorrecting(queryCtx); queryCtx.setChatWorkflowState(ChatWorkflowState.FINISHED); break; default: @@ -162,4 +167,25 @@ public class ChatWorkflowEngine { parseResult.setErrorMsg(String.join("\n", errorMsg)); } } + + private void performPhysicalSqlCorrecting(ChatQueryContext queryCtx) { + List candidateQueries = queryCtx.getCandidateQueries(); + if (CollectionUtils.isNotEmpty(candidateQueries)) { + for (SemanticQuery semanticQuery : candidateQueries) { + for (SemanticCorrector corrector : semanticCorrectors) { + if (corrector instanceof LLMPhysicalSqlCorrector) { + corrector.correct(queryCtx, semanticQuery.getParseInfo()); + // 如果物理SQL被修正了，更新querySQL为修正后的版本 + SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); + if (StringUtils.isNotBlank(parseInfo.getSqlInfo().getCorrectedQuerySQL())) { + parseInfo.getSqlInfo().setQuerySQL(parseInfo.getSqlInfo().getCorrectedQuerySQL()); + log.info("Physical SQL corrected and updated querySQL: {}", + parseInfo.getSqlInfo().getQuerySQL()); + } + break; + } + } + } + } + } } diff --git a/launchers/headless/src/main/resources/META-INF/spring.factories b/launchers/headless/src/main/resources/META-INF/spring.factories index 0adcac744..cd5643357 100644 --- a/launchers/headless/src/main/resources/META-INF/spring.factories +++ b/launchers/headless/src/main/resources/META-INF/spring.factories @@ -14,7 +14,8 @@ com.tencent.supersonic.headless.chat.parser.SemanticParser=\ com.tencent.supersonic.headless.chat.corrector.SemanticCorrector=\ com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector,\ - com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector + com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector,\ + com.tencent.supersonic.headless.chat.corrector.LLMPhysicalSqlCorrector com.tencent.supersonic.headless.chat.knowledge.file.FileHandler=\ com.tencent.supersonic.headless.chat.knowledge.file.FileHandlerImpl diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 8b17394eb..c12dfcea5 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -15,7 +15,8 @@ com.tencent.supersonic.headless.chat.parser.SemanticParser=\ com.tencent.supersonic.headless.chat.corrector.SemanticCorrector=\ com.tencent.supersonic.headless.chat.corrector.RuleSqlCorrector,\ - com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector + com.tencent.supersonic.headless.chat.corrector.LLMSqlCorrector,\ + com.tencent.supersonic.headless.chat.corrector.LLMPhysicalSqlCorrector com.tencent.supersonic.headless.chat.knowledge.file.FileHandler=\ com.tencent.supersonic.headless.chat.knowledge.file.FileHandlerImpl diff --git a/pom.xml b/pom.xml index 8db6a8336..1a81d9238 100644 --- a/pom.xml +++ b/pom.xml @@ -10,9 +10,9 @@ ${revision} + common auth chat - common launchers headless @@ -31,6 +31,7 @@ 21 21 21 + 21 UTF-8 4.9 6.1.0 @@ -254,6 +255,13 @@ ${java.source.version} ${java.target.version} ${file.encoding} + + + org.projectlombok + lombok + ${lombok.version} + + diff --git a/webapp/package.json b/webapp/package.json new file mode 100644 index 000000000..cc1b48756 --- /dev/null +++ b/webapp/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "supersonic-chat-sdk": "link:packages/chat-sdk" + } +} diff --git a/webapp/packages/chat-sdk/src/common/type.ts b/webapp/packages/chat-sdk/src/common/type.ts index da70d4c51..e706311e4 100644 --- a/webapp/packages/chat-sdk/src/common/type.ts +++ b/webapp/packages/chat-sdk/src/common/type.ts @@ -79,6 +79,7 @@ export type SqlInfoType = { parsedS2SQL: string; correctedS2SQL: string; querySQL: string; + correctedQuerySQL?: string; }; export type ChatContextType = { diff --git a/webapp/packages/chat-sdk/src/components/ChatItem/SqlItem.tsx b/webapp/packages/chat-sdk/src/components/ChatItem/SqlItem.tsx index 96439d770..d33275683 100644 --- a/webapp/packages/chat-sdk/src/components/ChatItem/SqlItem.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatItem/SqlItem.tsx @@ -58,32 +58,28 @@ const SqlItem: React.FC = ({ const getSchemaMapText = () => { return ` Schema映射 -${schema?.fieldNameList?.length > 0 ? `名称：${schema.fieldNameList.join('、')}` : ''}${ - schema?.values?.length > 0 +${schema?.fieldNameList?.length > 0 ? `名称：${schema.fieldNameList.join('、')}` : ''}${schema?.values?.length > 0 ? ` 取值：${schema.values - .map((item: any) => { - return `${item.fieldName}: ${item.fieldValue}`; - }) - .join('、')}` + .map((item: any) => { + return `${item.fieldName}: ${item.fieldValue}`; + }) + .join('、')}` : '' - }${ - priorExts + }${priorExts ? ` 附加：${priorExts}` : '' - }${ - terms?.length > 0 + }${terms?.length > 0 ? ` 术语：${terms - .map((item: any) => { - return `${item.name}${item.alias?.length > 0 ? `(${item.alias.join(',')})` : ''}: ${ - item.description + .map((item: any) => { + return `${item.name}${item.alias?.length > 0 ? `(${item.alias.join(',')})` : ''}: ${item.description }`; - }) - .join('、')}` + }) + .join('、')}` : '' - } + } `; }; @@ -91,16 +87,16 @@ ${schema?.fieldNameList?.length > 0 ? `名称：${schema.fieldNameList.join('、 const getFewShotText = () => { return ` Few-shot示例${fewShots - .map((item: any, index: number) => { - return ` + .map((item: any, index: number) => { + return ` 示例${index + 1}： 问题：${item.question} SQL： ${format(item.sql)} `; - }) - .join('')} + }) + .join('')} `; }; @@ -120,6 +116,14 @@ ${format(sqlInfo.correctedS2SQL)} `; }; + const getCorrectedQuerySQLText = () => { + return ` +物理SQL修正 + +${format(sqlInfo.correctedQuerySQL || '')} +`; + }; + const getQuerySQLText = () => { return ` 最终执行SQL @@ -155,6 +159,9 @@ ${executeErrorMsg} if (sqlInfo.correctedS2SQL) { text += getCorrectedS2SQLText(); } + if (sqlInfo.correctedQuerySQL) { + text += getCorrectedQuerySQLText(); + } if (sqlInfo.querySQL) { text += getQuerySQLText(); } @@ -183,9 +190,8 @@ ${executeErrorMsg}
{llmReq && (
{ setSqlType(sqlType === 'schemaMap' ? '' : 'schemaMap'); }} @@ -195,9 +201,8 @@ ${executeErrorMsg} )} {fewShots.length > 0 && (
{ setSqlType(sqlType === 'fewShots' ? '' : 'fewShots'); }} @@ -207,9 +212,8 @@ ${executeErrorMsg} )} {sqlInfo.parsedS2SQL && (
{ setSqlType(sqlType === 'parsedS2SQL' ? '' : 'parsedS2SQL'); }} @@ -219,9 +223,8 @@ ${executeErrorMsg} )} {sqlInfo.correctedS2SQL && (
{ setSqlType(sqlType === 'correctedS2SQL' ? '' : 'correctedS2SQL'); }} @@ -229,16 +232,26 @@ ${executeErrorMsg} 修正S2SQL
)} + {sqlInfo.correctedQuerySQL && ( +
{ + setSqlType(sqlType === 'correctedQuerySQL' ? '' : 'correctedQuerySQL'); + }} + > + 物理SQL修正 +
+ )} {sqlInfo.querySQL && (
{ setSqlType(sqlType === 'querySQL' ? '' : 'querySQL'); }} > - 最终执行SQL + {sqlInfo.correctedQuerySQL ? '最终执行SQL' : '最终执行SQL'}
)}
{sqlType === 'schemaMap' && (
@@ -290,9 +302,8 @@ ${executeErrorMsg}
{terms .map((item: any) => { - return `${item.name}${ - item.alias?.length > 0 ? `(${item.alias.join(',')})` : '' - }: ${item.description}`; + return `${item.name}${item.alias?.length > 0 ? `(${item.alias.join(',')})` : '' + }: ${item.description}`; }) .join('、')}
From 164d2a9e2350aace6437dc9e776ce3c0852be4cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=AF=E6=85=95=E7=81=B5?= <1985312383@qq.com> Date: Sat, 21 Jun 2025 05:00:54 +0800 Subject: [PATCH 04/16] update supersonic-build.bat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加路径创建时的判断，增加环境检测，修正打包的路径问题，与supersonic-daemen.bat 命令，符合文档中的运行方式 --- assembly/bin/supersonic-build.bat | 77 +++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/assembly/bin/supersonic-build.bat b/assembly/bin/supersonic-build.bat index b554cbd34..94d44822d 100644 --- a/assembly/bin/supersonic-build.bat +++ b/assembly/bin/supersonic-build.bat @@ -43,10 +43,26 @@ if "%service%"=="webapp" ( call mvn -f %projectDir% clean package -DskipTests -Dspotless.skip=true IF ERRORLEVEL 1 ( ECHO Failed to build backend Java modules. + ECHO Please check Maven and Java versions are compatible. + ECHO Current Java: %JAVA_HOME% + ECHO Current Maven: %MAVEN_HOME% EXIT /B 1 ) + + REM extract and copy files to deployment directory + cd %projectDir%\launchers\%model_name%\target + if exist "launchers-%model_name%-%MVN_VERSION%-bin.tar.gz" ( + echo "Extracting launchers-%model_name%-%MVN_VERSION%-bin.tar.gz..." + tar -xf "launchers-%model_name%-%MVN_VERSION%-bin.tar.gz" + if exist "launchers-%model_name%-%MVN_VERSION%" ( + echo "Copying files to deployment directory..." + xcopy /E /Y "launchers-%model_name%-%MVN_VERSION%\*" "%buildDir%\supersonic-%model_name%-%MVN_VERSION%\" + ) + ) + copy /y %projectDir%\launchers\%model_name%\target\*.tar.gz %buildDir%\ echo "finished building supersonic-%model_name% service" + cd %baseDir% goto :EOF @@ -72,22 +88,55 @@ if "%service%"=="webapp" ( cd %buildDir% if exist %release_dir% rmdir /s /q %release_dir% if exist %release_dir%.zip del %release_dir%.zip - mkdir %release_dir% - rem package webapp - tar xvf supersonic-webapp.tar.gz - move /y supersonic-webapp webapp - echo {"env": ""} > webapp\supersonic.config.json - move /y webapp %release_dir% - rem package java service - tar xvf %service_name%-bin.tar.gz - for /d %%D in ("%service_name%\*") do ( - move "%%D" "%release_dir%" + + rem check if release directory already exists from buildJavaService + if exist %release_dir% ( + echo "Release directory already prepared by buildJavaService" + ) else ( + mkdir %release_dir% + + rem package java service + tar xvf %service_name%-bin.tar.gz 2>nul + if errorlevel 1 ( + echo "Warning: tar command failed, trying PowerShell extraction..." + powershell -Command "Expand-Archive -Path '%service_name%-bin.tar.gz' -DestinationPath '.' -Force" + ) + for /d %%D in ("%service_name%\*") do ( + move "%%D" "%release_dir%" + ) + rmdir /s /q %service_name% 2>nul ) + + rem package webapp + if exist supersonic-webapp.tar.gz ( + tar xvf supersonic-webapp.tar.gz 2>nul + if errorlevel 1 ( + echo "Warning: tar command failed, trying PowerShell extraction..." + powershell -Command "Expand-Archive -Path 'supersonic-webapp.tar.gz' -DestinationPath '.' -Force" + ) + move /y supersonic-webapp webapp + echo {"env": ""} > webapp\supersonic.config.json + move /y webapp %release_dir% + del supersonic-webapp.tar.gz 2>nul + ) + + rem verify deployment structure + if exist "%release_dir%\lib\launchers-%model_name%-%MVN_VERSION%.jar" ( + echo "Deployment structure verified successfully" + ) else ( + echo "Warning: Main jar file not found in deployment structure" + echo "Expected: %release_dir%\lib\launchers-%model_name%-%MVN_VERSION%.jar" + ) + rem generate zip file - powershell Compress-Archive -Path %release_dir% -DestinationPath %release_dir%.zip - del %service_name%-bin.tar.gz - del supersonic-webapp.tar.gz - rmdir /s /q %service_name% + powershell -Command "Compress-Archive -Path '%release_dir%' -DestinationPath '%release_dir%.zip' -Force" + if errorlevel 1 ( + echo "Warning: PowerShell compression failed, release directory still available: %release_dir%" + ) else ( + echo "Successfully created release package: %release_dir%.zip" + ) + + del %service_name%-bin.tar.gz 2>nul echo "finished packaging supersonic release" goto :EOF From 08133ccbfb592bca59224d38da9134ee31ee2e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=AF=E6=85=95=E7=81=B5?= <1985312383@qq.com> Date: Sat, 21 Jun 2025 05:19:42 +0800 Subject: [PATCH 05/16] Update LLMPhysicalSqlCorrector.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正为合适的prompt --- .../chat/corrector/LLMPhysicalSqlCorrector.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java index 48a61f9ad..2d04e8ba7 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java @@ -35,13 +35,14 @@ public class LLMPhysicalSqlCorrector extends BaseSemanticCorrector { + "#Role: You are a senior database performance optimization expert experienced in SQL tuning." + "\n\n#Task: You will be provided with a user question and the corresponding physical SQL query," + " please analyze and optimize this SQL to improve query performance." + "\n\n#Rules:" - + "\n1. ALWAYS add appropriate index hints if the database supports them." - + "\n2. Optimize JOIN order by placing smaller tables first." - + "\n3. Add reasonable query limits to prevent large result sets if no LIMIT exists." - + "\n4. Optimize WHERE condition order by placing high-selectivity conditions first." - + "\n5. Ensure the optimized SQL is syntactically correct and logically equivalent." - + "\n6. If the SQL is already well-optimized, return 'positive'." - + "\n\n#Question: {{question}}" + "\n\n#OriginalSQL: {{sql}}" + "\n\n#Response:"; + + "\n1. DO NOT add or introduce any new fields, columns, or aliases that are not in the original SQL." + + "\n2. Push WHERE conditions into JOIN ON clauses when possible to reduce intermediate result sets." + + "\n3. Optimize JOIN order by placing smaller tables or tables with selective conditions first." + + "\n4. For date range conditions, ensure they are applied as early as possible in the query execution." + + "\n5. Remove or comment out database-specific index hints (like USE INDEX) that may cause syntax errors." + + "\n6. ONLY modify the structure and order of existing elements, do not change field names or add new ones." + + "\n7. Ensure the optimized SQL is syntactically correct and logically equivalent to the original." + + "\n\n#Question: {{question}}" + "\n\n#OriginalSQL: {{sql}}"; public LLMPhysicalSqlCorrector() { ChatAppManager.register(APP_KEY, ChatApp.builder().prompt(INSTRUCTION).name("物理SQL修正") From 7e6639df8323fa048d9ac1f048fcc9c1a5f2381f Mon Sep 17 00:00:00 2001 From: guilinlewis <185641548@qq.com> Date: Mon, 23 Jun 2025 09:47:48 +0800 Subject: [PATCH 06/16] =?UTF-8?q?(improvement)(common|headless|chat|auth)?= =?UTF-8?q?=20=E9=89=B4=E6=9D=83=E4=BC=98=E5=8C=96=E4=B8=8E=E5=8F=AC?= =?UTF-8?q?=E5=9B=9E=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 修复生成的用户token 一生成就失效的问题 2 如果用户设置的token ，需校验是否数据库存在，因为用户可设置一年的token 有泄露风险 3 结果解析优化， 去除不可以解析的情况，解析问题需要改写后的问， 4 召回样例，用相似度，保住至少有一个样例是高相似度的 5 数据集召回，填加完全匹配格式筛选逻辑 --- .../adaptor/DefaultUserAdaptor.java | 3 +- .../authentication/utils/TokenService.java | 21 ++++++++++ .../execute/DataInterpretProcessor.java | 13 ++++++- .../com/hankcs/hanlp/LoadRemoveService.java | 2 +- .../common/pojo/Text2SQLExemplar.java | 2 + .../service/impl/ExemplarServiceImpl.java | 5 ++- .../headless/api/pojo/SemanticParseInfo.java | 18 +++++++-- .../chat/parser/llm/PromptHelper.java | 38 ++++++++++++++----- 8 files changed, 84 insertions(+), 18 deletions(-) diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java index 8e24db424..bd950736a 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java @@ -222,8 +222,9 @@ public class DefaultUserAdaptor implements UserAdaptor { new UserWithPassword(userDO.getId(), userDO.getName(), userDO.getDisplayName(), userDO.getEmail(), userDO.getPassword(), userDO.getIsAdmin()); + // 使用令牌名称作为生成key ，这样可以区分正常请求和api 请求，api 的令牌失效时间很长，需考虑令牌泄露的情况 String token = - tokenService.generateToken(UserWithPassword.convert(userWithPassword), expireTime); + tokenService.generateToken(UserWithPassword.convert(userWithPassword),"SysDbToken:"+name, (new Date().getTime() + expireTime)); UserTokenDO userTokenDO = saveUserToken(name, userName, token, expireTime); return convertUserToken(userTokenDO); } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java index a8b249602..dae100f11 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java @@ -6,7 +6,10 @@ import javax.crypto.spec.SecretKeySpec; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword; +import com.tencent.supersonic.auth.authentication.persistence.dataobject.UserTokenDO; +import com.tencent.supersonic.auth.authentication.persistence.repository.UserRepository; import com.tencent.supersonic.common.pojo.exception.AccessException; +import com.tencent.supersonic.common.util.ContextUtils; import io.jsonwebtoken.Claims; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; @@ -71,6 +74,7 @@ public class TokenService { return generateToken(UserWithPassword.convert(appUser), request); } + public Optional getClaims(HttpServletRequest request) { String token = request.getHeader(authenticationConfig.getTokenHttpHeaderKey()); String appKey = getAppKey(request); @@ -90,6 +94,13 @@ public class TokenService { public Optional getClaims(String token, String appKey) { try { + if(StringUtils.isNotBlank(appKey)&&appKey.startsWith("SysDbToken:")) {// 如果是配置的长期令牌，需校验数据库是否存在该配置 + UserRepository userRepository = ContextUtils.getBean(UserRepository.class); + UserTokenDO dbToken= userRepository.getUserTokenByName(appKey.substring("SysDbToken:".length())); + if(dbToken==null||!dbToken.getToken().equals(token.replace("Bearer ",""))) { + throw new AccessException("Token does not exist :" + appKey); + } + } String tokenSecret = getTokenSecret(appKey); Claims claims = Jwts.parser().setSigningKey(tokenSecret.getBytes(StandardCharsets.UTF_8)) @@ -122,6 +133,16 @@ public class TokenService { Map appKeyToSecretMap = authenticationConfig.getAppKeyToSecretMap(); String secret = appKeyToSecretMap.get(appKey); if (StringUtils.isBlank(secret)) { + if(StringUtils.isNotBlank(appKey)&&appKey.startsWith("SysDbToken:")) { // 是配置的长期令牌 + String realAppKey=appKey.substring("SysDbToken:".length()); + String tmp = "WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ=="; + if(tmp.length()<=realAppKey.length()) { + return realAppKey; + } + else{ + return realAppKey+tmp.substring(realAppKey.length()); + } + } throw new AccessException("get secret from appKey failed :" + appKey); } return secret; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java index fc0f12e99..28b3e5f72 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java @@ -47,7 +47,8 @@ public class DataInterpretProcessor implements ExecuteResultProcessor { Agent agent = executeContext.getAgent(); ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY); return Objects.nonNull(chatApp) && chatApp.isEnable() - && StringUtils.isNotBlank(executeContext.getResponse().getTextResult()); // 如果都没结果，则无法处理，直接跳过 + && StringUtils.isNotBlank(executeContext.getResponse().getTextResult()) // 如果都没结果，则无法处理 + && StringUtils.isBlank(executeContext.getResponse().getTextSummary()); // 如果已经有汇总的结果了，无法再次处理 } @Override @@ -57,7 +58,15 @@ public class DataInterpretProcessor implements ExecuteResultProcessor { ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY); Map variable = new HashMap<>(); - variable.put("question", executeContext.getRequest().getQueryText()); + String question = executeContext.getResponse().getTextResult();// 结果解析应该用改写的问题，因为改写的内容信息量更大 + if(executeContext.getParseInfo().getProperties()!=null&& + executeContext.getParseInfo().getProperties().containsKey("CONTEXT")){ + Map context = (Map) executeContext.getParseInfo().getProperties().get("CONTEXT"); + if(context.get("queryText")!=null&&"".equals(context.get("queryText"))){ + question = context.get("queryText").toString(); + } + } + variable.put("question", question); variable.put("data", queryResult.getTextResult()); Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variable); diff --git a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java index 3472a6277..aea4d2421 100644 --- a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java +++ b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java @@ -21,7 +21,7 @@ public class LoadRemoveService { List resultList = new ArrayList<>(value); if (!CollectionUtils.isEmpty(modelIdOrDataSetIds)) { resultList.removeIf(nature -> { - if (Objects.isNull(nature)) { + if (Objects.isNull(nature)||!nature.startsWith("_")) { // 系统的字典是以 _ 开头的， 过滤因引用外部字典导致的异常 return false; } Long id = getId(nature); diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java index d878c13c2..c4785b3a1 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java @@ -22,4 +22,6 @@ public class Text2SQLExemplar implements Serializable { private String dbSchema; private String sql; + + protected double similarity; // 传递相似度，可以作为样本筛选的依据 } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java index 8852375e0..a25b6d08a 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java @@ -72,7 +72,10 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner { embeddingService.retrieveQuery(collection, retrieveQuery, num); results.forEach(ret -> { ret.getRetrieval().forEach(r -> { - exemplars.add(JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class)); + Text2SQLExemplar tmp = //传递相似度，可以作为样本筛选的依据 + JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class); + tmp.setSimilarity(r.getSimilarity()); + exemplars.add(tmp); }); }); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java index d7c9df4d8..6c10e824b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import static com.tencent.supersonic.common.pojo.Constants.DEFAULT_DETAIL_LIMIT; import static com.tencent.supersonic.common.pojo.Constants.DEFAULT_METRIC_LIMIT; @@ -65,12 +66,23 @@ public class SemanticParseInfo implements Serializable { DataSetMatchResult mr2 = getDataSetMatchResult(o2.getElementMatches()); double difference = mr1.getMaxDatesetSimilarity() - mr2.getMaxDatesetSimilarity(); - if (difference == 0) { + if (Math.abs(difference) < 0.0005) { // 看完全匹配的个数，实践证明，可以用户输入规范后，该逻辑具有优势 + if (!o1.getDataSetId().equals(o2.getDataSetId())) { + List elementMatches1 = o1.getElementMatches().stream() + .filter(e -> e.getSimilarity() == 1).collect(Collectors.toList()); + List elementMatches2 = o2.getElementMatches().stream() + .filter(e -> e.getSimilarity() == 1).collect(Collectors.toList()); + if (elementMatches1.size() > elementMatches2.size()) { + return -1; + } else if (elementMatches1.size() < elementMatches2.size()) { + return 1; + } + } difference = mr1.getMaxMetricSimilarity() - mr2.getMaxMetricSimilarity(); - if (difference == 0) { + if (Math.abs(difference) < 0.0005) { difference = mr1.getTotalSimilarity() - mr2.getTotalSimilarity(); } - if (difference == 0) { + if (Math.abs(difference) < 0.0005) { difference = mr1.getMaxMetricUseCnt() - mr2.getMaxMetricUseCnt(); } } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java index a319b8491..f438e8a43 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java @@ -14,10 +14,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; +import java.util.*; +import java.util.stream.Collectors; import static com.tencent.supersonic.headless.chat.parser.ParserConfig.*; @@ -51,13 +49,33 @@ public class PromptHelper { // use random collection of exemplars for each self-consistency inference for (int i = 0; i < selfConsistencyNumber; i++) { List shuffledList = new ArrayList<>(exemplars); - // only shuffle the exemplars from config - List subList = - shuffledList.subList(llmReq.getDynamicExemplars().size(), shuffledList.size()); - Collections.shuffle(subList); - results.add(shuffledList.subList(0, Math.min(shuffledList.size(), fewShotNumber))); + List same = shuffledList.stream() // 相似度极高的话，先找出来 + .filter(e -> e.getSimilarity() > 0.989).collect(Collectors.toList()); + List noSame = shuffledList.stream() + .filter(e -> e.getSimilarity() <= 0.989).collect(Collectors.toList()); + if ((noSame.size() - same.size()) > fewShotNumber) {// 去除部分最低分 + noSame.sort(Comparator.comparingDouble(Text2SQLExemplar::getSimilarity)); + noSame = noSame.subList((noSame.size() - fewShotNumber) / 2, noSame.size()); + } + Text2SQLExemplar mostSimilar = noSame.get(noSame.size() - 1); + Collections.shuffle(noSame); + List ts; + if (same.size() > 0) {// 一样的话，必须作为提示语 + ts = new ArrayList<>(); + int needSize = Math.min(noSame.size() + same.size(), fewShotNumber); + if (needSize > same.size()) { + ts.addAll(noSame.subList(0, needSize - same.size())); + } + ts.addAll(same); + } else { // 至少要一个最像的 + ts = noSame.subList(0, Math.min(noSame.size(), fewShotNumber)); + if (!ts.contains(mostSimilar)) { + ts.remove(ts.size() - 1); + ts.add(mostSimilar); + } + } + results.add(ts); } - return results; } From 1faf84e3725c66a11b53442abecd569cb3fc1c6f Mon Sep 17 00:00:00 2001 From: guilinlewis <185641548@qq.com> Date: Mon, 23 Jun 2025 10:03:17 +0800 Subject: [PATCH 07/16] =?UTF-8?q?(improvement)(common|headless|chat|auth)?= =?UTF-8?q?=20=E9=89=B4=E6=9D=83=E4=BC=98=E5=8C=96=E4=B8=8E=E5=8F=AC?= =?UTF-8?q?=E5=9B=9E=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 修复生成的用户token 一生成就失效的问题 2 如果用户设置的token ，需校验是否数据库存在，因为用户可设置一年的token 有泄露风险 3 结果解析优化， 去除不可以解析的情况，解析问题需要改写后的问， 4 召回样例，用相似度，保住至少有一个样例是高相似度的 5 数据集召回，填加完全匹配格式筛选逻辑 --- .../adaptor/DefaultUserAdaptor.java | 5 +++-- .../repository/UserRepository.java | 2 ++ .../repository/impl/UserRepositoryImpl.java | 7 +++++++ .../authentication/utils/TokenService.java | 21 ++++++++++--------- .../execute/DataInterpretProcessor.java | 13 ++++++------ .../com/hankcs/hanlp/LoadRemoveService.java | 3 ++- .../common/pojo/Text2SQLExemplar.java | 2 +- .../service/impl/ExemplarServiceImpl.java | 2 +- .../headless/api/pojo/SemanticParseInfo.java | 2 +- .../chat/parser/llm/PromptHelper.java | 2 +- 10 files changed, 36 insertions(+), 23 deletions(-) diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java index bd950736a..abff472e5 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/adaptor/DefaultUserAdaptor.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import java.sql.Timestamp; +import java.util.Date; import java.util.List; import java.util.Optional; import java.util.Set; @@ -223,8 +224,8 @@ public class DefaultUserAdaptor implements UserAdaptor { userDO.getEmail(), userDO.getPassword(), userDO.getIsAdmin()); // 使用令牌名称作为生成key ，这样可以区分正常请求和api 请求，api 的令牌失效时间很长，需考虑令牌泄露的情况 - String token = - tokenService.generateToken(UserWithPassword.convert(userWithPassword),"SysDbToken:"+name, (new Date().getTime() + expireTime)); + String token = tokenService.generateToken(UserWithPassword.convert(userWithPassword), + "SysDbToken:" + name, (new Date().getTime() + expireTime)); UserTokenDO userTokenDO = saveUserToken(name, userName, token, expireTime); return convertUserToken(userTokenDO); } diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java index a15ceaab0..d217c5082 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/UserRepository.java @@ -21,6 +21,8 @@ public interface UserRepository { UserTokenDO getUserToken(Long tokenId); + UserTokenDO getUserTokenByName(String tokenName); + void deleteUserTokenByName(String userName); void deleteUserToken(Long tokenId); diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java index 511494cdd..106623b96 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/persistence/repository/impl/UserRepositoryImpl.java @@ -65,6 +65,13 @@ public class UserRepositoryImpl implements UserRepository { return userTokenDOMapper.selectById(tokenId); } + @Override + public UserTokenDO getUserTokenByName(String tokenName) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.lambda().eq(UserTokenDO::getName, tokenName); + return userTokenDOMapper.selectOne(queryWrapper); + } + @Override public void deleteUserTokenByName(String userName) { QueryWrapper queryWrapper = new QueryWrapper<>(); diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java index dae100f11..85b1804ff 100644 --- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java +++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/utils/TokenService.java @@ -94,10 +94,11 @@ public class TokenService { public Optional getClaims(String token, String appKey) { try { - if(StringUtils.isNotBlank(appKey)&&appKey.startsWith("SysDbToken:")) {// 如果是配置的长期令牌，需校验数据库是否存在该配置 + if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) {// 如果是配置的长期令牌，需校验数据库是否存在该配置 UserRepository userRepository = ContextUtils.getBean(UserRepository.class); - UserTokenDO dbToken= userRepository.getUserTokenByName(appKey.substring("SysDbToken:".length())); - if(dbToken==null||!dbToken.getToken().equals(token.replace("Bearer ",""))) { + UserTokenDO dbToken = + userRepository.getUserTokenByName(appKey.substring("SysDbToken:".length())); + if (dbToken == null || !dbToken.getToken().equals(token.replace("Bearer ", ""))) { throw new AccessException("Token does not exist :" + appKey); } } @@ -133,14 +134,14 @@ public class TokenService { Map appKeyToSecretMap = authenticationConfig.getAppKeyToSecretMap(); String secret = appKeyToSecretMap.get(appKey); if (StringUtils.isBlank(secret)) { - if(StringUtils.isNotBlank(appKey)&&appKey.startsWith("SysDbToken:")) { // 是配置的长期令牌 - String realAppKey=appKey.substring("SysDbToken:".length()); - String tmp = "WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ=="; - if(tmp.length()<=realAppKey.length()) { + if (StringUtils.isNotBlank(appKey) && appKey.startsWith("SysDbToken:")) { // 是配置的长期令牌 + String realAppKey = appKey.substring("SysDbToken:".length()); + String tmp = + "WIaO9YRRVt+7QtpPvyWsARFngnEcbaKBk783uGFwMrbJBaochsqCH62L4Kijcb0sZCYoSsiKGV/zPml5MnZ3uQ=="; + if (tmp.length() <= realAppKey.length()) { return realAppKey; - } - else{ - return realAppKey+tmp.substring(realAppKey.length()); + } else { + return realAppKey + tmp.substring(realAppKey.length()); } } throw new AccessException("get secret from appKey failed :" + appKey); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java index 28b3e5f72..047d3ddfe 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DataInterpretProcessor.java @@ -47,8 +47,8 @@ public class DataInterpretProcessor implements ExecuteResultProcessor { Agent agent = executeContext.getAgent(); ChatApp chatApp = agent.getChatAppConfig().get(APP_KEY); return Objects.nonNull(chatApp) && chatApp.isEnable() - && StringUtils.isNotBlank(executeContext.getResponse().getTextResult()) // 如果都没结果，则无法处理 - && StringUtils.isBlank(executeContext.getResponse().getTextSummary()); // 如果已经有汇总的结果了，无法再次处理 + && StringUtils.isNotBlank(executeContext.getResponse().getTextResult()) // 如果都没结果，则无法处理 + && StringUtils.isBlank(executeContext.getResponse().getTextSummary()); // 如果已经有汇总的结果了，无法再次处理 } @Override @@ -59,10 +59,11 @@ public class DataInterpretProcessor implements ExecuteResultProcessor { Map variable = new HashMap<>(); String question = executeContext.getResponse().getTextResult();// 结果解析应该用改写的问题，因为改写的内容信息量更大 - if(executeContext.getParseInfo().getProperties()!=null&& - executeContext.getParseInfo().getProperties().containsKey("CONTEXT")){ - Map context = (Map) executeContext.getParseInfo().getProperties().get("CONTEXT"); - if(context.get("queryText")!=null&&"".equals(context.get("queryText"))){ + if (executeContext.getParseInfo().getProperties() != null + && executeContext.getParseInfo().getProperties().containsKey("CONTEXT")) { + Map context = (Map) executeContext.getParseInfo() + .getProperties().get("CONTEXT"); + if (context.get("queryText") != null && "".equals(context.get("queryText"))) { question = context.get("queryText").toString(); } } diff --git a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java index aea4d2421..09ee8e0f2 100644 --- a/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java +++ b/common/src/main/java/com/hankcs/hanlp/LoadRemoveService.java @@ -21,7 +21,8 @@ public class LoadRemoveService { List resultList = new ArrayList<>(value); if (!CollectionUtils.isEmpty(modelIdOrDataSetIds)) { resultList.removeIf(nature -> { - if (Objects.isNull(nature)||!nature.startsWith("_")) { // 系统的字典是以 _ 开头的， 过滤因引用外部字典导致的异常 + if (Objects.isNull(nature) || !nature.startsWith("_")) { // 系统的字典是以 _ 开头的， + // 过滤因引用外部字典导致的异常 return false; } Long id = getId(nature); diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java index c4785b3a1..d50f54121 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Text2SQLExemplar.java @@ -23,5 +23,5 @@ public class Text2SQLExemplar implements Serializable { private String sql; - protected double similarity; // 传递相似度，可以作为样本筛选的依据 + protected double similarity; // 传递相似度，可以作为样本筛选的依据 } diff --git a/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java b/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java index a25b6d08a..31457dde1 100644 --- a/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java +++ b/common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java @@ -72,7 +72,7 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner { embeddingService.retrieveQuery(collection, retrieveQuery, num); results.forEach(ret -> { ret.getRetrieval().forEach(r -> { - Text2SQLExemplar tmp = //传递相似度，可以作为样本筛选的依据 + Text2SQLExemplar tmp = // 传递相似度，可以作为样本筛选的依据 JsonUtil.mapToObject(r.getMetadata(), Text2SQLExemplar.class); tmp.setSimilarity(r.getSimilarity()); exemplars.add(tmp); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java index 6c10e824b..9be8c5cec 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticParseInfo.java @@ -66,7 +66,7 @@ public class SemanticParseInfo implements Serializable { DataSetMatchResult mr2 = getDataSetMatchResult(o2.getElementMatches()); double difference = mr1.getMaxDatesetSimilarity() - mr2.getMaxDatesetSimilarity(); - if (Math.abs(difference) < 0.0005) { // 看完全匹配的个数，实践证明，可以用户输入规范后，该逻辑具有优势 + if (Math.abs(difference) < 0.0005) { // 看完全匹配的个数，实践证明，可以用户输入规范后，该逻辑具有优势 if (!o1.getDataSetId().equals(o2.getDataSetId())) { List elementMatches1 = o1.getElementMatches().stream() .filter(e -> e.getSimilarity() == 1).collect(Collectors.toList()); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java index f438e8a43..8cf15a01d 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java @@ -49,7 +49,7 @@ public class PromptHelper { // use random collection of exemplars for each self-consistency inference for (int i = 0; i < selfConsistencyNumber; i++) { List shuffledList = new ArrayList<>(exemplars); - List same = shuffledList.stream() // 相似度极高的话，先找出来 + List same = shuffledList.stream() // 相似度极高的话，先找出来 .filter(e -> e.getSimilarity() > 0.989).collect(Collectors.toList()); List noSame = shuffledList.stream() .filter(e -> e.getSimilarity() <= 0.989).collect(Collectors.toList()); From 75fc83010c9c1966cf93df8774c6b80a1af3fe3d Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Wed, 25 Jun 2025 22:22:12 +0800 Subject: [PATCH 08/16] [fix][heaadless]Fix dim value replacement. --- .../supersonic/headless/server/aspect/DimValueAspect.java | 3 ++- .../headless/server/utils/DataSetSchemaBuilder.java | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index 75a91e969..8af101fe2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -102,7 +102,7 @@ public class DimValueAspect { continue; } for (DimensionResp dimension : dimensions) { - if (!expression.getFieldName().equals(dimension.getName()) + if (!expression.getFieldName().equals(dimension.getBizName()) || CollectionUtils.isEmpty(dimension.getDimValueMaps())) { continue; } @@ -124,6 +124,7 @@ public class DimValueAspect { sql = SqlReplaceHelper.replaceValue(sql, filedNameToValueMap); log.debug("correctorSql after replacing:{}", sql); querySqlReq.setSql(sql); + querySqlReq.getSqlInfo().setQuerySQL(sql); Map> techNameToBizName = getTechNameToBizName(dimensions); SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java index 3befbb47d..1e57e5f05 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java @@ -19,9 +19,10 @@ public class DataSetSchemaBuilder { public static DataSetSchema build(DataSetSchemaResp resp) { DataSetSchema dataSetSchema = new DataSetSchema(); dataSetSchema.setQueryConfig(resp.getQueryConfig()); - SchemaElement dataSet = SchemaElement.builder().dataSetId(resp.getId()) - .dataSetName(resp.getName()).id(resp.getId()).name(resp.getName()) - .bizName(resp.getBizName()).type(SchemaElementType.DATASET).build(); + SchemaElement dataSet = + SchemaElement.builder().dataSetId(resp.getId()).dataSetName(resp.getName()) + .id(resp.getId()).name(resp.getName()).description(resp.getDescription()) + .bizName(resp.getBizName()).type(SchemaElementType.DATASET).build(); dataSetSchema.setDataSet(dataSet); dataSetSchema.setDatabaseType(resp.getDatabaseType()); dataSetSchema.setDatabaseVersion(resp.getDatabaseVersion()); From 320fcf04bd6e5aacc5726c930ad37a64178c5704 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Wed, 25 Jun 2025 23:14:16 +0800 Subject: [PATCH 09/16] [fix][heaadless]bizName of ModelResp is not necessarily equal to table name. --- .../tencent/supersonic/headless/server/utils/DictUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index ce6319931..f11a83bb4 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -274,6 +274,8 @@ public class DictUtils { private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { ModelResp model = modelService.getModel(dictItemResp.getModelId()); + String tableStr = model.getModelDetail().getTableQuery() != null ? model.getModelDetail().getTableQuery() + : "(" + model.getModelDetail().getSqlQuery() + ")"; String sqlPattern = "select %s,count(1) from %s %s group by %s order by count(1) desc limit %d"; String dimBizName = dictItemResp.getBizName(); @@ -288,7 +290,7 @@ public class DictUtils { } String sql = - String.format(sqlPattern, dimBizName, model.getBizName(), where, dimBizName, limit); + String.format(sqlPattern, dimBizName, tableStr, where, dimBizName, limit); Set modelIds = new HashSet<>(); modelIds.add(dictItemResp.getModelId()); QuerySqlReq querySqlReq = new QuerySqlReq(); From 76cc5ee1117a424767598c09458d6bbfd315a76c Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Fri, 27 Jun 2025 19:46:00 +0800 Subject: [PATCH 10/16] [opt][heaadless]Add user field to QueryStatement for consistency. --- .../supersonic/headless/core/pojo/QueryStatement.java | 2 ++ .../server/facade/service/impl/S2SemanticLayerService.java | 3 +++ .../headless/server/utils/ChatWorkflowEngine.java | 5 +++-- .../tencent/supersonic/headless/server/utils/DictUtils.java | 6 +++--- .../supersonic/headless/server/utils/QueryUtils.java | 3 ++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index 83ac4fbf9..9381cac43 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.core.pojo; +import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import lombok.Data; @@ -24,6 +25,7 @@ public class QueryStatement { private SemanticSchemaResp semanticSchema; private Integer limit = 1000; private Boolean isTranslated = false; + private User user; public boolean isOk() { return StringUtils.isBlank(errMsg) && StringUtils.isNotBlank(sql); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index c9739bf99..3d3969cb8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -296,6 +296,9 @@ public class S2SemanticLayerService implements SemanticLayerService { queryStatement.setSql(semanticQueryReq.getSqlInfo().getQuerySQL()); queryStatement.setIsTranslated(true); } + if (queryStatement != null) { + queryStatement.setUser(user); + } return queryStatement; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java index 044fff206..05b1d582a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java @@ -178,8 +178,9 @@ public class ChatWorkflowEngine { // 如果物理SQL被修正了，更新querySQL为修正后的版本 SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); if (StringUtils.isNotBlank(parseInfo.getSqlInfo().getCorrectedQuerySQL())) { - parseInfo.getSqlInfo().setQuerySQL(parseInfo.getSqlInfo().getCorrectedQuerySQL()); - log.info("Physical SQL corrected and updated querySQL: {}", + parseInfo.getSqlInfo() + .setQuerySQL(parseInfo.getSqlInfo().getCorrectedQuerySQL()); + log.info("Physical SQL corrected and updated querySQL: {}", parseInfo.getSqlInfo().getQuerySQL()); } break; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index f11a83bb4..59713f940 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -274,7 +274,8 @@ public class DictUtils { private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { ModelResp model = modelService.getModel(dictItemResp.getModelId()); - String tableStr = model.getModelDetail().getTableQuery() != null ? model.getModelDetail().getTableQuery() + String tableStr = model.getModelDetail().getTableQuery() != null + ? model.getModelDetail().getTableQuery() : "(" + model.getModelDetail().getSqlQuery() + ")"; String sqlPattern = "select %s,count(1) from %s %s group by %s order by count(1) desc limit %d"; @@ -289,8 +290,7 @@ public class DictUtils { limit = Integer.MAX_VALUE; } - String sql = - String.format(sqlPattern, dimBizName, tableStr, where, dimBizName, limit); + String sql = String.format(sqlPattern, dimBizName, tableStr, where, dimBizName, limit); Set modelIds = new HashSet<>(); modelIds.add(dictItemResp.getModelId()); QuerySqlReq querySqlReq = new QuerySqlReq(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index e6b89a7fd..83300a5f6 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -109,7 +109,8 @@ public class QueryUtils { column.setModelId(metric.getModelId()); } // if column nameEn contains metric alias, use metric dataFormatType - if (column.getDataFormatType() == null && StringUtils.isNotEmpty(metric.getAlias())) { + if (column.getDataFormatType() == null + && StringUtils.isNotEmpty(metric.getAlias())) { for (String alias : metric.getAlias().split(",")) { if (nameEn.contains(alias)) { column.setDataFormatType(metric.getDataFormatType()); From 79a44b27ee23ce9593844b49c8c198f14497f6b0 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Sat, 28 Jun 2025 08:51:04 +0800 Subject: [PATCH 11/16] [fix][heaadless]Fix NPE issues. --- .../tencent/supersonic/headless/server/utils/DictUtils.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 59713f940..5e95063e0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -138,7 +138,8 @@ public class DictUtils { semanticQueryReq.setNeedAuth(false); String bizName = dictItemResp.getBizName(); try { - SemanticQueryResp semanticQueryResp = queryService.queryByReq(semanticQueryReq, null); + SemanticQueryResp semanticQueryResp = + queryService.queryByReq(semanticQueryReq, User.getDefaultUser()); if (Objects.isNull(semanticQueryResp) || CollectionUtils.isEmpty(semanticQueryResp.getResultList())) { return lines; @@ -274,7 +275,7 @@ public class DictUtils { private QuerySqlReq constructQuerySqlReq(DictItemResp dictItemResp) { ModelResp model = modelService.getModel(dictItemResp.getModelId()); - String tableStr = model.getModelDetail().getTableQuery() != null + String tableStr = StringUtils.isNotBlank(model.getModelDetail().getTableQuery()) ? model.getModelDetail().getTableQuery() : "(" + model.getModelDetail().getSqlQuery() + ")"; String sqlPattern = From 6f497b142ec090a1b3f64cb96e744dc101f63d25 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Wed, 2 Jul 2025 18:59:56 +0800 Subject: [PATCH 12/16] [fix][chat]Memory deletion should disable it first. --- .../chat/server/service/impl/MemoryServiceImpl.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java index 6ceeb1127..5f0ee7cf5 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/MemoryServiceImpl.java @@ -26,7 +26,6 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Objects; @@ -110,6 +109,14 @@ public class MemoryServiceImpl implements MemoryService, CommandLineRunner { @Override public void batchDelete(List ids) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.lambda().in(ChatMemoryDO::getId, ids); + List chatMemoryDOS = chatMemoryRepository.getMemories(queryWrapper); + chatMemoryDOS.forEach(chatMemoryDO -> { + if (MemoryStatus.ENABLED.toString().equals(chatMemoryDO.getStatus().trim())) { + disableMemory(chatMemoryDO); + } + }); chatMemoryRepository.batchDelete(ids); } From 269f146c1182efe9b95e1d6129097e50c966eaae Mon Sep 17 00:00:00 2001 From: wangyong <48429126+wangyong00@users.noreply.github.com> Date: Mon, 7 Jul 2025 17:19:59 +0800 Subject: [PATCH 13/16] =?UTF-8?q?(fix)(headless-server)=E8=AF=AD=E6=84=8F?= =?UTF-8?q?=E5=BB=BA=E6=A8=A1-X=E6=A8=A1=E5=9E=8B-=E7=BB=B4=E5=BA=A6?= =?UTF-8?q?=E7=AE=A1=E7=90=86-=E7=BB=B4=E5=BA=A6=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E5=B8=A6key=E6=9F=A5=E8=AF=A2=E6=97=B6=EF=BC=8C=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E7=9A=84=E7=BB=B4=E5=BA=A6=E7=BB=93=E6=9E=9C=E8=8C=83?= =?UTF-8?q?=E5=9B=B4=E8=B6=85=E5=87=BA=E4=BA=86X=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E8=8C=83=E5=9B=B4=20(#2328)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../persistence/repository/impl/DimensionRepositoryImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java index 95c48d6f4..7ba0d3ebf 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DimensionRepositoryImpl.java @@ -83,10 +83,10 @@ public class DimensionRepositoryImpl implements DimensionRepository { } if (StringUtils.isNotBlank(dimensionFilter.getKey())) { String key = dimensionFilter.getKey(); - queryWrapper.lambda().like(DimensionDO::getName, key).or() + queryWrapper.and(qw->qw.lambda().like(DimensionDO::getName, key).or() .like(DimensionDO::getBizName, key).or().like(DimensionDO::getDescription, key) .or().like(DimensionDO::getAlias, key).or() - .like(DimensionDO::getCreatedBy, key); + .like(DimensionDO::getCreatedBy, key)); } return dimensionDOMapper.selectList(queryWrapper); From e408204690d4ec9c4ff1cb69d09b6a5634bfbbf3 Mon Sep 17 00:00:00 2001 From: mroldx <974751082@qq.com> Date: Wed, 9 Jul 2025 17:18:08 +0800 Subject: [PATCH 14/16] Update sql-update-mysql.sql (#2293) --- .../src/main/resources/config.update/sql-update-mysql.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchers/standalone/src/main/resources/config.update/sql-update-mysql.sql b/launchers/standalone/src/main/resources/config.update/sql-update-mysql.sql index b7b6c1b9b..cf92e1b0c 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update-mysql.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update-mysql.sql @@ -420,4 +420,4 @@ ALTER TABLE s2_chat_model add column is_open tinyint DEFAULT NULL COMMENT '是 ALTER TABLE s2_database add column is_open tinyint DEFAULT NULL COMMENT '是否公开'; --20250321 -ALTER TABLE s2_user add column last_loin datetime DEFAULT NULL; \ No newline at end of file +ALTER TABLE s2_user add column last_login datetime DEFAULT NULL; From 32675387d7b221b6cebbdfbb58de0d8efadd49a0 Mon Sep 17 00:00:00 2001 From: FredTsang <979738295@qq.com> Date: Wed, 9 Jul 2025 17:18:45 +0800 Subject: [PATCH 15/16] (fix)(chat-sdk) rm table defaultSortOrder (#2295) --- .../packages/chat-sdk/src/components/ChatMsg/Table/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/packages/chat-sdk/src/components/ChatMsg/Table/index.tsx b/webapp/packages/chat-sdk/src/components/ChatMsg/Table/index.tsx index 288906b24..afbe099bc 100644 --- a/webapp/packages/chat-sdk/src/components/ChatMsg/Table/index.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatMsg/Table/index.tsx @@ -24,7 +24,6 @@ const Table: React.FC = ({ data, size, loading, question, onApplyAuth }) dataIndex: bizName, key: bizName, title: name || bizName, - defaultSortOrder: 'descend', sorter: showType === 'NUMBER' ? (a, b) => { @@ -73,10 +72,11 @@ const Table: React.FC = ({ data, size, loading, question, onApplyAuth }) return index % 2 !== 0 ? `${prefixCls}-even-row` : ''; }; - const dateColumn = queryColumns.find(column => column.type === 'DATE'); + const dateColumn = queryColumns.find(column => column.type === 'DATE' || column.showType === 'DATE'); const dataSource = dateColumn ? queryResults.sort((a, b) => moment(a[dateColumn.bizName]).diff(moment(b[dateColumn.bizName]))) : queryResults; + return (
{question && ( From 11d1264d386bfdcde83c90170049ccbe777950e2 Mon Sep 17 00:00:00 2001 From: FredTsang <979738295@qq.com> Date: Wed, 9 Jul 2025 17:19:16 +0800 Subject: [PATCH 16/16] (fix)(chat-sdk): Export data based on queryColumns (#2297) --- webapp/packages/chat-sdk/src/components/ChatItem/index.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/webapp/packages/chat-sdk/src/components/ChatItem/index.tsx b/webapp/packages/chat-sdk/src/components/ChatItem/index.tsx index 9be689b7b..b8af45bff 100644 --- a/webapp/packages/chat-sdk/src/components/ChatItem/index.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatItem/index.tsx @@ -416,11 +416,10 @@ const ChatItem: React.FC = ({ const onExportData = () => { const { queryColumns, queryResults } = data || {}; - if (!!queryResults) { + if (!!queryResults && !!queryColumns) { const exportData = queryResults.map(item => { - return Object.keys(item).reduce((result, key) => { - const columnName = queryColumns?.find(column => column.nameEn === key)?.name || key; - result[columnName] = item[key]; + return queryColumns.reduce((result, column) => { + result[column.name || column.nameEn] = item[column.nameEn]; return result; }, {}); });