From 7b580b7c9468affaee172504125ef51576388364 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Wed, 20 Dec 2023 22:45:20 +0800 Subject: [PATCH] (improvement)(chat) Introduce a separate log file to record key outputs in the pipeline. (#559) --- .../supersonic/chat/parser/JavaLLMProxy.java | 15 +++--- .../chat/parser/PythonLLMProxy.java | 9 +++- .../sql/llm/OnePassSCSqlGeneration.java | 11 +++- .../parser/sql/llm/OnePassSqlGeneration.java | 11 ++-- .../sql/llm/TwoPassSCSqlGeneration.java | 12 +++-- .../parser/sql/llm/TwoPassSqlGeneration.java | 15 ++++-- .../processor/parse/SqlInfoProcessor.java | 13 ++++- .../src/main/resources/logback-spring.xml | 51 ++++++++++++++----- .../src/main/resources/logback-spring.xml | 47 +++++++++++++---- 9 files changed, 142 insertions(+), 42 deletions(-) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/JavaLLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/JavaLLMProxy.java index 1375b0d1c..5af8e1d4c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/JavaLLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/JavaLLMProxy.java @@ -15,6 +15,8 @@ import dev.langchain4j.model.chat.ChatLanguageModel; import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; /** @@ -24,6 +26,8 @@ import org.springframework.stereotype.Component; @Component public class JavaLLMProxy implements LLMProxy { + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); + @Override public boolean isSkip(QueryContext queryContext) { ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); @@ -53,14 +57,13 @@ public class JavaLLMProxy implements LLMProxy { FunctionPromptGenerator promptGenerator = ContextUtils.getBean(FunctionPromptGenerator.class); + ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); String functionCallPrompt = promptGenerator.generateFunctionCallPrompt(functionReq.getQueryText(), functionReq.getPluginConfigs()); - - ChatLanguageModel chatLanguageModel = ContextUtils.getBean(ChatLanguageModel.class); - - String functionSelect = chatLanguageModel.generate(functionCallPrompt); - - return OutputFormat.functionCallParse(functionSelect); + keyPipelineLog.info("functionCallPrompt:{}", functionCallPrompt); + String response = chatLanguageModel.generate(functionCallPrompt); + keyPipelineLog.info("functionCall response:{}", response); + return OutputFormat.functionCallParse(response); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/PythonLLMProxy.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/PythonLLMProxy.java index c026cd321..51f052919 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/PythonLLMProxy.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/PythonLLMProxy.java @@ -14,6 +14,8 @@ import java.net.URI; import java.net.URL; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; @@ -30,6 +32,8 @@ import org.springframework.web.util.UriComponentsBuilder; @Component public class PythonLLMProxy implements LLMProxy { + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); + @Override public boolean isSkip(QueryContext queryContext) { LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class); @@ -41,9 +45,9 @@ public class PythonLLMProxy implements LLMProxy { } public LLMResp query2sql(LLMReq llmReq, String modelClusterKey) { - long startTime = System.currentTimeMillis(); log.info("requestLLM request, modelId:{},llmReq:{}", modelClusterKey, llmReq); + keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); try { LLMParserConfig llmParserConfig = ContextUtils.getBean(LLMParserConfig.class); @@ -57,6 +61,7 @@ public class PythonLLMProxy implements LLMProxy { log.info("requestLLM response,cost:{}, questUrl:{} \n entity:{} \n body:{}", System.currentTimeMillis() - startTime, url, entity, responseEntity.getBody()); + keyPipelineLog.info("LLMResp:{}", responseEntity.getBody()); return responseEntity.getBody(); } catch (Exception e) { log.error("requestLLM error", e); @@ -75,10 +80,12 @@ public class PythonLLMProxy implements LLMProxy { RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); try { log.info("requestFunction functionReq:{}", JsonUtil.toString(functionReq)); + keyPipelineLog.info("requestFunction functionReq:{}", JsonUtil.toString(functionReq)); ResponseEntity responseEntity = restTemplate.exchange(requestUrl, HttpMethod.POST, entity, FunctionResp.class); log.info("requestFunction responseEntity:{},cost:{}", responseEntity, System.currentTimeMillis() - startTime); + keyPipelineLog.info("response:{}", responseEntity.getBody()); return responseEntity.getBody(); } catch (Exception e) { log.error("requestFunction error", e); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSCSqlGeneration.java index 9c2b9cb72..f9450b345 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSCSqlGeneration.java @@ -17,6 +17,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -25,6 +27,7 @@ import org.springframework.stereotype.Service; @Slf4j public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); @Autowired private ChatLanguageModel chatLanguageModel; @@ -40,6 +43,7 @@ public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { @Override public Map generation(LLMReq llmReq, String modelClusterKey) { //1.retriever sqlExamples and generate exampleListPool + keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); List> sqlExamples = sqlExampleLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); @@ -52,8 +56,11 @@ public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { linkingSqlPromptPool.parallelStream().forEach(linkingSqlPrompt -> { Prompt prompt = PromptTemplate.from(JsonUtil.toString(linkingSqlPrompt)) .apply(new HashMap<>()); + keyPipelineLog.info("request prompt:{}", prompt.toSystemMessage()); Response response = chatLanguageModel.generate(prompt.toSystemMessage()); - llmResults.add(response.content().text()); + String result = response.content().text(); + llmResults.add(result); + keyPipelineLog.info("model response:{}", result); } ); //3.format response. @@ -64,7 +71,7 @@ public class OnePassSCSqlGeneration implements SqlGeneration, InitializingBean { List sqlList = llmResults.stream() .map(llmResult -> OutputFormat.getSql(llmResult)).collect(Collectors.toList()); Pair> sqlMap = OutputFormat.selfConsistencyVote(sqlList); - log.info("linkingMap result:{},sqlMap:{}", linkingMap, sqlMap); + keyPipelineLog.info("linkingMap:{} sqlMap:{}", linkingMap, sqlMap); return sqlMap.getRight(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSqlGeneration.java index 6a5d4dd28..1cb7d5959 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/OnePassSqlGeneration.java @@ -14,6 +14,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -22,6 +24,7 @@ import org.springframework.stereotype.Service; @Slf4j public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); @Autowired private ChatLanguageModel chatLanguageModel; @@ -37,6 +40,7 @@ public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { @Override public Map generation(LLMReq llmReq, String modelClusterKey) { //1.retriever sqlExamples + keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); List> sqlExamples = sqlExampleLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); @@ -44,15 +48,16 @@ public class OnePassSqlGeneration implements SqlGeneration, InitializingBean { String promptStr = sqlPromptGenerator.generatorLinkingAndSqlPrompt(llmReq, sqlExamples); Prompt prompt = PromptTemplate.from(JsonUtil.toString(promptStr)).apply(new HashMap<>()); + keyPipelineLog.info("request prompt:{}", prompt.toSystemMessage()); Response response = chatLanguageModel.generate(prompt.toSystemMessage()); - + String result = response.content().text(); + keyPipelineLog.info("model response:{}", result); //3.format response. - String llmResult = response.content().text(); String schemaLinkStr = OutputFormat.getSchemaLinks(response.content().text()); String sql = OutputFormat.getSql(response.content().text()); Map sqlMap = new HashMap<>(); sqlMap.put(sql, 1D); - log.info("llmResult:{},schemaLinkStr:{},sql:{}", llmResult, schemaLinkStr, sql); + keyPipelineLog.info("schemaLinkStr:{},sqlMap:{}", schemaLinkStr, sqlMap); return sqlMap; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSCSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSCSqlGeneration.java index 9519e3b44..1aeb9d4e9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSCSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSCSqlGeneration.java @@ -14,16 +14,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Service -@Slf4j public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); @Autowired private ChatLanguageModel chatLanguageModel; @@ -39,6 +40,7 @@ public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { @Override public Map generation(LLMReq llmReq, String modelClusterKey) { //1.retriever sqlExamples and generate exampleListPool + keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); List> sqlExamples = sqlExampleLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); @@ -51,8 +53,10 @@ public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { linkingPromptPool.parallelStream().forEach( linkingPrompt -> { Prompt prompt = PromptTemplate.from(JsonUtil.toString(linkingPrompt)).apply(new HashMap<>()); + keyPipelineLog.info("step one request prompt:{}", prompt.toSystemMessage()); Response linkingResult = chatLanguageModel.generate(prompt.toSystemMessage()); String result = linkingResult.content().text(); + keyPipelineLog.info("step one model response:{}", result); linkingResults.add(OutputFormat.getSchemaLink(result)); } ); @@ -63,13 +67,15 @@ public class TwoPassSCSqlGeneration implements SqlGeneration, InitializingBean { List sqlTaskPool = new CopyOnWriteArrayList<>(); sqlPromptPool.parallelStream().forEach(sqlPrompt -> { Prompt linkingPrompt = PromptTemplate.from(JsonUtil.toString(sqlPrompt)).apply(new HashMap<>()); + keyPipelineLog.info("step two request prompt:{}", linkingPrompt.toSystemMessage()); Response sqlResult = chatLanguageModel.generate(linkingPrompt.toSystemMessage()); String result = sqlResult.content().text(); + keyPipelineLog.info("step two model response:{}", result); sqlTaskPool.add(result); }); //4.format response. Pair> sqlMap = OutputFormat.selfConsistencyVote(sqlTaskPool); - log.info("linkingMap result:{},sqlMap:{}", linkingMap, sqlMap); + keyPipelineLog.info("linkingMap:{} sqlMap:{}", linkingMap, sqlMap); return sqlMap.getRight(); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSqlGeneration.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSqlGeneration.java index 75229f252..fba015ca0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSqlGeneration.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/sql/llm/TwoPassSqlGeneration.java @@ -14,6 +14,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -22,6 +24,7 @@ import org.springframework.stereotype.Service; @Slf4j public class TwoPassSqlGeneration implements SqlGeneration, InitializingBean { + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); @Autowired private ChatLanguageModel chatLanguageModel; @@ -36,23 +39,27 @@ public class TwoPassSqlGeneration implements SqlGeneration, InitializingBean { @Override public Map generation(LLMReq llmReq, String modelClusterKey) { - + keyPipelineLog.info("modelClusterKey:{},llmReq:{}", modelClusterKey, llmReq); List> sqlExamples = sqlExampleLoader.retrieverSqlExamples(llmReq.getQueryText(), optimizationConfig.getText2sqlCollectionName(), optimizationConfig.getText2sqlExampleNum()); String linkingPromptStr = sqlPromptGenerator.generateLinkingPrompt(llmReq, sqlExamples); Prompt prompt = PromptTemplate.from(JsonUtil.toString(linkingPromptStr)).apply(new HashMap<>()); + keyPipelineLog.info("step one request prompt:{}", prompt.toSystemMessage()); Response response = chatLanguageModel.generate(prompt.toSystemMessage()); - + keyPipelineLog.info("step one model response:{}", response.content().text()); String schemaLinkStr = OutputFormat.getSchemaLink(response.content().text()); - String generateSqlPrompt = sqlPromptGenerator.generateSqlPrompt(llmReq, schemaLinkStr, sqlExamples); Prompt sqlPrompt = PromptTemplate.from(JsonUtil.toString(generateSqlPrompt)).apply(new HashMap<>()); + keyPipelineLog.info("step two request prompt:{}", sqlPrompt.toSystemMessage()); Response sqlResult = chatLanguageModel.generate(sqlPrompt.toSystemMessage()); + String result = sqlResult.content().text(); + keyPipelineLog.info("step two model response:{}", result); Map sqlMap = new HashMap<>(); - sqlMap.put(sqlResult.content().text(), 1D); + sqlMap.put(result, 1D); + keyPipelineLog.info("schemaLinkStr:{},sqlMap:{}", schemaLinkStr, sqlMap); return sqlMap; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/processor/parse/SqlInfoProcessor.java b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/parse/SqlInfoProcessor.java index 703dc56d7..2d4b28fbd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/processor/parse/SqlInfoProcessor.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/processor/parse/SqlInfoProcessor.java @@ -6,8 +6,12 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; +import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.chat.query.QueryManager; +import com.tencent.supersonic.chat.query.llm.s2sql.LLMSqlQuery; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; import java.util.List; import java.util.Objects; @@ -19,6 +23,8 @@ import java.util.stream.Collectors; **/ public class SqlInfoProcessor implements ParseResultProcessor { + private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline"); + @Override public void process(ParseResp parseResp, QueryContext queryContext, ChatContext chatContext) { QueryReq queryReq = queryContext.getRequest(); @@ -52,7 +58,12 @@ public class SqlInfoProcessor implements ParseResultProcessor { if (StringUtils.isBlank(explainSql)) { return; } - parseInfo.getSqlInfo().setQuerySQL(explainSql); + SqlInfo sqlInfo = parseInfo.getSqlInfo(); + if (semanticQuery instanceof LLMSqlQuery) { + keyPipelineLog.info("s2sql:{}\ncorrectS2SQL:{}\nquerySQL:{}", sqlInfo.getS2SQL(), sqlInfo.getCorrectS2SQL(), + explainSql); + } + sqlInfo.setQuerySQL(explainSql); } } diff --git a/launchers/headless/src/main/resources/logback-spring.xml b/launchers/headless/src/main/resources/logback-spring.xml index 3e06f9e52..c18734316 100644 --- a/launchers/headless/src/main/resources/logback-spring.xml +++ b/launchers/headless/src/main/resources/logback-spring.xml @@ -3,7 +3,7 @@ logback - + @@ -29,7 +29,7 @@ UTF-8 - %d [%thread] %-5level [%X{TRACE_ID}] %logger{36} %line - %msg%n + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n @@ -46,7 +46,8 @@ - ${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + ${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + 90 @@ -55,16 +56,10 @@ UTF-8 - %d [%thread] %-5level [%X{TRACE_ID}] %logger{36} - %msg%n + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n - - - - - - - ${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + ${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + 30 @@ -83,11 +79,42 @@ UTF-8 - %d [%thread] %-5level [%X{TRACE_ID}] %logger{36} %line - %msg%n + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n + + + + ${LOG_PATH}/keyPipeline.log + + + + ${LOG_PATH}/keyPipeline.%d{yyyy-MM-dd}.log + + 30 + true + + + UTF-8 + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n + + + + INFO + + + + + + + + + + + + diff --git a/launchers/standalone/src/main/resources/logback-spring.xml b/launchers/standalone/src/main/resources/logback-spring.xml index 05ee6d7fe..c18734316 100644 --- a/launchers/standalone/src/main/resources/logback-spring.xml +++ b/launchers/standalone/src/main/resources/logback-spring.xml @@ -1,13 +1,13 @@ logback - + - %d{HH:mm:ss} [%thread] %-5level %logger{36} %line - %msg%n + %d{HH:mm:ss} [%thread] %-5level %logger{36} %line - %msg%n @@ -46,7 +46,8 @@ - ${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + ${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + 90 @@ -59,12 +60,6 @@ - - - - - - - ${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + ${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz + 30 @@ -90,4 +86,35 @@ + + + + ${LOG_PATH}/keyPipeline.log + + + + ${LOG_PATH}/keyPipeline.%d{yyyy-MM-dd}.log + + 30 + true + + + UTF-8 + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n + + + + INFO + + + + + + + + + + + +