[improvement][project]Optimize log files and outputs.

chat/server/src/main/java/com/tencent/supersonic/chat/server/memory/MemoryReviewTask.java

@@ -33,7 +33,7 @@ public class MemoryReviewTask {
 
     public static final String APP_KEY = "MEMORY_REVIEW";
     private static final String INSTRUCTION = ""
-            + "\n#Role: You are a senior data engineer experienced in writing SQL."
+            + "#Role: You are a senior data engineer experienced in writing SQL."
             + "\n#Task: Your will be provided with a user question and the SQL written by a junior engineer,"
             + "please take a review and give your opinion." + "\n#Rules: "
             + "\n1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),comment=(your comment)`."
@@ -82,12 +82,11 @@ public class MemoryReviewTask {
         String promptStr = createPromptString(m, chatApp.getPrompt());
         Prompt prompt = PromptTemplate.from(promptStr).apply(Collections.EMPTY_MAP);
 
-        keyPipelineLog.info("MemoryReviewTask reqPrompt:\n{}", promptStr);
         ChatLanguageModel chatLanguageModel =
                 ModelProvider.getChatModel(ModelConfigHelper.getChatModelConfig(chatApp));
         if (Objects.nonNull(chatLanguageModel)) {
             String response = chatLanguageModel.generate(prompt.toUserMessage()).content().text();
-            keyPipelineLog.info("MemoryReviewTask modelResp:\n{}", response);
+            keyPipelineLog.info("MemoryReviewTask modelReq:\n{} \nmodelResp:\n{}", promptStr, response);
             processResponse(response, m);
         } else {
             log.debug("ChatLanguageModel not found for agent:{}", chatAgent.getId());

chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java

@@ -203,13 +203,11 @@ public class NL2SQLParser implements ChatQueryParser {
         variables.put("history_sql", histSQL);
 
         Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variables);
-        keyPipelineLog.info("QueryRewrite reqPrompt:{}", prompt.text());
-
         ChatLanguageModel chatLanguageModel =
                 ModelProvider.getChatModel(ModelConfigHelper.getChatModelConfig(chatApp));
         Response<AiMessage> response = chatLanguageModel.generate(prompt.toUserMessage());
         String rewrittenQuery = response.content().text();
-        keyPipelineLog.info("QueryRewrite modelResp:{}", rewrittenQuery);
+        keyPipelineLog.info("QueryRewrite modelReq:\n{} \nmodelResp:\n{}", prompt.text(), response);
         parseContext.setQueryText(rewrittenQuery);
         QueryNLReq rewrittenQueryNLReq = QueryReqConverter.buildText2SqlQueryReq(parseContext);
         MapResp rewrittenQueryMapResult = chatLayerService.map(rewrittenQueryNLReq);
@@ -238,12 +236,11 @@ public class NL2SQLParser implements ChatQueryParser {
         variables.put("examples", exampleStr);
 
         Prompt prompt = PromptTemplate.from(chatApp.getPrompt()).apply(variables);
-        keyPipelineLog.info("ErrorRewrite reqPrompt:{}", prompt.text());
         ChatLanguageModel chatLanguageModel =
                 ModelProvider.getChatModel(ModelConfigHelper.getChatModelConfig(chatApp));
         Response<AiMessage> response = chatLanguageModel.generate(prompt.toUserMessage());
         String rewrittenMsg = response.content().text();
-        keyPipelineLog.info("ErrorRewrite modelResp:{}", rewrittenMsg);
+        keyPipelineLog.info("ErrorRewrite modelReq:\n{} \nmodelResp:\n{}", prompt.text(), response);
 
         return rewrittenMsg;
     }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java

@@ -29,7 +29,7 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
 
     public static final String APP_KEY = "S2SQL_CORRECTOR";
     private static final String INSTRUCTION = ""
-            + "\n#Role: You are a senior data engineer experienced in writing SQL."
+            + "#Role: You are a senior data engineer experienced in writing SQL."
             + "\n#Task: Your will be provided with a user question and the SQL written by a junior engineer,"
             + "please take a review and help correct it if necessary." + "\n#Rules: "
             + "\n1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),sql=(corrected sql if NEGATIVE; empty string if POSITIVE)`."
@@ -73,9 +73,8 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
                 AiServices.create(SemanticSqlExtractor.class, chatLanguageModel);
         Prompt prompt = generatePrompt(chatQueryContext.getQueryText(), semanticParseInfo,
                 chatApp.getPrompt());
-        keyPipelineLog.info("LLMSqlCorrector reqPrompt:\n{}", prompt.text());
         SemanticSql s2Sql = extractor.generateSemanticSql(prompt.toUserMessage().singleText());
-        keyPipelineLog.info("LLMSqlCorrector modelResp:\n{}", s2Sql);
+        keyPipelineLog.info("LLMSqlCorrector modelReq:\n{} \nmodelResp:\n{}", prompt.text(), s2Sql);
         if ("NEGATIVE".equals(s2Sql.getOpinion()) && StringUtils.isNotBlank(s2Sql.getSql())) {
             semanticParseInfo.getSqlInfo().setCorrectedS2SQL(s2Sql.getSql());
         }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java

@@ -15,6 +15,8 @@ import dev.langchain4j.service.AiServices;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 import java.util.HashMap;
@@ -26,9 +28,11 @@ import java.util.concurrent.ConcurrentHashMap;
 @Slf4j
 public class OnePassSCSqlGenStrategy extends SqlGenStrategy {
 
+    private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
+
     public static final String APP_KEY = "S2SQL_PARSER";
     public static final String INSTRUCTION = ""
-            + "\n#Role: You are a data analyst experienced in SQL languages."
+            + "#Role: You are a data analyst experienced in SQL languages."
             + "\n#Task: You will be provided with a natural language question asked by users,"
             + "please convert it to a SQL query so that relevant data could be returned "
             + "by executing the SQL query against underlying database." + "\n#Rules:"
@@ -66,7 +70,7 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy {
         LLMResp llmResp = new LLMResp();
         llmResp.setQuery(llmReq.getQueryText());
         // 1.recall exemplars
-        keyPipelineLog.info("OnePassSCSqlGenStrategy llmReq:\n{}", llmReq);
+        log.info("OnePassSCSqlGenStrategy llmReq:\n{}", llmReq);
         List<List<Text2SQLExemplar>> exemplarsList = promptHelper.getFewShotExemplars(llmReq);
 
         // 2.generate sql generation prompt for each self-consistency inference
@@ -85,10 +89,9 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy {
         // 3.perform multiple self-consistency inferences parallelly
         Map<String, Prompt> output2Prompt = new ConcurrentHashMap<>();
         prompt2Exemplar.keySet().parallelStream().forEach(prompt -> {
-            keyPipelineLog.info("OnePassSCSqlGenStrategy reqPrompt:\n{}", prompt.toUserMessage());
             SemanticSql s2Sql = extractor.generateSemanticSql(prompt.toUserMessage().singleText());
             output2Prompt.put(s2Sql.getSql(), prompt);
-            keyPipelineLog.info("OnePassSCSqlGenStrategy modelResp:\n{}", s2Sql.getSql());
+            keyPipelineLog.info("OnePassSCSqlGenStrategy modelReq:\n{} \nmodelResp:\n{}", prompt.text(), s2Sql);
         });
 
         // 4.format response.

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategy.java

@@ -5,8 +5,6 @@ import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq;
 import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp;
 import dev.langchain4j.model.chat.ChatLanguageModel;
 import dev.langchain4j.provider.ModelProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -18,8 +16,6 @@ import org.springframework.stereotype.Service;
 @Service
 public abstract class SqlGenStrategy implements InitializingBean {
 
-    protected static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
-
     @Autowired
     protected PromptHelper promptHelper;
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java

@@ -24,7 +24,7 @@ public class AliasGenerateHelper {
     private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
 
     private static final String NAME_ALIAS_INSTRUCTION = ""
-            + "\n#Role: You are a professional data analyst specializing in metrics and dimensions."
+            + "#Role: You are a professional data analyst specializing in metrics and dimensions."
             + "\n#Task: You will be provided with metadata about a metric or dimension, please help "
             + "generate a few aliases in the same language as its `fieldName`." + "\n#Rules:"
             + "1. Please do not generate aliases like xxx1, xxx2, xxx3."
@@ -57,9 +57,8 @@ public class AliasGenerateHelper {
         variable.put("desc", desc);
 
         Prompt prompt = PromptTemplate.from(NAME_ALIAS_INSTRUCTION).apply(variable);
-        keyPipelineLog.info("AliasGenerateHelper.generateNameAlias reqPrompt:{}", prompt.text());
         String response = getChatCompletion(prompt);
-        keyPipelineLog.info("AliasGenerateHelper.generateNameAlias modelResp:{}", response);
+        keyPipelineLog.info("AliasGenerateHelper.generateAlias modelReq:\n{} \nmodelResp:\n{}", prompt.text(), response);
         return response;
     }
 
@@ -68,9 +67,10 @@ public class AliasGenerateHelper {
         variable.put("values", json);
 
         Prompt prompt = PromptTemplate.from(VALUE_ALIAS_INSTRUCTION).apply(variable);
-        keyPipelineLog.info("AliasGenerateHelper.generateValueAlias reqPrompt:{}", prompt.text());
         String response = getChatCompletion(prompt);
-        keyPipelineLog.info("AliasGenerateHelper.generateValueAlias modelResp:{}", response);
+        keyPipelineLog.info("AliasGenerateHelper.generateValueAlias modelReq:\n{} " +
+                "\nmodelResp:\n{}", prompt.text(), response);
+
 
         return response;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java

@@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
@@ -32,7 +30,6 @@ import java.util.stream.Collectors;
 @Slf4j
 public class ChatWorkflowEngine {
 
-    private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
     private final List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
     private final List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
     private final List<SemanticCorrector> semanticCorrectors =
@@ -150,7 +147,7 @@ public class ChatWorkflowEngine {
                 if (StringUtils.isNotBlank(explain.getErrMsg())) {
                     errorMsg.add(explain.getErrMsg());
                 }
-                keyPipelineLog.info(
+                log.info(
                         "SqlInfoProcessor results:\n"
                                 + "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
                         StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()),

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java

@@ -213,7 +213,7 @@ public class DataSetSchemaBuilder {
     }
 
     private static void setDefaultTimeFormat(SchemaElement dimToAdd,
-                                             DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
+            DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
         if (null != dimensionTimeTypeParams && TimeDimensionEnum.DAY.name()
                 .equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
             dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT,

launchers/standalone/src/main/resources/logback-spring.xml

@@ -16,11 +16,11 @@
             如果同时有<File>和<FileNamePattern>，那么当天日志是<File>，明天会自动把今天
             的日志改名为今天的日期。即，<File> 的日志都是当天的。
         -->
-        <File>${LOG_PATH}/info.${LOG_APPNAME}.log</File>
+        <File>${LOG_PATH}/s2-info.log</File>
         <!--滚动策略，按照时间滚动 TimeBasedRollingPolicy-->
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <!--文件路径,定义了日志的切分方式——把每一天的日志归档到一个文件中,以防止日志填满整个磁盘空间-->
-            <FileNamePattern>${LOG_PATH}/info.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz</FileNamePattern>
+            <FileNamePattern>${LOG_PATH}/s2-info.%d{yyyy-MM-dd}.log.gz</FileNamePattern>
             <!--只保留最近30天的日志-->
             <maxHistory>30</maxHistory>
             <!--用来指定日志文件的上限大小，那么到了这个值，就会删除旧的日志-->
@@ -42,11 +42,11 @@
             如果同时有<File>和<FileNamePattern>，那么当天日志是<File>，明天会自动把今天
             的日志改名为今天的日期。即，<File> 的日志都是当天的。
         -->
-        <File>${LOG_PATH}/error.${LOG_APPNAME}.log</File>
+        <File>${LOG_PATH}/s2-error.log</File>
         <!--滚动策略，按照时间滚动 TimeBasedRollingPolicy-->
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <!--文件路径,定义了日志的切分方式——把每一天的日志归档到一个文件中,以防止日志填满整个磁盘空间-->
-            <FileNamePattern>${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz
+            <FileNamePattern>${LOG_PATH}/s2-error.%d{yyyy-MM-dd}.log.gz
             </FileNamePattern>
             <!--只保留最近90天的日志-->
             <maxHistory>90</maxHistory>
@@ -89,11 +89,11 @@
 
     <!-- 业务日志输出 -->
     <appender name="keyPipelineAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <File>${LOG_PATH}/keyPipeline.log</File>
+        <File>${LOG_PATH}/s2-llm.log</File>
         <!-- 循环政策：基于时间创建日志文件 -->
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <!-- 日志文件名格式 -->
-            <fileNamePattern>${LOG_PATH}/keyPipeline.%d{yyyy-MM-dd}.log</fileNamePattern>
+            <fileNamePattern>${LOG_PATH}/s2-llm.%d{yyyy-MM-dd}.log</fileNamePattern>
             <!--只保留最近30天的日志-->
             <maxHistory>30</maxHistory>
             <cleanHistoryOnStart>true</cleanHistoryOnStart>