topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][project]Optimize log files and outputs.

Browse Source
This commit is contained in:
jerryjzhang
2024-10-18 14:17:47 +08:00
parent 55a4d5a198
commit 5c8b9a0ee1
9 changed files with 26 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMSqlCorrector.java
View File

@@ -29,7 +29,7 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
public static final String APP_KEY = "S2SQL_CORRECTOR";
private static final String INSTRUCTION = ""
+ "\n#Role: You are a senior data engineer experienced in writing SQL."
+ "#Role: You are a senior data engineer experienced in writing SQL."
+ "\n#Task: Your will be provided with a user question and the SQL written by a junior engineer,"
+ "please take a review and help correct it if necessary." + "\n#Rules: "
+ "\n1.ALWAYS follow the output format: `opinion=(POSITIVE|NEGATIVE),sql=(corrected sql if NEGATIVE; empty string if POSITIVE)`."
@@ -73,9 +73,8 @@ public class LLMSqlCorrector extends BaseSemanticCorrector {
AiServices.create(SemanticSqlExtractor.class, chatLanguageModel);
Prompt prompt = generatePrompt(chatQueryContext.getQueryText(), semanticParseInfo,
chatApp.getPrompt());
keyPipelineLog.info("LLMSqlCorrector reqPrompt:\n{}", prompt.text());
SemanticSql s2Sql = extractor.generateSemanticSql(prompt.toUserMessage().singleText());
keyPipelineLog.info("LLMSqlCorrector modelResp:\n{}", s2Sql);
keyPipelineLog.info("LLMSqlCorrector modelReq:\n{} \nmodelResp:\n{}", prompt.text(), s2Sql);
if ("NEGATIVE".equals(s2Sql.getOpinion()) && StringUtils.isNotBlank(s2Sql.getSql())) {
semanticParseInfo.getSqlInfo().setCorrectedS2SQL(s2Sql.getSql());
}

11
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java
View File

@@ -15,6 +15,8 @@ import dev.langchain4j.service.AiServices;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.HashMap;
@@ -26,9 +28,11 @@ import java.util.concurrent.ConcurrentHashMap;
@Slf4j
public class OnePassSCSqlGenStrategy extends SqlGenStrategy {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
public static final String APP_KEY = "S2SQL_PARSER";
public static final String INSTRUCTION = ""
+ "\n#Role: You are a data analyst experienced in SQL languages."
+ "#Role: You are a data analyst experienced in SQL languages."
+ "\n#Task: You will be provided with a natural language question asked by users,"
+ "please convert it to a SQL query so that relevant data could be returned "
+ "by executing the SQL query against underlying database." + "\n#Rules:"
@@ -66,7 +70,7 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy {
LLMResp llmResp = new LLMResp();
llmResp.setQuery(llmReq.getQueryText());
// 1.recall exemplars
keyPipelineLog.info("OnePassSCSqlGenStrategy llmReq:\n{}", llmReq);
log.info("OnePassSCSqlGenStrategy llmReq:\n{}", llmReq);
List<List<Text2SQLExemplar>> exemplarsList = promptHelper.getFewShotExemplars(llmReq);
// 2.generate sql generation prompt for each self-consistency inference
@@ -85,10 +89,9 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy {
// 3.perform multiple self-consistency inferences parallelly
Map<String, Prompt> output2Prompt = new ConcurrentHashMap<>();
prompt2Exemplar.keySet().parallelStream().forEach(prompt -> {
keyPipelineLog.info("OnePassSCSqlGenStrategy reqPrompt:\n{}", prompt.toUserMessage());
SemanticSql s2Sql = extractor.generateSemanticSql(prompt.toUserMessage().singleText());
output2Prompt.put(s2Sql.getSql(), prompt);
keyPipelineLog.info("OnePassSCSqlGenStrategy modelResp:\n{}", s2Sql.getSql());
keyPipelineLog.info("OnePassSCSqlGenStrategy modelReq:\n{} \nmodelResp:\n{}", prompt.text(), s2Sql);
});
// 4.format response.

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/SqlGenStrategy.java
View File

@@ -5,8 +5,6 @@ import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp;
import dev.langchain4j.model.chat.ChatLanguageModel;
import dev.langchain4j.provider.ModelProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -18,8 +16,6 @@ import org.springframework.stereotype.Service;
@Service
public abstract class SqlGenStrategy implements InitializingBean {
protected static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
@Autowired
protected PromptHelper promptHelper;

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/AliasGenerateHelper.java
View File

@@ -24,7 +24,7 @@ public class AliasGenerateHelper {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
private static final String NAME_ALIAS_INSTRUCTION = ""
+ "\n#Role: You are a professional data analyst specializing in metrics and dimensions."
+ "#Role: You are a professional data analyst specializing in metrics and dimensions."
+ "\n#Task: You will be provided with metadata about a metric or dimension, please help "
+ "generate a few aliases in the same language as its `fieldName`." + "\n#Rules:"
+ "1. Please do not generate aliases like xxx1, xxx2, xxx3."
@@ -57,9 +57,8 @@ public class AliasGenerateHelper {
variable.put("desc", desc);
Prompt prompt = PromptTemplate.from(NAME_ALIAS_INSTRUCTION).apply(variable);
keyPipelineLog.info("AliasGenerateHelper.generateNameAlias reqPrompt:{}", prompt.text());
String response = getChatCompletion(prompt);
keyPipelineLog.info("AliasGenerateHelper.generateNameAlias modelResp:{}", response);
keyPipelineLog.info("AliasGenerateHelper.generateAlias modelReq:\n{} \nmodelResp:\n{}", prompt.text(), response);
return response;
}
@@ -68,9 +67,10 @@ public class AliasGenerateHelper {
variable.put("values", json);
Prompt prompt = PromptTemplate.from(VALUE_ALIAS_INSTRUCTION).apply(variable);
keyPipelineLog.info("AliasGenerateHelper.generateValueAlias reqPrompt:{}", prompt.text());
String response = getChatCompletion(prompt);
keyPipelineLog.info("AliasGenerateHelper.generateValueAlias modelResp:{}", response);
keyPipelineLog.info("AliasGenerateHelper.generateValueAlias modelReq:\n{} " +
"\nmodelResp:\n{}", prompt.text(), response);
return response;
}

5
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ChatWorkflowEngine.java
View File

@@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@@ -32,7 +30,6 @@ import java.util.stream.Collectors;
@Slf4j
public class ChatWorkflowEngine {
private static final Logger keyPipelineLog = LoggerFactory.getLogger("keyPipeline");
private final List<SchemaMapper> schemaMappers = ComponentFactory.getSchemaMappers();
private final List<SemanticParser> semanticParsers = ComponentFactory.getSemanticParsers();
private final List<SemanticCorrector> semanticCorrectors =
@@ -150,7 +147,7 @@ public class ChatWorkflowEngine {
if (StringUtils.isNotBlank(explain.getErrMsg())) {
errorMsg.add(explain.getErrMsg());
}
keyPipelineLog.info(
log.info(
"SqlInfoProcessor results:\n"
+ "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
StringUtils.normalizeSpace(parseInfo.getSqlInfo().getParsedS2SQL()),

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java
View File

@@ -213,7 +213,7 @@ public class DataSetSchemaBuilder {
}
private static void setDefaultTimeFormat(SchemaElement dimToAdd,
DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
DimensionTimeTypeParams dimensionTimeTypeParams, String timeFormat) {
if (null != dimensionTimeTypeParams && TimeDimensionEnum.DAY.name()
.equalsIgnoreCase(dimensionTimeTypeParams.getTimeGranularity())) {
dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TIME_FORMAT,