From bb4cd880b0bd7da005f9273e0d001405c5dff37f Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Thu, 4 Jul 2024 18:23:55 +0800 Subject: [PATCH] (improvement)(headless)Normalize space SQL strings in the log messages. --- .../chat/server/service/impl/AgentServiceImpl.java | 5 ++++- evaluation/{README_CN.md => README.md} | 0 .../headless/chat/corrector/GroupByCorrector.java | 4 ++-- .../headless/chat/parser/llm/LLMRequestService.java | 8 +------- .../headless/chat/parser/llm/LLMSqlParser.java | 4 ++-- .../headless/core/cache/DefaultQueryCache.java | 2 +- .../headless/core/executor/JdbcExecutor.java | 4 +--- .../translator/calcite/sql/render/JoinRender.java | 3 ++- .../headless/server/processor/SqlInfoProcessor.java | 13 ++++++++----- 9 files changed, 21 insertions(+), 22 deletions(-) rename evaluation/{README_CN.md => README.md} (100%) diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index c0147788d..682dbecab 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import java.util.List; import java.util.concurrent.ExecutorService; @@ -85,9 +86,11 @@ public class AgentServiceImpl extends ServiceImpl } private synchronized void doExecuteAgentExamples(Agent agent) { - if (!agent.containsLLMParserTool() || !LLMConnHelper.testConnection(agent.getLlmConfig())) { + if (!agent.containsLLMParserTool() || !LLMConnHelper.testConnection(agent.getLlmConfig()) + || CollectionUtils.isEmpty(agent.getExamples())) { return; } + List examples = agent.getExamples(); ChatMemoryFilter chatMemoryFilter = ChatMemoryFilter.builder().agentId(agent.getId()) .questions(examples).build(); diff --git a/evaluation/README_CN.md b/evaluation/README.md similarity index 100% rename from evaluation/README_CN.md rename to evaluation/README.md diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java index bd6f0f3e0..e0ccc96ca 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java @@ -39,7 +39,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { SemanticSchema semanticSchema = queryContext.getSemanticSchema(); // check has distinct if (SqlSelectHelper.hasDistinct(correctS2SQL)) { - log.info("not add group by ,exist distinct in correctS2SQL:{}", correctS2SQL); + log.debug("no need to add groupby ,existed distinct in s2sql:{}", correctS2SQL); return false; } //add alias field name @@ -53,7 +53,7 @@ public class GroupByCorrector extends BaseSemanticCorrector { return false; } if (SqlSelectHelper.hasGroupBy(correctS2SQL)) { - log.info("not add group by ,exist group by in correctS2SQL:{}", correctS2SQL); + log.debug("No need to add groupby, existed groupby in s2sql:{}", correctS2SQL); return false; } Environment environment = ContextUtils.getBean(Environment.class); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java index c01e22ab0..3252fe16e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java @@ -17,7 +17,6 @@ import com.tencent.supersonic.headless.chat.parser.SatisfactionChecker; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.headless.chat.utils.ComponentFactory; -import com.tencent.supersonic.headless.chat.utils.S2SqlDateHelper; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -28,7 +27,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -101,11 +99,7 @@ public class LLMRequestService { } llmReq.setLinking(linking); - String currentDate = S2SqlDateHelper.getReferenceDate(queryCtx, dataSetId); - if (StringUtils.isEmpty(currentDate)) { - currentDate = DateUtils.getBeforeDate(0); - } - llmReq.setCurrentDate(currentDate); + llmReq.setCurrentDate(DateUtils.getBeforeDate(0)); llmReq.setSqlGenType(LLMReq.SqlGenType.valueOf(parserConfig.getParameterValue(PARSER_STRATEGY_TYPE))); llmReq.setLlmConfig(queryCtx.getLlmConfig()); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java index f4c9e377e..3855cc69b 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java @@ -35,12 +35,12 @@ public class LLMSqlParser implements SemanticParser { if (dataSetId == null) { return; } - log.info("Try generating query statement for dataSetId:{}", dataSetId); + log.info("try generating query statement for dataSetId:{}", dataSetId); //3.invoke LLM service to do parsing. tryParse(queryCtx, dataSetId); } catch (Exception e) { - log.error("Failed to parse query:", e); + log.error("failed to parse query:", e); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java index 35fd89ce5..711ac0f3c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java @@ -33,7 +33,7 @@ public class DefaultQueryCache implements QueryCache { log.warn("exception:", exception); return null; }); - log.info("put to cache, key:{}", cacheKey); + log.debug("put to cache, key: {}", cacheKey); return true; } return false; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index 181879933..a9f40dfa5 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -12,8 +12,6 @@ import org.springframework.stereotype.Component; import java.util.Objects; -; - @Component("JdbcExecutor") @Slf4j public class JdbcExecutor implements QueryExecutor { @@ -40,7 +38,7 @@ public class JdbcExecutor implements QueryExecutor { log.warn("data base id is empty"); return null; } - log.info("query SQL: {}", queryStatement.getSql()); + log.info("executing SQL: {}", StringUtils.normalizeSpace(queryStatement.getSql())); Database database = queryStatement.getSemanticModel().getDatabase(); SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); SqlUtils sqlUtil = sqlUtils.init(database); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java index df132c2e9..0252f3da8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java @@ -27,6 +27,7 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Triple; import org.springframework.util.CollectionUtils; @@ -106,7 +107,7 @@ public class JoinRender extends Renderer { addZipperField(dataSource, dataSourceWhere); TableView tableView = SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension, metricCommand.getWhere(), dataSources.get(i), scope, schema, true); - log.info("tableView {}", tableView.getTable().toString()); + log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString())); String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); tableView.setAlias(alias); tableView.setPrimary(primary); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java index 3e42649a5..75fd19e4f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java @@ -69,16 +69,19 @@ public class SqlInfoProcessor implements ResultProcessor { ExplainSqlReq explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq) .queryTypeEnum(QueryMethod.SQL).build(); ExplainResp explain = queryService.explain(explainSqlReq, queryContext.getUser()); - String explainSql = explain.getSql(); - if (StringUtils.isBlank(explainSql)) { + String querySql = explain.getSql(); + if (StringUtils.isBlank(querySql)) { return; } SqlInfo sqlInfo = parseInfo.getSqlInfo(); if (semanticQuery instanceof LLMSqlQuery) { - keyPipelineLog.info("SqlInfoProcessor results:\nParsed S2SQL:{}\nCorrected S2SQL:{}\nFinal SQL:{}", - sqlInfo.getS2SQL(), sqlInfo.getCorrectS2SQL(), explainSql); + keyPipelineLog.info("SqlInfoProcessor results:\n" + + "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}", + StringUtils.normalizeSpace(sqlInfo.getS2SQL()), + StringUtils.normalizeSpace(sqlInfo.getCorrectS2SQL()), + StringUtils.normalizeSpace(querySql)); } - sqlInfo.setQuerySQL(explainSql); + sqlInfo.setQuerySQL(querySql); sqlInfo.setSourceId(explain.getSourceId()); }