topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 02:46:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)Normalize space SQL strings in the log messages.

Browse Source
This commit is contained in:
jerryjzhang
2024-07-04 18:23:55 +08:00
parent fa5abc58a5
commit bb4cd880b0
9 changed files with 21 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java
View File

@@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -85,9 +86,11 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO>
}
private synchronized void doExecuteAgentExamples(Agent agent) {
if (!agent.containsLLMParserTool() || !LLMConnHelper.testConnection(agent.getLlmConfig())) {
if (!agent.containsLLMParserTool() || !LLMConnHelper.testConnection(agent.getLlmConfig())
|| CollectionUtils.isEmpty(agent.getExamples())) {
return;
}
List<String> examples = agent.getExamples();
ChatMemoryFilter chatMemoryFilter = ChatMemoryFilter.builder().agentId(agent.getId())
.questions(examples).build();

0
evaluation/README_CN.md → evaluation/README.md
View File

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java
View File

@@ -39,7 +39,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
// check has distinct
if (SqlSelectHelper.hasDistinct(correctS2SQL)) {
log.info("not add group by ,exist distinct in correctS2SQL:{}", correctS2SQL);
log.debug("no need to add groupby ,existed distinct in s2sql:{}", correctS2SQL);
return false;
}
//add alias field name
@@ -53,7 +53,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
return false;
}
if (SqlSelectHelper.hasGroupBy(correctS2SQL)) {
log.info("not add group by ,exist group by in correctS2SQL:{}", correctS2SQL);
log.debug("No need to add groupby, existed groupby in s2sql:{}", correctS2SQL);
return false;
}
Environment environment = ContextUtils.getBean(Environment.class);

8
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java
View File

@@ -17,7 +17,6 @@ import com.tencent.supersonic.headless.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp;
import com.tencent.supersonic.headless.chat.utils.ComponentFactory;
import com.tencent.supersonic.headless.chat.utils.S2SqlDateHelper;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -28,7 +27,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -101,11 +99,7 @@ public class LLMRequestService {
}
llmReq.setLinking(linking);
String currentDate = S2SqlDateHelper.getReferenceDate(queryCtx, dataSetId);
if (StringUtils.isEmpty(currentDate)) {
currentDate = DateUtils.getBeforeDate(0);
}
llmReq.setCurrentDate(currentDate);
llmReq.setCurrentDate(DateUtils.getBeforeDate(0));
llmReq.setSqlGenType(LLMReq.SqlGenType.valueOf(parserConfig.getParameterValue(PARSER_STRATEGY_TYPE)));
llmReq.setLlmConfig(queryCtx.getLlmConfig());

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java
View File

@@ -35,12 +35,12 @@ public class LLMSqlParser implements SemanticParser {
if (dataSetId == null) {
return;
}
log.info("Try generating query statement for dataSetId:{}", dataSetId);
log.info("try generating query statement for dataSetId:{}", dataSetId);
//3.invoke LLM service to do parsing.
tryParse(queryCtx, dataSetId);
} catch (Exception e) {
log.error("Failed to parse query:", e);
log.error("failed to parse query:", e);
}
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java vendored
View File

@@ -33,7 +33,7 @@ public class DefaultQueryCache implements QueryCache {
log.warn("exception:", exception);
return null;
});
log.info("put to cache, key:{}", cacheKey);
log.debug("put to cache, key: {}", cacheKey);
return true;
}
return false;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java
View File

@@ -12,8 +12,6 @@ import org.springframework.stereotype.Component;
import java.util.Objects;
;
@Component("JdbcExecutor")
@Slf4j
public class JdbcExecutor implements QueryExecutor {
@@ -40,7 +38,7 @@ public class JdbcExecutor implements QueryExecutor {
log.warn("data base id is empty");
return null;
}
log.info("query SQL: {}", queryStatement.getSql());
log.info("executing SQL: {}", StringUtils.normalizeSpace(queryStatement.getSql()));
Database database = queryStatement.getSemanticModel().getDatabase();
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
SqlUtils sqlUtil = sqlUtils.init(database);

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -27,6 +27,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.util.CollectionUtils;
@@ -106,7 +107,7 @@ public class JoinRender extends Renderer {
addZipperField(dataSource, dataSourceWhere);
TableView tableView = SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension,
metricCommand.getWhere(), dataSources.get(i), scope, schema, true);
log.info("tableView {}", tableView.getTable().toString());
log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString()));
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
tableView.setAlias(alias);
tableView.setPrimary(primary);

13
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java
View File

@@ -69,16 +69,19 @@ public class SqlInfoProcessor implements ResultProcessor {
ExplainSqlReq<Object> explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq)
.queryTypeEnum(QueryMethod.SQL).build();
ExplainResp explain = queryService.explain(explainSqlReq, queryContext.getUser());
String explainSql = explain.getSql();
if (StringUtils.isBlank(explainSql)) {
String querySql = explain.getSql();
if (StringUtils.isBlank(querySql)) {
return;
}
SqlInfo sqlInfo = parseInfo.getSqlInfo();
if (semanticQuery instanceof LLMSqlQuery) {
keyPipelineLog.info("SqlInfoProcessor results:\nParsed S2SQL:{}\nCorrected S2SQL:{}\nFinal SQL:{}",
sqlInfo.getS2SQL(), sqlInfo.getCorrectS2SQL(), explainSql);
keyPipelineLog.info("SqlInfoProcessor results:\n"
+ "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
StringUtils.normalizeSpace(sqlInfo.getS2SQL()),
StringUtils.normalizeSpace(sqlInfo.getCorrectS2SQL()),
StringUtils.normalizeSpace(querySql));
}
sqlInfo.setQuerySQL(explainSql);
sqlInfo.setQuerySQL(querySql);
sqlInfo.setSourceId(explain.getSourceId());
}