topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)Normalize space SQL strings in the log messages.

Browse Source
This commit is contained in:
jerryjzhang
2024-07-04 18:23:55 +08:00
parent fa5abc58a5
commit bb4cd880b0
9 changed files with 21 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java
View File

@@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
@@ -85,9 +86,11 @@ public class AgentServiceImpl extends ServiceImpl<AgentDOMapper, AgentDO>
} }
private synchronized void doExecuteAgentExamples(Agent agent) { private synchronized void doExecuteAgentExamples(Agent agent) {
if (!agent.containsLLMParserTool() || !LLMConnHelper.testConnection(agent.getLlmConfig())) { if (!agent.containsLLMParserTool() || !LLMConnHelper.testConnection(agent.getLlmConfig())
|| CollectionUtils.isEmpty(agent.getExamples())) {
return; return;
} }
List<String> examples = agent.getExamples(); List<String> examples = agent.getExamples();
ChatMemoryFilter chatMemoryFilter = ChatMemoryFilter.builder().agentId(agent.getId()) ChatMemoryFilter chatMemoryFilter = ChatMemoryFilter.builder().agentId(agent.getId())
.questions(examples).build(); .questions(examples).build();

0
evaluation/README_CN.md → evaluation/README.md
View File

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/GroupByCorrector.java
View File

@@ -39,7 +39,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
SemanticSchema semanticSchema = queryContext.getSemanticSchema(); SemanticSchema semanticSchema = queryContext.getSemanticSchema();
// check has distinct // check has distinct
if (SqlSelectHelper.hasDistinct(correctS2SQL)) { if (SqlSelectHelper.hasDistinct(correctS2SQL)) {
log.info("not add group by ,exist distinct in correctS2SQL:{}", correctS2SQL); log.debug("no need to add groupby ,existed distinct in s2sql:{}", correctS2SQL);
return false; return false;
} }
//add alias field name //add alias field name
@@ -53,7 +53,7 @@ public class GroupByCorrector extends BaseSemanticCorrector {
return false; return false;
} }
if (SqlSelectHelper.hasGroupBy(correctS2SQL)) { if (SqlSelectHelper.hasGroupBy(correctS2SQL)) {
log.info("not add group by ,exist group by in correctS2SQL:{}", correctS2SQL); log.debug("No need to add groupby, existed groupby in s2sql:{}", correctS2SQL);
return false; return false;
} }
Environment environment = ContextUtils.getBean(Environment.class); Environment environment = ContextUtils.getBean(Environment.class);

8
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java
View File

@@ -17,7 +17,6 @@ import com.tencent.supersonic.headless.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp;
import com.tencent.supersonic.headless.chat.utils.ComponentFactory; import com.tencent.supersonic.headless.chat.utils.ComponentFactory;
import com.tencent.supersonic.headless.chat.utils.S2SqlDateHelper;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
@@ -28,7 +27,6 @@ import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -101,11 +99,7 @@ public class LLMRequestService {
} }
llmReq.setLinking(linking); llmReq.setLinking(linking);
String currentDate = S2SqlDateHelper.getReferenceDate(queryCtx, dataSetId); llmReq.setCurrentDate(DateUtils.getBeforeDate(0));
if (StringUtils.isEmpty(currentDate)) {
currentDate = DateUtils.getBeforeDate(0);
}
llmReq.setCurrentDate(currentDate);
llmReq.setSqlGenType(LLMReq.SqlGenType.valueOf(parserConfig.getParameterValue(PARSER_STRATEGY_TYPE))); llmReq.setSqlGenType(LLMReq.SqlGenType.valueOf(parserConfig.getParameterValue(PARSER_STRATEGY_TYPE)));
llmReq.setLlmConfig(queryCtx.getLlmConfig()); llmReq.setLlmConfig(queryCtx.getLlmConfig());

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMSqlParser.java
View File

@@ -35,12 +35,12 @@ public class LLMSqlParser implements SemanticParser {
if (dataSetId == null) { if (dataSetId == null) {
return; return;
} }
log.info("Try generating query statement for dataSetId:{}", dataSetId); log.info("try generating query statement for dataSetId:{}", dataSetId);
//3.invoke LLM service to do parsing. //3.invoke LLM service to do parsing.
tryParse(queryCtx, dataSetId); tryParse(queryCtx, dataSetId);
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to parse query:", e); log.error("failed to parse query:", e);
} }
} }

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java vendored
View File

@@ -33,7 +33,7 @@ public class DefaultQueryCache implements QueryCache {
log.warn("exception:", exception); log.warn("exception:", exception);
return null; return null;
}); });
log.info("put to cache, key:{}", cacheKey); log.debug("put to cache, key: {}", cacheKey);
return true; return true;
} }
return false; return false;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java
View File

@@ -12,8 +12,6 @@ import org.springframework.stereotype.Component;
import java.util.Objects; import java.util.Objects;
;
@Component("JdbcExecutor") @Component("JdbcExecutor")
@Slf4j @Slf4j
public class JdbcExecutor implements QueryExecutor { public class JdbcExecutor implements QueryExecutor {
@@ -40,7 +38,7 @@ public class JdbcExecutor implements QueryExecutor {
log.warn("data base id is empty"); log.warn("data base id is empty");
return null; return null;
} }
log.info("query SQL: {}", queryStatement.getSql()); log.info("executing SQL: {}", StringUtils.normalizeSpace(queryStatement.getSql()));
Database database = queryStatement.getSemanticModel().getDatabase(); Database database = queryStatement.getSemanticModel().getDatabase();
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
SqlUtils sqlUtil = sqlUtils.init(database); SqlUtils sqlUtil = sqlUtils.init(database);

3
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java
View File

@@ -27,6 +27,7 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorScope;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Triple; import org.apache.commons.lang3.tuple.Triple;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@@ -106,7 +107,7 @@ public class JoinRender extends Renderer {
addZipperField(dataSource, dataSourceWhere); addZipperField(dataSource, dataSourceWhere);
TableView tableView = SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension, TableView tableView = SourceRender.renderOne("", dataSourceWhere, queryMetrics, queryDimension,
metricCommand.getWhere(), dataSources.get(i), scope, schema, true); metricCommand.getWhere(), dataSources.get(i), scope, schema, true);
log.info("tableView {}", tableView.getTable().toString()); log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString()));
String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName(); String alias = Constants.JOIN_TABLE_PREFIX + dataSource.getName();
tableView.setAlias(alias); tableView.setAlias(alias);
tableView.setPrimary(primary); tableView.setPrimary(primary);

13
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java
View File

@@ -69,16 +69,19 @@ public class SqlInfoProcessor implements ResultProcessor {
ExplainSqlReq<Object> explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq) ExplainSqlReq<Object> explainSqlReq = ExplainSqlReq.builder().queryReq(semanticQueryReq)
.queryTypeEnum(QueryMethod.SQL).build(); .queryTypeEnum(QueryMethod.SQL).build();
ExplainResp explain = queryService.explain(explainSqlReq, queryContext.getUser()); ExplainResp explain = queryService.explain(explainSqlReq, queryContext.getUser());
String explainSql = explain.getSql(); String querySql = explain.getSql();
if (StringUtils.isBlank(explainSql)) { if (StringUtils.isBlank(querySql)) {
return; return;
} }
SqlInfo sqlInfo = parseInfo.getSqlInfo(); SqlInfo sqlInfo = parseInfo.getSqlInfo();
if (semanticQuery instanceof LLMSqlQuery) { if (semanticQuery instanceof LLMSqlQuery) {
keyPipelineLog.info("SqlInfoProcessor results:\nParsed S2SQL:{}\nCorrected S2SQL:{}\nFinal SQL:{}", keyPipelineLog.info("SqlInfoProcessor results:\n"
sqlInfo.getS2SQL(), sqlInfo.getCorrectS2SQL(), explainSql); + "Parsed S2SQL: {}\nCorrected S2SQL: {}\nQuery SQL: {}",
StringUtils.normalizeSpace(sqlInfo.getS2SQL()),
StringUtils.normalizeSpace(sqlInfo.getCorrectS2SQL()),
StringUtils.normalizeSpace(querySql));
} }
sqlInfo.setQuerySQL(explainSql); sqlInfo.setQuerySQL(querySql);
sqlInfo.setSourceId(explain.getSourceId()); sqlInfo.setSourceId(explain.getSourceId());
} }