diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java index 9af3946a9..8e2c476ff 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseResp.java @@ -22,6 +22,7 @@ public class ParseResp { private List selectedParses; private List candidateParses; private List similarSolvedQuery; + private ParseTimeCostDO parseTimeCost; public enum ParseState { COMPLETED, diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseTimeCostDO.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseTimeCostDO.java new file mode 100644 index 000000000..df18d34db --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ParseTimeCostDO.java @@ -0,0 +1,9 @@ +package com.tencent.supersonic.chat.api.pojo.response; + +import lombok.Data; + +@Data +public class ParseTimeCostDO { + private Long parseTime; + private Long sqlTime; +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryRecallResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryRecallResp.java new file mode 100644 index 000000000..f5d764336 --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryRecallResp.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.chat.api.pojo.response; + +import lombok.Data; + +import java.util.List; + +@Data +public class QueryRecallResp { + private List solvedQueryRecallRespList; + private Long queryTimeCost; +} diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java index 3858f64f2..093f7e0c1 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java @@ -21,4 +21,5 @@ public class QueryResult { private SemanticParseInfo chatContext; private Object response; private List> queryResults; + private Long queryTimeCost; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java index 1b95f9e56..69fa1e77d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/llm/s2ql/LLMS2QLParser.java @@ -172,7 +172,7 @@ public class LLMS2QLParser implements SemanticParser { } private List getDimensionFilter(Map fieldNameToElement, - List filterExpressions) { + List filterExpressions) { List result = Lists.newArrayList(); for (FilterExpression expression : filterExpressions) { QueryFilter dimensionFilter = new QueryFilter(); @@ -229,7 +229,7 @@ public class LLMS2QLParser implements SemanticParser { } private boolean containOperators(FilterExpression expression, FilterOperatorEnum firstOperator, - FilterOperatorEnum... operatorEnums) { + FilterOperatorEnum... operatorEnums) { return (Arrays.asList(operatorEnums).contains(firstOperator) && Objects.nonNull(expression.getFieldValue())); } @@ -257,7 +257,7 @@ public class LLMS2QLParser implements SemanticParser { } private SemanticParseInfo getParseInfo(QueryContext queryCtx, Long modelId, CommonAgentTool commonAgentTool, - ParseResult parseResult) { + ParseResult parseResult) { PluginSemanticQuery semanticQuery = QueryManager.createPluginQuery(S2QLQuery.QUERY_MODE); SemanticParseInfo parseInfo = semanticQuery.getParseInfo(); parseInfo.getElementMatches().addAll(queryCtx.getMapInfo().getMatchedElements(modelId)); @@ -414,7 +414,7 @@ public class LLMS2QLParser implements SemanticParser { protected List getFieldNameList(QueryContext queryCtx, Long modelId, SemanticSchema semanticSchema, - LLMParserConfig llmParserConfig) { + LLMParserConfig llmParserConfig) { Set results = getTopNFieldNames(modelId, semanticSchema, llmParserConfig); @@ -450,7 +450,7 @@ public class LLMS2QLParser implements SemanticParser { } private Set getTopNFieldNames(Long modelId, SemanticSchema semanticSchema, - LLMParserConfig llmParserConfig) { + LLMParserConfig llmParserConfig) { Set results = semanticSchema.getDimensions(modelId).stream() .sorted(Comparator.comparing(SchemaElement::getUseCnt).reversed()) .limit(llmParserConfig.getDimensionTopN()) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ExplainSqlParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ExplainSqlParseResponder.java index e8d4e2476..ed4604c40 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ExplainSqlParseResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/ExplainSqlParseResponder.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; +import com.tencent.supersonic.chat.api.pojo.response.ParseTimeCostDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.common.util.JsonUtil; @@ -22,8 +23,11 @@ public class ExplainSqlParseResponder implements ParseResponder { public void fillResponse(ParseResp parseResp, QueryContext queryContext, List chatParseDOS) { QueryReq queryReq = queryContext.getRequest(); + Long startTime = System.currentTimeMillis(); addExplainSql(queryReq, parseResp.getSelectedParses()); addExplainSql(queryReq, parseResp.getCandidateParses()); + parseResp.setParseTimeCost(new ParseTimeCostDO()); + parseResp.getParseTimeCost().setSqlTime(System.currentTimeMillis() - startTime); if (!CollectionUtils.isEmpty(chatParseDOS)) { Map chatParseDOMap = chatParseDOS.stream() .collect(Collectors.toMap(ChatParseDO::getParseId, diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java index 41dd227f0..1cce6735a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/ChatController.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.rest; import com.github.pagehelper.PageInfo; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.chat.api.pojo.response.QueryRecallResp; import com.tencent.supersonic.chat.api.pojo.response.ShowCaseResp; import com.tencent.supersonic.chat.api.pojo.response.SolvedQueryRecallResp; import com.tencent.supersonic.chat.persistence.dataobject.ChatDO; @@ -89,7 +90,12 @@ public class ChatController { @RequestMapping("/getSolvedQuery") public List getSolvedQuery(@RequestParam(value = "queryText") String queryText, @RequestParam(value = "agentId") Integer agentId) { - return chatService.getSolvedQuery(queryText, agentId); + QueryRecallResp queryRecallResp = new QueryRecallResp(); + Long startTime = System.currentTimeMillis(); + List solvedQueryRecallRespList = chatService.getSolvedQuery(queryText, agentId); + queryRecallResp.setSolvedQueryRecallRespList(solvedQueryRecallRespList); + queryRecallResp.setQueryTimeCost(System.currentTimeMillis() - startTime); + return solvedQueryRecallRespList; } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 642375d85..f0c2cace7 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -112,6 +112,7 @@ public class QueryServiceImpl implements QueryService { @Override public ParseResp performParsing(QueryReq queryReq) { + Long parseTime = System.currentTimeMillis(); QueryContext queryCtx = new QueryContext(queryReq); // in order to support multi-turn conversation, chat context is needed ChatContext chatCtx = chatService.getOrCreateContext(queryReq.getChatId()); @@ -169,6 +170,8 @@ public class QueryServiceImpl implements QueryService { queryReq.getUser().getName(), queryReq.getChatId().longValue()); } chatService.updateChatParse(chatParseDOS); + parseResult.getParseTimeCost().setParseTime( + System.currentTimeMillis() - parseTime - parseResult.getParseTimeCost().getSqlTime()); return parseResult; } @@ -204,6 +207,7 @@ public class QueryServiceImpl implements QueryService { @Override public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception { + Long executeTime = System.currentTimeMillis(); ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(), queryReq.getParseId()); ChatQueryDO chatQueryDO = chatService.getLastQuery(queryReq.getChatId()); @@ -224,6 +228,7 @@ public class QueryServiceImpl implements QueryService { if (queryResult != null) { timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime)) .interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build()); + queryResult.setQueryTimeCost(timeCostDOList.get(0).getCost().longValue()); saveInfo(timeCostDOList, queryReq.getQueryText(), queryReq.getQueryId(), queryReq.getUser().getName(), queryReq.getChatId().longValue()); queryResult.setChatContext(parseInfo); @@ -242,7 +247,7 @@ public class QueryServiceImpl implements QueryService { } else { chatService.deleteChatQuery(queryReq.getQueryId()); } - + queryResult.setQueryTimeCost(System.currentTimeMillis() - executeTime); return queryResult; } @@ -349,20 +354,14 @@ public class QueryServiceImpl implements QueryService { parseInfo.getDimensionFilters(), addWhereConditions, removeWhereFieldNames); updateDateInfo(queryData, parseInfo, filedNameToValueMap, whereExpressionList, addWhereConditions, removeWhereFieldNames); - log.info("filedNameToValueMap:{}", filedNameToValueMap); - log.info("removeWhereFieldNames:{}", removeWhereFieldNames); correctorSql = SqlParserReplaceHelper.replaceValue(correctorSql, filedNameToValueMap); correctorSql = SqlParserRemoveHelper.removeWhereCondition(correctorSql, removeWhereFieldNames); updateFilters(havingFiledNameToValueMap, havingExpressionList, queryData.getDimensionFilters(), parseInfo.getDimensionFilters(), addHavingConditions, removeHavingFieldNames); - log.info("havingFiledNameToValueMap:{}", havingFiledNameToValueMap); - log.info("removeHavingFieldNames:{}", removeHavingFieldNames); correctorSql = SqlParserReplaceHelper.replaceHavingValue(correctorSql, havingFiledNameToValueMap); correctorSql = SqlParserRemoveHelper.removeHavingCondition(correctorSql, removeHavingFieldNames); - log.info("addWhereConditions:{}", addWhereConditions); - log.info("addHavingConditions:{}", addHavingConditions); correctorSql = SqlParserAddHelper.addWhere(correctorSql, addWhereConditions); correctorSql = SqlParserAddHelper.addHaving(correctorSql, addHavingConditions); diff --git a/common/pom.xml b/common/pom.xml index c02715ba1..3d0eca123 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -133,6 +133,10 @@ jsqlparser ${jsqlparser.version} + + org.springframework.boot + spring-boot-starter-web + diff --git a/common/src/main/java/com/tencent/supersonic/common/interceptor/LogInterceptor.java b/common/src/main/java/com/tencent/supersonic/common/interceptor/LogInterceptor.java new file mode 100644 index 000000000..69324b181 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/interceptor/LogInterceptor.java @@ -0,0 +1,41 @@ +package com.tencent.supersonic.common.interceptor; + +import com.tencent.supersonic.common.util.TraceIdUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.HandlerInterceptor; +import org.springframework.web.servlet.ModelAndView; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@Component +@Slf4j +public class LogInterceptor implements HandlerInterceptor { + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + //use previous traceId + String traceId = request.getHeader(TraceIdUtil.TRACE_ID); + if (StringUtils.isBlank(traceId)) { + TraceIdUtil.setTraceId(TraceIdUtil.generateTraceId()); + } else { + TraceIdUtil.setTraceId(traceId); + } + return true; + } + + @Override + public void postHandle(HttpServletRequest request, HttpServletResponse response, + Object handler, ModelAndView modelAndView) + throws Exception { + + } + + @Override + public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) + throws Exception { + //remove after Completing + TraceIdUtil.remove(); + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java index 0314ec35c..12fb49919 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ResultData.java @@ -1,6 +1,8 @@ package com.tencent.supersonic.common.pojo; +import com.tencent.supersonic.common.util.TraceIdUtil; import lombok.Data; +import org.slf4j.MDC; /*** * result data @@ -11,6 +13,7 @@ public class ResultData { private String msg; private T data; private long timestamp; + private String traceId; public ResultData() { this.timestamp = System.currentTimeMillis(); @@ -21,6 +24,7 @@ public class ResultData { resultData.setCode(ReturnCode.SUCCESS.getCode()); resultData.setMsg(ReturnCode.SUCCESS.getMessage()); resultData.setData(data); + resultData.setTraceId(MDC.get(TraceIdUtil.TRACE_ID)); return resultData; } @@ -28,7 +32,8 @@ public class ResultData { ResultData resultData = new ResultData<>(); resultData.setCode(code); resultData.setMsg(message); + resultData.setTraceId(MDC.get(TraceIdUtil.TRACE_ID)); return resultData; } -} \ No newline at end of file +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/ThreadPoolExecutorMdcWrapper.java b/common/src/main/java/com/tencent/supersonic/common/pojo/ThreadPoolExecutorMdcWrapper.java new file mode 100644 index 000000000..3227828c7 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/ThreadPoolExecutorMdcWrapper.java @@ -0,0 +1,30 @@ +package com.tencent.supersonic.common.pojo; + +import com.tencent.supersonic.common.util.ThreadMdcUtil; +import org.slf4j.MDC; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Callable; +import java.util.concurrent.Future; + +public class ThreadPoolExecutorMdcWrapper extends ThreadPoolTaskExecutor { + private static final long serialVersionUID = 3940722618853093830L; + + @Override + public void execute(Runnable task) { + super.execute(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + } + + @Override + public Future submit(Callable task) { + return super + .submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + } + + @Override + public Future submit(Runnable task) { + return super + .submit(ThreadMdcUtil.wrap(task, MDC.getCopyOfContextMap())); + } +} + diff --git a/common/src/main/java/com/tencent/supersonic/common/util/ThreadMdcUtil.java b/common/src/main/java/com/tencent/supersonic/common/util/ThreadMdcUtil.java new file mode 100644 index 000000000..81bf74313 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/ThreadMdcUtil.java @@ -0,0 +1,49 @@ +package com.tencent.supersonic.common.util; + +import org.slf4j.MDC; + +import java.util.Map; +import java.util.concurrent.Callable; + +public class ThreadMdcUtil { + public static void setTraceIdIfAbsent() { + if (MDC.get(TraceIdUtil.TRACE_ID) == null) { + MDC.put(TraceIdUtil.TRACE_ID, TraceIdUtil.generateTraceId()); + } + } + + public static Callable wrap(final Callable callable, final Map context) { + return () -> { + if (context == null) { + MDC.clear(); + } else { + MDC.setContextMap(context); + } + setTraceIdIfAbsent(); + try { + return callable.call(); + } finally { + MDC.clear(); + } + }; + } + + public static Runnable wrap(final Runnable runnable, final Map context) { + return () -> { + if (context == null) { + MDC.clear(); + } else { + MDC.setContextMap(context); + } + //设置traceId + setTraceIdIfAbsent(); + try { + runnable.run(); + } finally { + MDC.clear(); + } + }; + } +} + + diff --git a/common/src/main/java/com/tencent/supersonic/common/util/TraceIdUtil.java b/common/src/main/java/com/tencent/supersonic/common/util/TraceIdUtil.java new file mode 100644 index 000000000..025997bc2 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/TraceIdUtil.java @@ -0,0 +1,34 @@ +package com.tencent.supersonic.common.util; + +import org.slf4j.MDC; + +import java.util.UUID; + +public class TraceIdUtil { + public static final String TRACE_ID = "traceId"; + + public static final String PREFIX = "supersonic"; + + public static String getTraceId() { + String traceId = (String) MDC.get(TRACE_ID); + return traceId == null ? "" : traceId; + } + + public static void setTraceId(String traceId) { + MDC.put(TRACE_ID, traceId); + } + + public static void remove() { + MDC.remove(TRACE_ID); + + } + + public static void clear() { + MDC.clear(); + } + + public static String generateTraceId() { + String uuid = UUID.randomUUID().toString().replace("-", ""); + return PREFIX + "_" + uuid; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/WebMvcConfig.java b/common/src/main/java/com/tencent/supersonic/common/util/WebMvcConfig.java new file mode 100644 index 000000000..80bdf7cb5 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/WebMvcConfig.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.common.util; + +import com.tencent.supersonic.common.interceptor.LogInterceptor; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +public class WebMvcConfig implements WebMvcConfigurer { + + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(new LogInterceptor()).addPathPatterns("/**"); + } +} + diff --git a/launchers/standalone/src/main/resources/logback-spring.xml b/launchers/standalone/src/main/resources/logback-spring.xml index 810eda9b3..05ee6d7fe 100644 --- a/launchers/standalone/src/main/resources/logback-spring.xml +++ b/launchers/standalone/src/main/resources/logback-spring.xml @@ -29,7 +29,7 @@ UTF-8 - %d [%thread] %-5level [%X{TRACE_ID}] %logger{36} %line - %msg%n + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n @@ -55,7 +55,7 @@ UTF-8 - %d [%thread] %-5level [%X{TRACE_ID}] %logger{36} %line - %msg%n + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n @@ -83,11 +83,11 @@ UTF-8 - %d [%thread] %-5level [%X{TRACE_ID}] %logger{36} %line - %msg%n + %d [%thread] %-5level [%X{traceId}] %logger{36} %line - %msg%n - \ No newline at end of file + diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java index d3c145d6c..25be95246 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/S2QLDataAspect.java @@ -75,12 +75,11 @@ public class S2QLDataAspect { //1. determine whether admin of the model if (authCommonService.doModelAdmin(user, modelId)) { + log.info("determine whether admin of the model!"); return joinPoint.proceed(); } - // 2. determine whether the subject field is visible authCommonService.doModelVisible(user, modelId); - // 3. fetch data permission meta information Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2QLReq, user); log.info("modelId:{}, res4Privilege:{}", modelId, res4Privilege);