(improvement)(project) support explain in semantic and show explain sql in web and fix chat start error (#103)

chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java

@@ -8,9 +8,11 @@ import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
 import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
 import com.tencent.supersonic.semantic.api.model.response.DomainResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
@@ -32,14 +34,27 @@ import java.util.List;
 public interface SemanticLayer {
 
     QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
+
     QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
+
     QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user);
+
     QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
+
     List<ModelSchema> getModelSchema();
+
     List<ModelSchema> getModelSchema(List<Long> ids);
+
     ModelSchema getModelSchema(Long model, Boolean cacheEnable);
+
     PageInfo<DimensionResp> getDimensionPage(PageDimensionReq pageDimensionCmd);
+
     PageInfo<MetricResp> getMetricPage(PageMetricReq pageMetricCmd);
+
     List<DomainResp> getDomainList(User user);
+
     List<ModelResp> getModelList(AuthType authType, Long domainId, User user);
+
+    <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
+
 }

chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.api.component;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import org.apache.calcite.sql.parser.SqlParseException;
 
 /**
@@ -14,6 +15,8 @@ public interface SemanticQuery {
 
     QueryResult execute(User user) throws SqlParseException;
 
+    ExplainResp explain(User user);
+
     SemanticParseInfo getParseInfo();
 
     void setParseInfo(SemanticParseInfo parseInfo);

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java

@@ -37,6 +37,8 @@ public class SemanticParseInfo {
     private List<SchemaElementMatch> elementMatches = new ArrayList<>();
     private Map<String, Object> properties = new HashMap<>();
     private EntityInfo entityInfo;
+    private String sql;
+
     public Long getModelId() {
         return model != null ? model.getId() : 0L;
     }
@@ -46,6 +48,7 @@ public class SemanticParseInfo {
     }
 
     private static class SchemaNameLengthComparator implements Comparator<SchemaElement> {
+
         @Override
         public int compare(SchemaElement o1, SchemaElement o2) {
             int len1 = o1.getName().length();

chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java

@@ -15,7 +15,10 @@ import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
+import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
 import java.util.ArrayList;
 import java.util.List;
@@ -42,12 +45,10 @@ public class DslQuery extends PluginSemanticQuery {
 
     @Override
     public QueryResult execute(User user) {
-        String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
+        LLMResp llmResp = getLlmResp();
-        DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class);
-        LLMResp llmResp = dslParseResult.getLlmResp();
 
         long startTime = System.currentTimeMillis();
-        QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId());
+        QueryDslReq queryDslReq = getQueryDslReq(llmResp);
         QueryResultWithSchemaResp queryResp = semanticLayer.queryByDsl(queryDslReq, user);
 
         log.info("queryByDsl cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput());
@@ -71,4 +72,30 @@ public class DslQuery extends PluginSemanticQuery {
         parseInfo.setProperties(null);
         return queryResult;
     }
+
+    private LLMResp getLlmResp() {
+        String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT));
+        DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class);
+        return dslParseResult.getLlmResp();
+    }
+
+    private QueryDslReq getQueryDslReq(LLMResp llmResp) {
+        QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId());
+        return queryDslReq;
+    }
+
+    @Override
+    public ExplainResp explain(User user) {
+        ExplainSqlReq explainSqlReq = null;
+        try {
+            explainSqlReq = ExplainSqlReq.builder()
+                    .queryTypeEnum(QueryTypeEnum.SQL)
+                    .queryReq(getQueryDslReq(getLlmResp()))
+                    .build();
+            return semanticLayer.explain(explainSqlReq, user);
+        } catch (Exception e) {
+            log.error("explain error explainSqlReq:{}", explainSqlReq, e);
+        }
+        return null;
+    }
 }

chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/PluginSemanticQuery.java

@@ -1,7 +1,9 @@
 package com.tencent.supersonic.chat.query.plugin;
 
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.component.SemanticQuery;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -17,5 +19,8 @@ public abstract class PluginSemanticQuery implements SemanticQuery {
         return parseInfo;
     }
 
-
+    @Override
+    public ExplainResp explain(User user) {
+        return null;
+    }
 }

chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java

@@ -21,8 +21,11 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
 import com.tencent.supersonic.chat.utils.QueryReqBuilder;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import java.io.Serializable;
@@ -215,6 +218,22 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable {
         return queryResult;
     }
 
+
+    @Override
+    public ExplainResp explain(User user) {
+        ExplainSqlReq explainSqlReq = null;
+        try {
+            explainSqlReq = ExplainSqlReq.builder()
+                    .queryTypeEnum(QueryTypeEnum.STRUCT)
+                    .queryReq(convertQueryStruct())
+                    .build();
+            return semanticLayer.explain(explainSqlReq, user);
+        } catch (Exception e) {
+            log.error("explain error explainSqlReq:{}", explainSqlReq, e);
+        }
+        return null;
+    }
+
     public QueryResult multiStructExecute(User user) {
         String queryMode = parseInfo.getQueryMode();
 

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java

@@ -27,6 +27,7 @@ import com.tencent.supersonic.chat.service.SemanticService;
 import com.tencent.supersonic.chat.service.StatisticsService;
 import com.tencent.supersonic.chat.utils.ComponentFactory;
 
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Set;
@@ -35,9 +36,7 @@ import java.util.Comparator;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-//import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
-//import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
@@ -64,8 +63,6 @@ public class QueryServiceImpl implements QueryService {
     @Autowired
     private StatisticsService statisticsService;
 
-    private final String entity = "ENTITY";
-
     @Value("${time.threshold: 100}")
     private Integer timeThreshold;
 
@@ -109,12 +106,16 @@ public class QueryServiceImpl implements QueryService {
                     .map(SemanticQuery::getParseInfo)
                     .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed())
                     .collect(Collectors.toList());
+
             selectedParses.forEach(parseInfo -> {
-                if (parseInfo.getQueryMode().contains(entity)) {
+                String queryMode = parseInfo.getQueryMode();
+                if (QueryManager.isEntityQuery(queryMode)) {
                     EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class)
                             .getEntityInfo(parseInfo, queryReq.getUser());
                     parseInfo.setEntityInfo(entityInfo);
                 }
+                addExplainSql(queryReq, parseInfo);
+
             });
             List<SemanticParseInfo> candidateParses = queryCtx.getCandidateQueries().stream()
                     .map(SemanticQuery::getParseInfo).collect(Collectors.toList());
@@ -138,6 +139,19 @@ public class QueryServiceImpl implements QueryService {
         return parseResult;
     }
 
+    private void addExplainSql(QueryReq queryReq, SemanticParseInfo parseInfo) {
+        SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode());
+        if (Objects.isNull(semanticQuery)) {
+            return;
+        }
+        semanticQuery.setParseInfo(parseInfo);
+        ExplainResp explain = semanticQuery.explain(queryReq.getUser());
+        if (Objects.isNull(explain)) {
+            return;
+        }
+        parseInfo.setSql(explain.getSql());
+    }
+
     @Override
     public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception {
         ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(),
@@ -155,9 +169,9 @@ public class QueryServiceImpl implements QueryService {
         chatCtx.setAgentId(queryReq.getAgentId());
         Long startTime = System.currentTimeMillis();
         QueryResult queryResult = semanticQuery.execute(queryReq.getUser());
-        Long endTime = System.currentTimeMillis();
+
         if (queryResult != null) {
-            timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime))
+            timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime))
                     .interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build());
             saveInfo(timeCostDOList, queryReq.getQueryText(), queryReq.getQueryId(),
                     queryReq.getUser().getName(), queryReq.getChatId().longValue());
@@ -169,7 +183,6 @@ public class QueryServiceImpl implements QueryService {
             }
             chatCtx.setQueryText(queryReq.getQueryText());
             chatCtx.setUser(queryReq.getUser().getName());
-            //chatService.addQuery(queryResult, chatCtx);
             chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx);
         } else {
             chatService.deleteChatQuery(queryReq.getQueryId());
@@ -179,8 +192,8 @@ public class QueryServiceImpl implements QueryService {
     }
 
     public void saveInfo(List<StatisticsDO> timeCostDOList,
-                         String queryText, Long queryId,
+            String queryText, Long queryId,
-                         String userName, Long chatId) {
+            String userName, Long chatId) {
         List<StatisticsDO> list = timeCostDOList.stream()
                 .filter(o -> o.getCost() > timeThreshold).collect(Collectors.toList());
         list.forEach(o -> {
@@ -264,13 +277,6 @@ public class QueryServiceImpl implements QueryService {
         dateConf.setPeriod("DAY");
         queryStructReq.setDateInfo(dateConf);
         queryStructReq.setLimit(20L);
-
-        //        List<Aggregator> aggregators = new ArrayList<>();
-        //        Aggregator aggregator = new Aggregator(dimensionValueReq.getQueryFilter().getBizName(),
-        //                AggOperatorEnum.DISTINCT);
-        //        aggregators.add(aggregator);
-        //        queryStructReq.setAggregators(aggregators);
-
         queryStructReq.setModelId(dimensionValueReq.getModelId());
         queryStructReq.setNativeQuery(true);
         List<String> groups = new ArrayList<>();

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java

@@ -38,4 +38,6 @@ public class DefaultSemanticConfig {
     @Value("${fetchModelList.path:/api/semantic/schema/model/list}")
     private String fetchModelListPath;
 
+    @Value("${explain.path:/api/semantic/query/explain}")
+    private String explainPath;
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java

@@ -8,12 +8,14 @@ import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
 import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
 import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelResp;
 import com.tencent.supersonic.semantic.api.model.response.MetricResp;
 import com.tencent.supersonic.semantic.api.model.response.DomainResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
@@ -95,6 +97,12 @@ public class LocalSemanticLayer extends BaseSemanticLayer {
         return schemaService.getModelList(user, authType, domainId);
     }
 
+    @Override
+    public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
+        queryService = ContextUtils.getBean(QueryService.class);
+        return queryService.explain(explainSqlReq, user);
+    }
+
     @Override
     public PageInfo<DimensionResp> getDimensionPage(PageDimensionReq pageDimensionCmd) {
         dimensionService = ContextUtils.getBean(DimensionService.class);

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java

@@ -1,38 +1,44 @@
 package com.tencent.supersonic.knowledge.semantic;
 
+import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
+import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
+import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
+import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
+
 import com.alibaba.fastjson.JSON;
 import com.github.pagehelper.PageInfo;
 import com.google.gson.Gson;
 import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
 import com.tencent.supersonic.auth.api.authentication.constant.UserConstants;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.common.pojo.ResultData;
+import com.tencent.supersonic.common.pojo.ReturnCode;
+import com.tencent.supersonic.common.pojo.enums.AuthType;
+import com.tencent.supersonic.common.pojo.exception.CommonException;
 import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.common.util.S2ThreadContext;
 import com.tencent.supersonic.common.util.ThreadContext;
-import com.tencent.supersonic.common.util.JsonUtil;
-import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
 import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq;
 import com.tencent.supersonic.semantic.api.model.request.PageMetricReq;
-import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
-import com.tencent.supersonic.semantic.api.model.response.ModelResp;
-import com.tencent.supersonic.semantic.api.model.response.MetricResp;
-import com.tencent.supersonic.semantic.api.model.response.DomainResp;
 import com.tencent.supersonic.semantic.api.model.response.DimensionResp;
+import com.tencent.supersonic.semantic.api.model.response.DomainResp;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
+import com.tencent.supersonic.semantic.api.model.response.MetricResp;
+import com.tencent.supersonic.semantic.api.model.response.ModelResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
+import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
-import com.tencent.supersonic.common.pojo.exception.CommonException;
-import com.tencent.supersonic.common.pojo.ResultData;
-import com.tencent.supersonic.common.pojo.ReturnCode;
-
 import java.net.URI;
+import java.net.URL;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Objects;
-import java.util.LinkedHashMap;
-
 import lombok.extern.slf4j.Slf4j;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.beans.BeanUtils;
@@ -45,11 +51,6 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.util.UriComponentsBuilder;
 
-import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER;
-import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER;
-import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER;
-import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER;
-
 @Slf4j
 public class RemoteSemanticLayer extends BaseSemanticLayer {
 
@@ -61,6 +62,10 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
             new ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>>() {
             };
 
+    private ParameterizedTypeReference<ResultData<ExplainResp>> explainTypeRef =
+            new ParameterizedTypeReference<ResultData<ExplainResp>>() {
+            };
+
     @Override
     public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
         DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
@@ -130,9 +135,10 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
         fillToken(headers);
         DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
 
-        URI requestUrl = UriComponentsBuilder.fromHttpUrl(
+        String semanticUrl = defaultSemanticConfig.getSemanticUrl();
-                defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchModelSchemaPath()).build()
+        String fetchModelSchemaPath = defaultSemanticConfig.getFetchModelSchemaPath();
-                .encode().toUri();
+        URI requestUrl = UriComponentsBuilder.fromHttpUrl(semanticUrl + fetchModelSchemaPath)
+                .build().encode().toUri();
         ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
         filter.setModelIds(ids);
         ParameterizedTypeReference<ResultData<List<ModelSchemaResp>>> responseTypeRef =
@@ -179,6 +185,39 @@ public class RemoteSemanticLayer extends BaseSemanticLayer {
         return JsonUtil.toList(JsonUtil.toString(domainDescListObject), ModelResp.class);
     }
 
+    @Override
+    public <T> ExplainResp explain(ExplainSqlReq<T> explainResp, User user) throws Exception {
+        DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
+        String semanticUrl = defaultSemanticConfig.getSemanticUrl();
+        String explainPath = defaultSemanticConfig.getExplainPath();
+        URL url = new URL(new URL(semanticUrl), explainPath);
+        return explain(url.toString(), JsonUtil.toString(explainResp));
+    }
+
+    public ExplainResp explain(String url, String jsonReq) {
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_JSON);
+        fillToken(headers);
+        URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
+        HttpEntity<String> entity = new HttpEntity<>(jsonReq, headers);
+        log.info("url:{},explain:{}", url, entity.getBody());
+        ResultData<ExplainResp> responseBody;
+        try {
+            RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
+
+            ResponseEntity<ResultData<ExplainResp>> responseEntity = restTemplate.exchange(
+                    requestUrl, HttpMethod.POST, entity, explainTypeRef);
+            log.info("ApiResponse<ExplainResp> responseBody:{}", responseEntity);
+            responseBody = responseEntity.getBody();
+            if (Objects.nonNull(responseBody.getData())) {
+                return responseBody.getData();
+            }
+            return null;
+        } catch (Exception e) {
+            throw new RuntimeException("explain interface error,url:" + url, e);
+        }
+    }
+
     public Object fetchHttpResult(String url, String bodyJson, HttpMethod httpMethod) {
         HttpHeaders headers = new HttpHeaders();
         headers.setContentType(MediaType.APPLICATION_JSON);

common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java

@@ -75,6 +75,13 @@ class SqlParserSelectHelperTest {
                         + "user_id like '%alice%' AND  publish_date > 10000   ORDER BY pv DESC LIMIT 1");
 
         System.out.println(filterExpression);
+
+        filterExpression = SqlParserSelectHelper.getFilterExpression(
+                "SELECT department, pv FROM s2 WHERE "
+                        + "user_id like '%alice%' AND  publish_date > 10000  "
+                        + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1");
+
+        System.out.println(filterExpression);
     }
 
 

launchers/chat/src/main/resources/db/chat-schema-h2.sql

@@ -1,3 +1,4 @@
+-- chat tables
 CREATE TABLE IF NOT EXISTS `s2_chat_context`
 (
     `chat_id`        BIGINT NOT NULL , -- context chat id
@@ -7,7 +8,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat_context`
     `semantic_parse` LONGVARCHAR DEFAULT NULL , -- parse data
     `ext_data`       LONGVARCHAR DEFAULT NULL , -- extend data
     PRIMARY KEY (`chat_id`)
-);
+    );
 
 CREATE TABLE IF NOT EXISTS `s2_chat`
 (
@@ -21,7 +22,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat`
     `is_delete`     INT DEFAULT '0' COMMENT 'is deleted',
     `is_top`        INT DEFAULT '0' COMMENT 'is top',
     PRIMARY KEY (`chat_id`)
-) ;
+    ) ;
 
 
 CREATE TABLE `s2_chat_query`
@@ -64,72 +65,21 @@ CREATE TABLE `s2_chat_statistics`
 );
 
 CREATE TABLE IF NOT EXISTS `s2_chat_config` (
-    `id` INT NOT NULL AUTO_INCREMENT,
+                                                `id` INT NOT NULL AUTO_INCREMENT,
-    `model_id` INT DEFAULT NULL ,
+                                                `model_id` INT DEFAULT NULL ,
-    `chat_detail_config` varchar(655) ,
+                                                `chat_detail_config` varchar(655) ,
     `chat_agg_config` varchar(655)    ,
-    `recommended_questions` varchar(1500)    ,
+    `recommended_questions`  varchar(1500)    ,
     `created_at` TIMESTAMP  NOT NULL   ,
     `updated_at` TIMESTAMP  NOT NULL   ,
     `created_by` varchar(100) NOT NULL   ,
     `updated_by` varchar(100) NOT NULL   ,
     `status` INT NOT NULL  DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted
     PRIMARY KEY (`id`)
-) ;
+    ) ;
-
-
--- CREATE TABLE IF NOT EXISTS `s2_chat_config` (
---     `id` INT NOT NULL AUTO_INCREMENT,
---     `domain_id` INT DEFAULT NULL ,
---     `default_metrics` varchar(655) DEFAULT NULL,
---     `visibility` varchar(655)    , -- invisible dimension metric information
---     `entity_info` varchar(655)    ,
---     `dictionary_info` varchar(655)    , -- dictionary-related dimension setting information
---     `created_at` TIMESTAMP  NOT NULL   ,
---     `updated_at` TIMESTAMP  NOT NULL   ,
---     `created_by` varchar(100) NOT NULL   ,
---     `updated_by` varchar(100) NOT NULL   ,
---     `status` INT NOT NULL  DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted
---     PRIMARY KEY (`id`)
--- ) ;
 COMMENT ON TABLE s2_chat_config IS 'chat config information table ';
 
 
-
-
-CREATE TABLE IF NOT EXISTS `s2_dictionary` (
-    `id` INT NOT NULL AUTO_INCREMENT,
-    `domain_id` INT NOT NULL ,
-    `dim_value_infos` LONGVARCHAR , -- dimension value setting information
-    `created_at` TIMESTAMP  NOT NULL ,
-    `updated_at` TIMESTAMP  NOT NULL ,
-    `created_by` varchar(100) NOT NULL ,
-    `updated_by` varchar(100) DEFAULT NULL ,
-    `status` INT NOT NULL  DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted
-    PRIMARY KEY (`id`),
-    UNIQUE (domain_id)
-    );
-COMMENT ON TABLE s2_dictionary IS 'dictionary configuration information table';
-
-
-CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
-   `id` INT NOT NULL AUTO_INCREMENT,
-   `name` varchar(255) NOT NULL , -- task name
-   `description` varchar(255) ,
-   `command`LONGVARCHAR  NOT NULL , -- task Request Parameters
-   `command_md5` varchar(255)  NOT NULL , -- task Request Parameters md5
-   `dimension_ids` varchar(500) ,
-   `status` INT NOT NULL , -- the final status of the task
-   `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP  ,
-   `created_by` varchar(100) NOT NULL ,
-   `progress` DOUBLE default 0.00  ,  -- task real-time progress
-   `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
-   `message` LONGVARCHAR  , -- remark related information
-   PRIMARY KEY (`id`)
-);
-COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';
-
-
 create table s2_user
 (
     id       INT AUTO_INCREMENT,
@@ -141,6 +91,28 @@ create table s2_user
 );
 COMMENT ON TABLE s2_user IS 'user information table';
 
+
+CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info` (
+                                                        `id` INT NOT NULL AUTO_INCREMENT,
+                                                        `trace_id` varchar(200) NOT NULL  ,
+    `model_id` INT  NOT NULL ,
+    `dimensions`LONGVARCHAR ,
+    `metrics`LONGVARCHAR ,
+    `orders`LONGVARCHAR ,
+    `filters`LONGVARCHAR ,
+    `date_info`LONGVARCHAR ,
+    `limit` INT NOT NULL ,
+    `native_query` TINYINT NOT NULL DEFAULT '0' ,
+    `sql`LONGVARCHAR ,
+    `created_at` TIMESTAMP  NOT NULL ,
+    `created_by` varchar(100) NOT NULL ,
+    `status` INT NOT NULL ,
+    `elapsed_ms` bigINT DEFAULT NULL ,
+    PRIMARY KEY (`id`)
+    );
+COMMENT ON TABLE s2_semantic_pasre_info IS 'semantic layer sql parsing information table';
+
+
 CREATE TABLE IF NOT EXISTS `s2_plugin`
 (
     `id`         INT AUTO_INCREMENT,
@@ -157,5 +129,50 @@ CREATE TABLE IF NOT EXISTS `s2_plugin`
     `config`     LONGVARCHAR  NULL,
     `comment`     LONGVARCHAR  NULL,
     PRIMARY KEY (`id`)
-); COMMENT ON TABLE s2_plugin IS 'plugin information table';
+    ); COMMENT ON TABLE s2_plugin IS 'plugin information table';
+
+CREATE TABLE IF NOT EXISTS s2_agent
+(
+    id          int AUTO_INCREMENT,
+    name        varchar(100)  null,
+    description varchar(500) null,
+    status       int null,
+    examples    varchar(500) null,
+    config      varchar(2000)  null,
+    created_by  varchar(100) null,
+    created_at  TIMESTAMP  null,
+    updated_by  varchar(100) null,
+    updated_at  TIMESTAMP null,
+    enable_search int null,
+    PRIMARY KEY (`id`)
+    ); COMMENT ON TABLE s2_agent IS 'agent information table';
+
+
+-------demo for semantic and chat
+CREATE TABLE IF NOT EXISTS `s2_user_department` (
+    `user_name` varchar(200) NOT NULL,
+    `department` varchar(200) NOT NULL -- department of user
+    );
+COMMENT ON TABLE s2_user_department IS 'user_department_info';
+
+
+CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
+                                                    `id` INT NOT NULL AUTO_INCREMENT,
+                                                    `name` varchar(255) NOT NULL , -- task name
+    `description` varchar(255) ,
+    `command`LONGVARCHAR  NOT NULL , -- task Request Parameters
+    `command_md5` varchar(255)  NOT NULL , -- task Request Parameters md5
+    `status` INT NOT NULL , -- the final status of the task
+    `dimension_ids` varchar(500)  NULL ,
+    `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP  ,
+    `created_by` varchar(100) NOT NULL ,
+    `progress` DOUBLE default 0.00  ,  -- task real-time progress
+    `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
+    `message` LONGVARCHAR  , -- remark related information
+    PRIMARY KEY (`id`)
+    );
+COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';
+
+
+
 

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ExplainResp.java

@@ -0,0 +1,19 @@
+package com.tencent.supersonic.semantic.api.model.response;
+
+import java.io.Serializable;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+@Data
+@ToString
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class ExplainResp implements Serializable {
+
+    private String sql;
+
+}

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/ExplainSqlReq.java

@@ -0,0 +1,20 @@
+package com.tencent.supersonic.semantic.api.query.request;
+
+import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+@Data
+@ToString
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class ExplainSqlReq<T> {
+
+    private QueryTypeEnum queryTypeEnum;
+
+    private T queryReq;
+}

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java

@@ -2,18 +2,22 @@ package com.tencent.supersonic.semantic.query.rest;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
+import com.tencent.supersonic.common.util.JsonUtil;
+import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.SqlParserResp;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
+import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
+import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
-import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq;
-import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
-import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
+import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
-import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
-import com.tencent.supersonic.semantic.query.service.QueryService;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
+import com.tencent.supersonic.semantic.query.service.QueryService;
+import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine;
 import java.util.List;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -89,10 +93,36 @@ public class QueryController {
 
     @PostMapping("/queryDimValue")
     public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
-                                                   HttpServletRequest request,
+            HttpServletRequest request,
-                                                   HttpServletResponse response) {
+            HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
         return queryService.queryDimValue(queryDimValueReq, user);
     }
 
+    @PostMapping("/explain")
+    public <T> ExplainResp explain(@RequestBody ExplainSqlReq<T> explainSqlReq,
+            HttpServletRequest request,
+            HttpServletResponse response) throws Exception {
+
+        User user = UserHolder.findUser(request, response);
+        String queryReqJson = JsonUtil.toString(explainSqlReq.getQueryReq());
+        QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
+
+        if (QueryTypeEnum.SQL.equals(queryTypeEnum)) {
+            QueryDslReq queryDslReq = JsonUtil.toObject(queryReqJson, QueryDslReq.class);
+            ExplainSqlReq<QueryDslReq> explainSqlReqNew = ExplainSqlReq.<QueryDslReq>builder()
+                    .queryReq(queryDslReq)
+                    .queryTypeEnum(queryTypeEnum).build();
+            return queryService.explain(explainSqlReqNew, user);
+        }
+        if (QueryTypeEnum.STRUCT.equals(queryTypeEnum)) {
+            QueryStructReq queryStructReq = JsonUtil.toObject(queryReqJson, QueryStructReq.class);
+            ExplainSqlReq<QueryStructReq> explainSqlReqNew = ExplainSqlReq.<QueryStructReq>builder()
+                    .queryReq(queryStructReq)
+                    .queryTypeEnum(queryTypeEnum).build();
+            return queryService.explain(explainSqlReqNew, user);
+        }
+        return null;
+    }
+
 }

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java

@@ -1,11 +1,13 @@
 package com.tencent.supersonic.semantic.query.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
-import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
+import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
 import com.tencent.supersonic.semantic.api.query.response.ItemUseResp;
 import java.util.List;
@@ -25,4 +27,5 @@ public interface QueryService {
 
     List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
 
+    <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception;
 }

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java

@@ -6,12 +6,15 @@ import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
 import com.tencent.supersonic.common.util.cache.CacheUtils;
 import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum;
 import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq;
+import com.tencent.supersonic.semantic.api.model.response.ExplainResp;
 import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp;
 import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
 import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
 import com.tencent.supersonic.semantic.api.query.pojo.Cache;
 import com.tencent.supersonic.semantic.api.query.pojo.Filter;
+import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq;
 import com.tencent.supersonic.semantic.api.query.request.ItemUseReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq;
 import com.tencent.supersonic.semantic.api.query.request.QueryDslReq;
@@ -64,6 +67,11 @@ public class QueryServiceImpl implements QueryService {
 
     @Override
     public Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception {
+        QueryStatement queryStatement = convertToQueryStatement(querySqlCmd, user);
+        return semanticQueryEngine.execute(queryStatement);
+    }
+
+    private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception {
         ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
         List<Long> modelIds = new ArrayList<>();
         modelIds.add(querySqlCmd.getModelId());
@@ -74,7 +82,7 @@ public class QueryServiceImpl implements QueryService {
 
         QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchemas);
         queryStatement.setModelId(querySqlCmd.getModelId());
-        return semanticQueryEngine.execute(queryStatement);
+        return queryStatement;
     }
 
     @Override
@@ -183,6 +191,32 @@ public class QueryServiceImpl implements QueryService {
         return statInfos;
     }
 
+    @Override
+    public <T> ExplainResp explain(ExplainSqlReq<T> explainSqlReq, User user) throws Exception {
+        QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum();
+        T queryReq = explainSqlReq.getQueryReq();
+
+        if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryDslReq) {
+            QueryStatement queryStatement = convertToQueryStatement((QueryDslReq) queryReq, user);
+            return getExplainResp(queryStatement);
+        }
+        if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {
+            QueryStatement queryStatement = semanticQueryEngine.plan((QueryStructReq) queryReq);
+            return getExplainResp(queryStatement);
+        }
+
+        throw new IllegalArgumentException("Parameters are invalid, explainSqlReq: " + explainSqlReq);
+    }
+
+    private ExplainResp getExplainResp(QueryStatement queryStatement) {
+        String sql = "";
+        if (Objects.nonNull(queryStatement)) {
+            sql = queryStatement.getSql();
+        }
+        return ExplainResp.builder().sql(sql).build();
+    }
+
+
     private boolean isCache(QueryStructReq queryStructCmd) {
         if (!cacheEnable) {
             return false;