diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java index dbd0226ee..12d336ed2 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java @@ -8,9 +8,11 @@ import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; @@ -32,14 +34,27 @@ import java.util.List; public interface SemanticLayer { QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user); + QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); + QueryResultWithSchemaResp queryByDsl(QueryDslReq queryDslReq, User user); + QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); + List getModelSchema(); + List getModelSchema(List ids); + ModelSchema getModelSchema(Long model, Boolean cacheEnable); + PageInfo getDimensionPage(PageDimensionReq pageDimensionCmd); + PageInfo getMetricPage(PageMetricReq pageMetricCmd); + List getDomainList(User user); + List getModelList(AuthType authType, Long domainId, User user); + + ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception; + } diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java index 91acf9016..70d263153 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticQuery.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.chat.api.component; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import org.apache.calcite.sql.parser.SqlParseException; /** @@ -14,6 +15,8 @@ public interface SemanticQuery { QueryResult execute(User user) throws SqlParseException; + ExplainResp explain(User user); + SemanticParseInfo getParseInfo(); void setParseInfo(SemanticParseInfo parseInfo); diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index 671aa91a4..e8edc777b 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -37,6 +37,8 @@ public class SemanticParseInfo { private List elementMatches = new ArrayList<>(); private Map properties = new HashMap<>(); private EntityInfo entityInfo; + private String sql; + public Long getModelId() { return model != null ? model.getId() : 0L; } @@ -46,6 +48,7 @@ public class SemanticParseInfo { } private static class SchemaNameLengthComparator implements Comparator { + @Override public int compare(SchemaElement o1, SchemaElement o2) { int len1 = o1.getName().length(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java index 64b97bbf0..58764882a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/dsl/DslQuery.java @@ -15,7 +15,10 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import java.util.ArrayList; import java.util.List; @@ -42,12 +45,10 @@ public class DslQuery extends PluginSemanticQuery { @Override public QueryResult execute(User user) { - String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)); - DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class); - LLMResp llmResp = dslParseResult.getLlmResp(); + LLMResp llmResp = getLlmResp(); long startTime = System.currentTimeMillis(); - QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId()); + QueryDslReq queryDslReq = getQueryDslReq(llmResp); QueryResultWithSchemaResp queryResp = semanticLayer.queryByDsl(queryDslReq, user); log.info("queryByDsl cost:{},querySql:{}", System.currentTimeMillis() - startTime, llmResp.getSqlOutput()); @@ -71,4 +72,30 @@ public class DslQuery extends PluginSemanticQuery { parseInfo.setProperties(null); return queryResult; } + + private LLMResp getLlmResp() { + String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)); + DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class); + return dslParseResult.getLlmResp(); + } + + private QueryDslReq getQueryDslReq(LLMResp llmResp) { + QueryDslReq queryDslReq = QueryReqBuilder.buildDslReq(llmResp.getCorrectorSql(), parseInfo.getModelId()); + return queryDslReq; + } + + @Override + public ExplainResp explain(User user) { + ExplainSqlReq explainSqlReq = null; + try { + explainSqlReq = ExplainSqlReq.builder() + .queryTypeEnum(QueryTypeEnum.SQL) + .queryReq(getQueryDslReq(getLlmResp())) + .build(); + return semanticLayer.explain(explainSqlReq, user); + } catch (Exception e) { + log.error("explain error explainSqlReq:{}", explainSqlReq, e); + } + return null; + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/PluginSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/PluginSemanticQuery.java index da2b48072..6e0341af0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/PluginSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/plugin/PluginSemanticQuery.java @@ -1,7 +1,9 @@ package com.tencent.supersonic.chat.query.plugin; +import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -17,5 +19,8 @@ public abstract class PluginSemanticQuery implements SemanticQuery { return parseInfo; } - + @Override + public ExplainResp explain(User user) { + return null; + } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java index d8348c077..4b5a8ac74 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java @@ -21,8 +21,11 @@ import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import java.io.Serializable; @@ -215,6 +218,22 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { return queryResult; } + + @Override + public ExplainResp explain(User user) { + ExplainSqlReq explainSqlReq = null; + try { + explainSqlReq = ExplainSqlReq.builder() + .queryTypeEnum(QueryTypeEnum.STRUCT) + .queryReq(convertQueryStruct()) + .build(); + return semanticLayer.explain(explainSqlReq, user); + } catch (Exception e) { + log.error("explain error explainSqlReq:{}", explainSqlReq, e); + } + return null; + } + public QueryResult multiStructExecute(User user) { String queryMode = parseInfo.getQueryMode(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 0e1a1e59f..e569efe60 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -27,6 +27,7 @@ import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.chat.service.StatisticsService; import com.tencent.supersonic.chat.utils.ComponentFactory; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import java.util.List; import java.util.ArrayList; import java.util.Set; @@ -35,9 +36,7 @@ import java.util.Comparator; import java.util.Objects; import java.util.stream.Collectors; -//import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; -//import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; @@ -64,8 +63,6 @@ public class QueryServiceImpl implements QueryService { @Autowired private StatisticsService statisticsService; - private final String entity = "ENTITY"; - @Value("${time.threshold: 100}") private Integer timeThreshold; @@ -109,12 +106,16 @@ public class QueryServiceImpl implements QueryService { .map(SemanticQuery::getParseInfo) .sorted(Comparator.comparingDouble(SemanticParseInfo::getScore).reversed()) .collect(Collectors.toList()); + selectedParses.forEach(parseInfo -> { - if (parseInfo.getQueryMode().contains(entity)) { + String queryMode = parseInfo.getQueryMode(); + if (QueryManager.isEntityQuery(queryMode)) { EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class) .getEntityInfo(parseInfo, queryReq.getUser()); parseInfo.setEntityInfo(entityInfo); } + addExplainSql(queryReq, parseInfo); + }); List candidateParses = queryCtx.getCandidateQueries().stream() .map(SemanticQuery::getParseInfo).collect(Collectors.toList()); @@ -138,6 +139,19 @@ public class QueryServiceImpl implements QueryService { return parseResult; } + private void addExplainSql(QueryReq queryReq, SemanticParseInfo parseInfo) { + SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); + if (Objects.isNull(semanticQuery)) { + return; + } + semanticQuery.setParseInfo(parseInfo); + ExplainResp explain = semanticQuery.explain(queryReq.getUser()); + if (Objects.isNull(explain)) { + return; + } + parseInfo.setSql(explain.getSql()); + } + @Override public QueryResult performExecution(ExecuteQueryReq queryReq) throws Exception { ChatParseDO chatParseDO = chatService.getParseInfo(queryReq.getQueryId(), @@ -155,9 +169,9 @@ public class QueryServiceImpl implements QueryService { chatCtx.setAgentId(queryReq.getAgentId()); Long startTime = System.currentTimeMillis(); QueryResult queryResult = semanticQuery.execute(queryReq.getUser()); - Long endTime = System.currentTimeMillis(); + if (queryResult != null) { - timeCostDOList.add(StatisticsDO.builder().cost((int) (endTime - startTime)) + timeCostDOList.add(StatisticsDO.builder().cost((int) (System.currentTimeMillis() - startTime)) .interfaceName(semanticQuery.getClass().getSimpleName()).type(CostType.QUERY.getType()).build()); saveInfo(timeCostDOList, queryReq.getQueryText(), queryReq.getQueryId(), queryReq.getUser().getName(), queryReq.getChatId().longValue()); @@ -169,7 +183,6 @@ public class QueryServiceImpl implements QueryService { } chatCtx.setQueryText(queryReq.getQueryText()); chatCtx.setUser(queryReq.getUser().getName()); - //chatService.addQuery(queryResult, chatCtx); chatService.updateQuery(queryReq.getQueryId(), queryResult, chatCtx); } else { chatService.deleteChatQuery(queryReq.getQueryId()); @@ -179,8 +192,8 @@ public class QueryServiceImpl implements QueryService { } public void saveInfo(List timeCostDOList, - String queryText, Long queryId, - String userName, Long chatId) { + String queryText, Long queryId, + String userName, Long chatId) { List list = timeCostDOList.stream() .filter(o -> o.getCost() > timeThreshold).collect(Collectors.toList()); list.forEach(o -> { @@ -264,13 +277,6 @@ public class QueryServiceImpl implements QueryService { dateConf.setPeriod("DAY"); queryStructReq.setDateInfo(dateConf); queryStructReq.setLimit(20L); - - // List aggregators = new ArrayList<>(); - // Aggregator aggregator = new Aggregator(dimensionValueReq.getQueryFilter().getBizName(), - // AggOperatorEnum.DISTINCT); - // aggregators.add(aggregator); - // queryStructReq.setAggregators(aggregators); - queryStructReq.setModelId(dimensionValueReq.getModelId()); queryStructReq.setNativeQuery(true); List groups = new ArrayList<>(); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java index a28b71682..9038330cf 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/DefaultSemanticConfig.java @@ -38,4 +38,6 @@ public class DefaultSemanticConfig { @Value("${fetchModelList.path:/api/semantic/schema/model/list}") private String fetchModelListPath; + @Value("${explain.path:/api/semantic/query/explain}") + private String explainPath; } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java index d0dceaaa5..c675a7217 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/LocalSemanticLayer.java @@ -8,12 +8,14 @@ import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.MetricResp; import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; @@ -95,6 +97,12 @@ public class LocalSemanticLayer extends BaseSemanticLayer { return schemaService.getModelList(user, authType, domainId); } + @Override + public ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception { + queryService = ContextUtils.getBean(QueryService.class); + return queryService.explain(explainSqlReq, user); + } + @Override public PageInfo getDimensionPage(PageDimensionReq pageDimensionCmd) { dimensionService = ContextUtils.getBean(DimensionService.class); diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java index 05630263d..723dc6a40 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/semantic/RemoteSemanticLayer.java @@ -1,38 +1,44 @@ package com.tencent.supersonic.knowledge.semantic; +import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER; +import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER; + import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageInfo; import com.google.gson.Gson; import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig; import com.tencent.supersonic.auth.api.authentication.constant.UserConstants; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.ResultData; +import com.tencent.supersonic.common.pojo.ReturnCode; +import com.tencent.supersonic.common.pojo.enums.AuthType; +import com.tencent.supersonic.common.pojo.exception.CommonException; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.S2ThreadContext; import com.tencent.supersonic.common.util.ThreadContext; -import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.common.pojo.enums.AuthType; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; import com.tencent.supersonic.semantic.api.model.request.PageDimensionReq; import com.tencent.supersonic.semantic.api.model.request.PageMetricReq; -import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.semantic.api.model.response.ModelResp; -import com.tencent.supersonic.semantic.api.model.response.MetricResp; -import com.tencent.supersonic.semantic.api.model.response.DomainResp; import com.tencent.supersonic.semantic.api.model.response.DimensionResp; +import com.tencent.supersonic.semantic.api.model.response.DomainResp; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; +import com.tencent.supersonic.semantic.api.model.response.MetricResp; +import com.tencent.supersonic.semantic.api.model.response.ModelResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; +import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; -import com.tencent.supersonic.common.pojo.exception.CommonException; -import com.tencent.supersonic.common.pojo.ResultData; -import com.tencent.supersonic.common.pojo.ReturnCode; - import java.net.URI; +import java.net.URL; +import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; -import java.util.LinkedHashMap; - import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.BeanUtils; @@ -45,11 +51,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; -import static com.tencent.supersonic.common.pojo.Constants.TRUE_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.LIST_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.TOTAL_LOWER; -import static com.tencent.supersonic.common.pojo.Constants.PAGESIZE_LOWER; - @Slf4j public class RemoteSemanticLayer extends BaseSemanticLayer { @@ -61,6 +62,10 @@ public class RemoteSemanticLayer extends BaseSemanticLayer { new ParameterizedTypeReference>() { }; + private ParameterizedTypeReference> explainTypeRef = + new ParameterizedTypeReference>() { + }; + @Override public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); @@ -130,9 +135,10 @@ public class RemoteSemanticLayer extends BaseSemanticLayer { fillToken(headers); DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); - URI requestUrl = UriComponentsBuilder.fromHttpUrl( - defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getFetchModelSchemaPath()).build() - .encode().toUri(); + String semanticUrl = defaultSemanticConfig.getSemanticUrl(); + String fetchModelSchemaPath = defaultSemanticConfig.getFetchModelSchemaPath(); + URI requestUrl = UriComponentsBuilder.fromHttpUrl(semanticUrl + fetchModelSchemaPath) + .build().encode().toUri(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); filter.setModelIds(ids); ParameterizedTypeReference>> responseTypeRef = @@ -179,6 +185,39 @@ public class RemoteSemanticLayer extends BaseSemanticLayer { return JsonUtil.toList(JsonUtil.toString(domainDescListObject), ModelResp.class); } + @Override + public ExplainResp explain(ExplainSqlReq explainResp, User user) throws Exception { + DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); + String semanticUrl = defaultSemanticConfig.getSemanticUrl(); + String explainPath = defaultSemanticConfig.getExplainPath(); + URL url = new URL(new URL(semanticUrl), explainPath); + return explain(url.toString(), JsonUtil.toString(explainResp)); + } + + public ExplainResp explain(String url, String jsonReq) { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + fillToken(headers); + URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); + HttpEntity entity = new HttpEntity<>(jsonReq, headers); + log.info("url:{},explain:{}", url, entity.getBody()); + ResultData responseBody; + try { + RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); + + ResponseEntity> responseEntity = restTemplate.exchange( + requestUrl, HttpMethod.POST, entity, explainTypeRef); + log.info("ApiResponse responseBody:{}", responseEntity); + responseBody = responseEntity.getBody(); + if (Objects.nonNull(responseBody.getData())) { + return responseBody.getData(); + } + return null; + } catch (Exception e) { + throw new RuntimeException("explain interface error,url:" + url, e); + } + } + public Object fetchHttpResult(String url, String bodyJson, HttpMethod httpMethod) { HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); diff --git a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java index 65d870252..e17430eda 100644 --- a/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelperTest.java @@ -75,6 +75,13 @@ class SqlParserSelectHelperTest { + "user_id like '%alice%' AND publish_date > 10000 ORDER BY pv DESC LIMIT 1"); System.out.println(filterExpression); + + filterExpression = SqlParserSelectHelper.getFilterExpression( + "SELECT department, pv FROM s2 WHERE " + + "user_id like '%alice%' AND publish_date > 10000 " + + "group by department having sum(pv) > 2000 ORDER BY pv DESC LIMIT 1"); + + System.out.println(filterExpression); } diff --git a/launchers/chat/src/main/resources/db/chat-schema-h2.sql b/launchers/chat/src/main/resources/db/chat-schema-h2.sql index 8bcad7b6d..8e28d349e 100644 --- a/launchers/chat/src/main/resources/db/chat-schema-h2.sql +++ b/launchers/chat/src/main/resources/db/chat-schema-h2.sql @@ -1,3 +1,4 @@ +-- chat tables CREATE TABLE IF NOT EXISTS `s2_chat_context` ( `chat_id` BIGINT NOT NULL , -- context chat id @@ -7,7 +8,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat_context` `semantic_parse` LONGVARCHAR DEFAULT NULL , -- parse data `ext_data` LONGVARCHAR DEFAULT NULL , -- extend data PRIMARY KEY (`chat_id`) -); + ); CREATE TABLE IF NOT EXISTS `s2_chat` ( @@ -21,7 +22,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat` `is_delete` INT DEFAULT '0' COMMENT 'is deleted', `is_top` INT DEFAULT '0' COMMENT 'is top', PRIMARY KEY (`chat_id`) -) ; + ) ; CREATE TABLE `s2_chat_query` @@ -64,72 +65,21 @@ CREATE TABLE `s2_chat_statistics` ); CREATE TABLE IF NOT EXISTS `s2_chat_config` ( - `id` INT NOT NULL AUTO_INCREMENT, - `model_id` INT DEFAULT NULL , - `chat_detail_config` varchar(655) , + `id` INT NOT NULL AUTO_INCREMENT, + `model_id` INT DEFAULT NULL , + `chat_detail_config` varchar(655) , `chat_agg_config` varchar(655) , - `recommended_questions` varchar(1500) , + `recommended_questions` varchar(1500) , `created_at` TIMESTAMP NOT NULL , `updated_at` TIMESTAMP NOT NULL , `created_by` varchar(100) NOT NULL , `updated_by` varchar(100) NOT NULL , `status` INT NOT NULL DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted PRIMARY KEY (`id`) -) ; - - --- CREATE TABLE IF NOT EXISTS `s2_chat_config` ( --- `id` INT NOT NULL AUTO_INCREMENT, --- `domain_id` INT DEFAULT NULL , --- `default_metrics` varchar(655) DEFAULT NULL, --- `visibility` varchar(655) , -- invisible dimension metric information --- `entity_info` varchar(655) , --- `dictionary_info` varchar(655) , -- dictionary-related dimension setting information --- `created_at` TIMESTAMP NOT NULL , --- `updated_at` TIMESTAMP NOT NULL , --- `created_by` varchar(100) NOT NULL , --- `updated_by` varchar(100) NOT NULL , --- `status` INT NOT NULL DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted --- PRIMARY KEY (`id`) --- ) ; + ) ; COMMENT ON TABLE s2_chat_config IS 'chat config information table '; - - -CREATE TABLE IF NOT EXISTS `s2_dictionary` ( - `id` INT NOT NULL AUTO_INCREMENT, - `domain_id` INT NOT NULL , - `dim_value_infos` LONGVARCHAR , -- dimension value setting information - `created_at` TIMESTAMP NOT NULL , - `updated_at` TIMESTAMP NOT NULL , - `created_by` varchar(100) NOT NULL , - `updated_by` varchar(100) DEFAULT NULL , - `status` INT NOT NULL DEFAULT '0' , -- domain extension information status : 0 is normal, 1 is off the shelf, 2 is deleted - PRIMARY KEY (`id`), - UNIQUE (domain_id) - ); -COMMENT ON TABLE s2_dictionary IS 'dictionary configuration information table'; - - -CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( - `id` INT NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL , -- task name - `description` varchar(255) , - `command`LONGVARCHAR NOT NULL , -- task Request Parameters - `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 - `dimension_ids` varchar(500) , - `status` INT NOT NULL , -- the final status of the task - `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , - `created_by` varchar(100) NOT NULL , - `progress` DOUBLE default 0.00 , -- task real-time progress - `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds - `message` LONGVARCHAR , -- remark related information - PRIMARY KEY (`id`) -); -COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; - - create table s2_user ( id INT AUTO_INCREMENT, @@ -141,6 +91,28 @@ create table s2_user ); COMMENT ON TABLE s2_user IS 'user information table'; + +CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info` ( + `id` INT NOT NULL AUTO_INCREMENT, + `trace_id` varchar(200) NOT NULL , + `model_id` INT NOT NULL , + `dimensions`LONGVARCHAR , + `metrics`LONGVARCHAR , + `orders`LONGVARCHAR , + `filters`LONGVARCHAR , + `date_info`LONGVARCHAR , + `limit` INT NOT NULL , + `native_query` TINYINT NOT NULL DEFAULT '0' , + `sql`LONGVARCHAR , + `created_at` TIMESTAMP NOT NULL , + `created_by` varchar(100) NOT NULL , + `status` INT NOT NULL , + `elapsed_ms` bigINT DEFAULT NULL , + PRIMARY KEY (`id`) + ); +COMMENT ON TABLE s2_semantic_pasre_info IS 'semantic layer sql parsing information table'; + + CREATE TABLE IF NOT EXISTS `s2_plugin` ( `id` INT AUTO_INCREMENT, @@ -157,5 +129,50 @@ CREATE TABLE IF NOT EXISTS `s2_plugin` `config` LONGVARCHAR NULL, `comment` LONGVARCHAR NULL, PRIMARY KEY (`id`) -); COMMENT ON TABLE s2_plugin IS 'plugin information table'; + ); COMMENT ON TABLE s2_plugin IS 'plugin information table'; + +CREATE TABLE IF NOT EXISTS s2_agent +( + id int AUTO_INCREMENT, + name varchar(100) null, + description varchar(500) null, + status int null, + examples varchar(500) null, + config varchar(2000) null, + created_by varchar(100) null, + created_at TIMESTAMP null, + updated_by varchar(100) null, + updated_at TIMESTAMP null, + enable_search int null, + PRIMARY KEY (`id`) + ); COMMENT ON TABLE s2_agent IS 'agent information table'; + + +-------demo for semantic and chat +CREATE TABLE IF NOT EXISTS `s2_user_department` ( + `user_name` varchar(200) NOT NULL, + `department` varchar(200) NOT NULL -- department of user + ); +COMMENT ON TABLE s2_user_department IS 'user_department_info'; + + +CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( + `id` INT NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL , -- task name + `description` varchar(255) , + `command`LONGVARCHAR NOT NULL , -- task Request Parameters + `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 + `status` INT NOT NULL , -- the final status of the task + `dimension_ids` varchar(500) NULL , + `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , + `created_by` varchar(100) NOT NULL , + `progress` DOUBLE default 0.00 , -- task real-time progress + `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds + `message` LONGVARCHAR , -- remark related information + PRIMARY KEY (`id`) + ); +COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table'; + + + diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ExplainResp.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ExplainResp.java new file mode 100644 index 000000000..96f5fa8e6 --- /dev/null +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/response/ExplainResp.java @@ -0,0 +1,19 @@ +package com.tencent.supersonic.semantic.api.model.response; + +import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +@Data +@ToString +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ExplainResp implements Serializable { + + private String sql; + +} diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/ExplainSqlReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/ExplainSqlReq.java new file mode 100644 index 000000000..eabf702fa --- /dev/null +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/ExplainSqlReq.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.semantic.api.query.request; + +import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +@Data +@ToString +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ExplainSqlReq { + + private QueryTypeEnum queryTypeEnum; + + private T queryReq; +} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java index ef77abbe5..eafe6c0d8 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/rest/QueryController.java @@ -2,18 +2,22 @@ package com.tencent.supersonic.semantic.query.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.model.response.SqlParserResp; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; +import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; +import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; -import com.tencent.supersonic.semantic.api.query.request.ParseSqlReq; -import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; -import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; +import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; -import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine; -import com.tencent.supersonic.semantic.query.service.QueryService; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; +import com.tencent.supersonic.semantic.query.service.QueryService; +import com.tencent.supersonic.semantic.query.service.SemanticQueryEngine; import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -89,10 +93,36 @@ public class QueryController { @PostMapping("/queryDimValue") public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq, - HttpServletRequest request, - HttpServletResponse response) { + HttpServletRequest request, + HttpServletResponse response) { User user = UserHolder.findUser(request, response); return queryService.queryDimValue(queryDimValueReq, user); } + @PostMapping("/explain") + public ExplainResp explain(@RequestBody ExplainSqlReq explainSqlReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + + User user = UserHolder.findUser(request, response); + String queryReqJson = JsonUtil.toString(explainSqlReq.getQueryReq()); + QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); + + if (QueryTypeEnum.SQL.equals(queryTypeEnum)) { + QueryDslReq queryDslReq = JsonUtil.toObject(queryReqJson, QueryDslReq.class); + ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() + .queryReq(queryDslReq) + .queryTypeEnum(queryTypeEnum).build(); + return queryService.explain(explainSqlReqNew, user); + } + if (QueryTypeEnum.STRUCT.equals(queryTypeEnum)) { + QueryStructReq queryStructReq = JsonUtil.toObject(queryReqJson, QueryStructReq.class); + ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() + .queryReq(queryStructReq) + .queryTypeEnum(queryTypeEnum).build(); + return queryService.explain(explainSqlReqNew, user); + } + return null; + } + } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java index 28b04ed95..8e8a6e4c8 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryService.java @@ -1,11 +1,13 @@ package com.tencent.supersonic.semantic.query.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; -import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; +import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; import com.tencent.supersonic.semantic.api.query.response.ItemUseResp; import java.util.List; @@ -25,4 +27,5 @@ public interface QueryService { List getStatInfo(ItemUseReq itemUseCommend); + ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception; } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index 1b94f9651..3c0c0fc11 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -6,12 +6,15 @@ import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.semantic.api.model.enums.QueryTypeEnum; import com.tencent.supersonic.semantic.api.model.request.ModelSchemaFilterReq; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.ModelSchemaResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Cache; import com.tencent.supersonic.semantic.api.query.pojo.Filter; +import com.tencent.supersonic.semantic.api.query.request.ExplainSqlReq; import com.tencent.supersonic.semantic.api.query.request.ItemUseReq; import com.tencent.supersonic.semantic.api.query.request.QueryDimValueReq; import com.tencent.supersonic.semantic.api.query.request.QueryDslReq; @@ -64,6 +67,11 @@ public class QueryServiceImpl implements QueryService { @Override public Object queryBySql(QueryDslReq querySqlCmd, User user) throws Exception { + QueryStatement queryStatement = convertToQueryStatement(querySqlCmd, user); + return semanticQueryEngine.execute(queryStatement); + } + + private QueryStatement convertToQueryStatement(QueryDslReq querySqlCmd, User user) throws Exception { ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); List modelIds = new ArrayList<>(); modelIds.add(querySqlCmd.getModelId()); @@ -74,7 +82,7 @@ public class QueryServiceImpl implements QueryService { QueryStatement queryStatement = queryReqConverter.convert(querySqlCmd, domainSchemas); queryStatement.setModelId(querySqlCmd.getModelId()); - return semanticQueryEngine.execute(queryStatement); + return queryStatement; } @Override @@ -183,6 +191,32 @@ public class QueryServiceImpl implements QueryService { return statInfos; } + @Override + public ExplainResp explain(ExplainSqlReq explainSqlReq, User user) throws Exception { + QueryTypeEnum queryTypeEnum = explainSqlReq.getQueryTypeEnum(); + T queryReq = explainSqlReq.getQueryReq(); + + if (QueryTypeEnum.SQL.equals(queryTypeEnum) && queryReq instanceof QueryDslReq) { + QueryStatement queryStatement = convertToQueryStatement((QueryDslReq) queryReq, user); + return getExplainResp(queryStatement); + } + if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) { + QueryStatement queryStatement = semanticQueryEngine.plan((QueryStructReq) queryReq); + return getExplainResp(queryStatement); + } + + throw new IllegalArgumentException("Parameters are invalid, explainSqlReq: " + explainSqlReq); + } + + private ExplainResp getExplainResp(QueryStatement queryStatement) { + String sql = ""; + if (Objects.nonNull(queryStatement)) { + sql = queryStatement.getSql(); + } + return ExplainResp.builder().sql(sql).build(); + } + + private boolean isCache(QueryStructReq queryStructCmd) { if (!cacheEnable) { return false;