diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java index bedc1f9ea..920f7e35d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.request.PageDimensionReq; import com.tencent.supersonic.headless.api.request.PageMetricReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.DomainResp; @@ -19,12 +19,12 @@ import com.tencent.supersonic.headless.api.response.ExplainResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.QueryService; import com.tencent.supersonic.headless.server.service.SchemaService; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -40,20 +40,20 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { @SneakyThrows @Override - public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { + public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { - QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); - queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL()); - queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet()); - queryS2SQLReq.setVariables(new HashMap<>()); - return queryByS2SQL(queryS2SQLReq, user); + QuerySqlReq querySQLReq = new QuerySqlReq(); + querySQLReq.setSql(queryStructReq.getCorrectS2SQL()); + querySQLReq.setModelIds(queryStructReq.getModelIdSet()); + querySQLReq.setParams(new ArrayList<>()); + return queryByS2SQL(querySQLReq, user); } queryService = ContextUtils.getBean(QueryService.class); return queryService.queryByStructWithAuth(queryStructReq, user); } @Override - public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { + public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { try { queryService = ContextUtils.getBean(QueryService.class); return queryService.queryByMultiStruct(queryMultiStructReq, user); @@ -65,15 +65,15 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { @Override @SneakyThrows - public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) { + public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) { queryService = ContextUtils.getBean(QueryService.class); - Object object = queryService.queryBySql(queryS2SQLReq, user); - return JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class); + Object object = queryService.queryBySql(querySQLReq, user); + return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class); } @Override @SneakyThrows - public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { + public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { queryService = ContextUtils.getBean(QueryService.class); return queryService.queryDimValue(queryDimValueReq, user); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java index b408aece8..010cdeee1 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java @@ -20,24 +20,24 @@ import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.S2ThreadContext; import com.tencent.supersonic.common.util.ThreadContext; +import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.request.PageDimensionReq; import com.tencent.supersonic.headless.api.request.PageMetricReq; +import com.tencent.supersonic.headless.api.request.QueryDimValueReq; +import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; +import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.DomainResp; import com.tencent.supersonic.headless.api.response.ExplainResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.api.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.request.QueryDimValueReq; -import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; -import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import java.net.URI; import java.net.URL; -import java.util.HashMap; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; @@ -61,8 +61,8 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { private AuthenticationConfig authenticationConfig; - private ParameterizedTypeReference> structTypeRef = - new ParameterizedTypeReference>() { + private ParameterizedTypeReference> structTypeRef = + new ParameterizedTypeReference>() { }; private ParameterizedTypeReference> explainTypeRef = @@ -70,13 +70,13 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { }; @Override - public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) { + public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { - QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); - queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL()); - queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet()); - queryS2SQLReq.setVariables(new HashMap<>()); - return queryByS2SQL(queryS2SQLReq, user); + QuerySqlReq querySQLReq = new QuerySqlReq(); + querySQLReq.setSql(queryStructReq.getCorrectS2SQL()); + querySQLReq.setModelIds(queryStructReq.getModelIdSet()); + querySQLReq.setParams(new ArrayList<>()); + return queryByS2SQL(querySQLReq, user); } DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); @@ -86,7 +86,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { + public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); return searchByRestTemplate( defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByMultiStructPath(), @@ -94,30 +94,30 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) { + public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(), - new Gson().toJson(queryS2SQLReq)); + new Gson().toJson(querySQLReq)); } - public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) { + public SemanticQueryResp searchByRestTemplate(String url, String jsonReq) { HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); fillToken(headers); URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri(); HttpEntity entity = new HttpEntity<>(jsonReq, headers); log.info("url:{},searchByRestTemplate:{}", url, entity.getBody()); - ResultData responseBody; + ResultData responseBody; try { RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class); - ResponseEntity> responseEntity = restTemplate.exchange( + ResponseEntity> responseEntity = restTemplate.exchange( requestUrl, HttpMethod.POST, entity, structTypeRef); responseBody = responseEntity.getBody(); log.info("ApiResponse responseBody:{}", responseBody); - QueryResultWithSchemaResp schemaResp = new QueryResultWithSchemaResp(); + SemanticQueryResp schemaResp = new SemanticQueryResp(); if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) { - QueryResultWithSchemaResp data = responseBody.getData(); + SemanticQueryResp data = responseBody.getData(); schemaResp.setColumns(data.getColumns()); schemaResp.setResultList(data.getResultList()); schemaResp.setSql(data.getSql()); @@ -131,7 +131,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { + public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getQueryDimValuePath(), diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java index c9bd7a244..bca60d8d8 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java @@ -12,10 +12,10 @@ import com.tencent.supersonic.headless.api.response.ExplainResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; @@ -34,13 +34,13 @@ import java.util.List; */ public interface SemanticInterpreter { - QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user); + SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user); - QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); + SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); - QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user); + SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user); - QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); + SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); List getModelSchema(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java index 0e2e9865b..eaecdc360 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java @@ -16,7 +16,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.enums.QueryType; import com.tencent.supersonic.headless.api.request.ExplainSqlReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.ExplainResp; import java.io.Serializable; @@ -121,9 +121,9 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } QueryStructReq queryStructReq = convertQueryStruct(); convertBizNameToName(semanticSchema, queryStructReq); - QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq); - parseInfo.getSqlInfo().setS2SQL(queryS2SQLReq.getSql()); - parseInfo.getSqlInfo().setCorrectS2SQL(queryS2SQLReq.getSql()); + QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); + parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql()); + parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql()); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java index 667ea3fc3..64a6ff54d 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java @@ -20,7 +20,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -59,8 +59,8 @@ public class MetricAnalyzeQuery extends LLMSemanticQuery { queryStructReq.setS2SQL(parseInfo.getSqlInfo().getQuerySQL()); } - QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user); - String text = generateTableText(queryResultWithSchemaResp); + SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructReq, user); + String text = generateTableText(semanticQueryResp); Map properties = parseInfo.getProperties(); Map replacedMap = new HashMap<>(); String textReplaced = replaceText((String) properties.get("queryText"), @@ -130,7 +130,7 @@ public class MetricAnalyzeQuery extends LLMSemanticQuery { return text; } - public static String generateTableText(QueryResultWithSchemaResp result) { + public static String generateTableText(SemanticQueryResp result) { StringBuilder tableBuilder = new StringBuilder(); for (QueryColumn column : result.getColumns()) { tableBuilder.append(column.getName()).append("\t"); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java index 5ca48dbb4..7ea44ef5a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java @@ -11,8 +11,8 @@ import com.tencent.supersonic.chat.core.utils.QueryReqBuilder; import com.tencent.supersonic.chat.core.query.QueryManager; import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery; import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -42,8 +42,8 @@ public class LLMSqlQuery extends LLMSemanticQuery { long startTime = System.currentTimeMillis(); String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); - QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds()); - QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2SQL(queryS2SQLReq, user); + QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds()); + SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySQLReq, user); log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java index 7799fd646..c645c07fc 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java @@ -22,7 +22,7 @@ import com.tencent.supersonic.common.pojo.ModelCluster; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import lombok.ToString; @@ -207,7 +207,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL()); queryStructReq.setCorrectS2SQL(parseInfo.getSqlInfo().getCorrectS2SQL()); } - QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); + SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); if (queryResp != null) { queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); @@ -242,7 +242,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { QueryResult queryResult = new QueryResult(); QueryMultiStructReq queryMultiStructReq = convertQueryMultiStruct(); - QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByMultiStruct(queryMultiStructReq, user); + SemanticQueryResp queryResp = semanticInterpreter.queryByMultiStruct(queryMultiStructReq, user); if (queryResp != null) { queryResult.setQueryAuthorization(queryResp.getQueryAuthorization()); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java index 8ed82db07..57e95a24c 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java @@ -36,7 +36,7 @@ import com.tencent.supersonic.common.pojo.enums.RatioOverType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import java.text.DecimalFormat; import java.time.DayOfWeek; import java.time.LocalDate; @@ -105,7 +105,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { public void fillAggregateInfo(User user, QueryResult queryResult) { if (Objects.nonNull(queryResult)) { - QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp(); + SemanticQueryResp queryResp = new SemanticQueryResp(); queryResp.setColumns(queryResult.getQueryColumns()); queryResp.setResultList(queryResult.getQueryResults()); AggregateInfo aggregateInfo = getAggregateInfo(user, parseInfo, queryResp); @@ -114,7 +114,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { } public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo, - QueryResultWithSchemaResp result) { + SemanticQueryResp result) { AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class); if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio()) { @@ -170,7 +170,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { } private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric, - AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results) { + AggOperatorEnum aggOperatorEnum, SemanticQueryResp results) { MetricInfo metricInfo = new MetricInfo(); metricInfo.setStatistics(new HashMap<>()); QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum); @@ -180,7 +180,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField))); queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, results)); - QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); + SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user); if (Objects.nonNull(queryResp) && !CollectionUtils.isEmpty(queryResp.getResultList())) { @@ -220,7 +220,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery { } private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo, - QueryResultWithSchemaResp results) { + SemanticQueryResp results) { String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo()); Optional lastDayOp = results.getResultList().stream() .map(r -> r.get(dateField).toString()) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/CacheUtils.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/CacheUtils.java deleted file mode 100644 index 0ca948845..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/CacheUtils.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.tencent.supersonic.chat.core.utils; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.tencent.supersonic.chat.core.pojo.ChatContext; -import com.tencent.supersonic.chat.core.pojo.QueryContext; - -import java.util.concurrent.TimeUnit; - -public class CacheUtils { - private static final Cache cache = Caffeine.newBuilder() - .expireAfterWrite(1200, TimeUnit.SECONDS) - .expireAfterAccess(1200, TimeUnit.SECONDS) - .maximumSize(1000) - .build(); - - public static void put(QueryContext queryContext, ChatContext chatCtx, Object v) { - String key = chatCtx.getUser() + "_" + chatCtx.getChatId() + "_" + queryContext.getRequest().getQueryText(); - cache.put(key, v); - } - - public static Object get(QueryContext queryContext, ChatContext chatCtx) { - String key = chatCtx.getUser() + "_" + chatCtx.getChatId() + "_" + queryContext.getRequest().getQueryText(); - return cache.getIfPresent(key); - } -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java index f4980150b..48e6c37f5 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java @@ -18,7 +18,7 @@ import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.request.QueryStructReq; import java.util.ArrayList; import java.util.Arrays; @@ -53,20 +53,20 @@ public class DictQueryHelper { List data = new ArrayList<>(); QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict); try { - QueryResultWithSchemaResp queryResultWithColumns = semanticInterpreter.queryByStruct(queryStructCmd, user); + SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructCmd, user); - log.info("fetchDimValueSingle sql:{}", queryResultWithColumns.getSql()); + log.info("fetchDimValueSingle sql:{}", semanticQueryResp.getSql()); String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId()); - String dimNameRewrite = rewriteDimName(queryResultWithColumns.getColumns(), dim4Dict.getBizName()); - data = generateFileData(queryResultWithColumns.getResultList(), nature, dimNameRewrite, + String dimNameRewrite = rewriteDimName(semanticQueryResp.getColumns(), dim4Dict.getBizName()); + data = generateFileData(semanticQueryResp.getResultList(), nature, dimNameRewrite, defaultMetricDesc.getBizName(), dim4Dict); if (!CollectionUtils.isEmpty(data)) { int size = (data.size() > printDataShow) ? printDataShow : data.size(); log.info("data:{}", data.subList(0, size)); } else { log.warn("data is empty. nature:{}", nature); - if (Objects.nonNull(queryResultWithColumns)) { - log.warn("sql:{}", queryResultWithColumns.getSql()); + if (Objects.nonNull(semanticQueryResp)) { + log.warn("sql:{}", semanticQueryResp.getSql()); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java index 85d9a819a..db190ab51 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java @@ -14,7 +14,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -133,13 +133,13 @@ public class QueryReqBuilder { * @param modelIds * @return */ - public static QueryS2SQLReq buildS2SQLReq(String querySql, Set modelIds) { - QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); + public static QuerySqlReq buildS2SQLReq(String querySql, Set modelIds) { + QuerySqlReq querySQLReq = new QuerySqlReq(); if (Objects.nonNull(querySql)) { - queryS2SQLReq.setSql(querySql); + querySQLReq.setSql(querySql); } - queryS2SQLReq.setModelIds(modelIds); - return queryS2SQLReq; + querySQLReq.setModelIds(modelIds); + return querySQLReq; } private static List getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java index 7c0d346aa..3afa09d47 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java @@ -27,7 +27,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; @@ -162,7 +162,7 @@ public class SemanticService { List entities = Collections.singletonList(entity); - QueryResultWithSchemaResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, parseInfo, entities, + SemanticQueryResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, parseInfo, entities, user); if (queryResultWithColumns != null) { @@ -183,7 +183,7 @@ public class SemanticService { } } - public QueryResultWithSchemaResp getQueryResultWithSchemaResp(EntityInfo modelInfo, SemanticParseInfo parseInfo, + public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo modelInfo, SemanticParseInfo parseInfo, List entities, User user) { if (CollectionUtils.isEmpty(entities)) { return null; @@ -219,7 +219,7 @@ public class SemanticService { chatFilters.add(chatFilter); semanticParseInfo.setDimensionFilters(chatFilters); - QueryResultWithSchemaResp queryResultWithColumns = null; + SemanticQueryResp queryResultWithColumns = null; try { QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo); queryResultWithColumns = semanticInterpreter.queryByStruct(queryStructReq, user); diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java index 69b2c9c36..b0832f42e 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java @@ -64,7 +64,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -628,7 +628,7 @@ public class QueryServiceImpl implements QueryService { @Override public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { - QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp(); + SemanticQueryResp semanticQueryResp = new SemanticQueryResp(); SemanticService semanticService = ContextUtils.getBean(SemanticService.class); SemanticSchema semanticSchema = semanticService.getSemanticSchema(); SchemaElement schemaElement = semanticSchema.getDimensions(dimensionValueReq.getElementID()); @@ -638,8 +638,8 @@ public class QueryServiceImpl implements QueryService { List dimensionValues = getDimensionValues(dimensionValueReq, detectModelIds); // if the search results is null,search dimensionValue from database if (CollectionUtils.isEmpty(dimensionValues)) { - queryResultWithSchemaResp = queryDatabase(dimensionValueReq, user); - return queryResultWithSchemaResp; + semanticQueryResp = queryDatabase(dimensionValueReq, user); + return semanticQueryResp; } List columns = new ArrayList<>(); QueryColumn queryColumn = new QueryColumn(); @@ -654,9 +654,9 @@ public class QueryServiceImpl implements QueryService { map.put(dimensionValueReq.getBizName(), o); resultList.add(map); }); - queryResultWithSchemaResp.setColumns(columns); - queryResultWithSchemaResp.setResultList(resultList); - return queryResultWithSchemaResp; + semanticQueryResp.setColumns(columns); + semanticQueryResp.setResultList(resultList); + return semanticQueryResp; } private List getDimensionValues(DimensionValueReq dimensionValueReq, Set detectModelIds) { @@ -682,7 +682,7 @@ public class QueryServiceImpl implements QueryService { .collect(Collectors.toList()); } - private QueryResultWithSchemaResp queryDatabase(DimensionValueReq dimensionValueReq, User user) { + private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) { QueryStructReq queryStructReq = new QueryStructReq(); DateConf dateConf = new DateConf(); diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java index 88982e86e..6b86110b2 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java @@ -10,7 +10,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import java.util.ArrayList; import java.util.Arrays; @@ -51,18 +51,18 @@ class QueryReqBuilderTest { orders.add(order); queryStructReq.setOrders(orders); - QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq); + QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); Assert.assertEquals( "SELECT department, SUM(pv) AS pv FROM 内容库 " + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP " - + "BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql()); + + "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql()); queryStructReq.setQueryType(QueryType.TAG); - queryS2SQLReq = queryStructReq.convert(queryStructReq); + querySQLReq = queryStructReq.convert(queryStructReq); Assert.assertEquals( "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " + "ORDER BY uv LIMIT 2000", - queryS2SQLReq.getSql()); + querySQLReq.getSql()); } diff --git a/common/src/main/META-INF/spring.factories b/common/src/main/META-INF/spring.factories index c4e3d022e..381f79ce8 100644 --- a/common/src/main/META-INF/spring.factories +++ b/common/src/main/META-INF/spring.factories @@ -1,2 +1,2 @@ -com.tencent.supersonic.common.util.cache.CacheUtils=\ - com.tencent.supersonic.common.util.cache.CaffeineCacheImpl +com.tencent.supersonic.headless.server.cache.CacheManager=\ + com.tencent.supersonic.headless.server.cache.CaffeineCacheManager diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java index 1d10f90cf..8c09598ee 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java @@ -1,7 +1,7 @@ package com.tencent.supersonic.headless.api.pojo; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import lombok.Data; @Data @@ -9,6 +9,6 @@ public class SingleItemQueryResult { private Item item; - private QueryResultWithSchemaResp result; + private SemanticQueryResp result; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryMultiStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryMultiStructReq.java index f8b8bf376..bb7724998 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryMultiStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryMultiStructReq.java @@ -1,15 +1,17 @@ package com.tencent.supersonic.headless.api.request; import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.headless.api.pojo.Cache; +import java.util.ArrayList; import java.util.List; import lombok.Data; import lombok.ToString; import org.apache.commons.codec.digest.DigestUtils; +import org.springframework.util.CollectionUtils; @Data @ToString -public class QueryMultiStructReq { - +public class QueryMultiStructReq extends SemanticQueryReq { List queryStructReqs; @@ -21,4 +23,18 @@ public class QueryMultiStructReq { return DigestUtils.md5Hex(this.toCustomizedString()); } + public List getModelIds() { + if (CollectionUtils.isEmpty(this.getQueryStructReqs())) { + return new ArrayList<>(); + } + return this.getQueryStructReqs().get(0).getModelIds(); + } + + public Cache getCacheInfo() { + if (CollectionUtils.isEmpty(this.getQueryStructReqs())) { + return getCacheInfo(); + } + return this.getQueryStructReqs().get(0).getCacheInfo(); + } + } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java deleted file mode 100644 index 51d2c9c45..000000000 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.tencent.supersonic.headless.api.request; - -import com.google.common.collect.Lists; -import lombok.Data; -import lombok.ToString; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -@Data -@ToString -public class QueryS2SQLReq { - - private Set modelIds; - - private String sql; - - private Map variables; - - public void setModelId(Long modelId) { - modelIds = new HashSet<>(); - modelIds.add(modelId); - } - - public List getModelIds() { - return Lists.newArrayList(modelIds); - } - - public String getModelIdStr() { - return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); - } - -} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QuerySqlReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QuerySqlReq.java new file mode 100644 index 000000000..f1af2ce4e --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QuerySqlReq.java @@ -0,0 +1,27 @@ +package com.tencent.supersonic.headless.api.request; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class QuerySqlReq extends SemanticQueryReq { + + private String sql; + + @Override + public String toCustomizedString() { + StringBuilder stringBuilder = new StringBuilder("{"); + stringBuilder.append("\"modelId\":") + .append(modelIds); + stringBuilder.append(",\"params\":") + .append(params); + stringBuilder.append(",\"cacheInfo\":") + .append(cacheInfo); + stringBuilder.append(",\"sql\":") + .append(sql); + stringBuilder.append('}'); + return stringBuilder.toString(); + } + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java index 039a20334..bf971de12 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java @@ -6,14 +6,16 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; -import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.headless.api.pojo.Cache; -import com.tencent.supersonic.headless.api.pojo.Param; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; @@ -37,20 +39,10 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - @Data @Slf4j -public class QueryStructReq { - - private Set modelIds; +public class QueryStructReq extends SemanticQueryReq { private String modelName; private List groups = new ArrayList<>(); @@ -58,11 +50,10 @@ public class QueryStructReq { private List orders = new ArrayList<>(); private List dimensionFilters = new ArrayList<>(); private List metricFilters = new ArrayList<>(); - private List params = new ArrayList<>(); private DateConf dateInfo; private Long limit = 2000L; private QueryType queryType = QueryType.ID; - private Cache cacheInfo; + /** * Later deleted for compatibility only @@ -73,23 +64,6 @@ public class QueryStructReq { */ private String correctS2SQL; - public void setModelId(Long modelId) { - modelIds = new HashSet<>(); - modelIds.add(modelId); - } - - public List getModelIds() { - return Lists.newArrayList(modelIds); - } - - public String getModelIdStr() { - return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); - } - - public Set getModelIdSet() { - return modelIds; - } - public List getGroups() { if (!CollectionUtils.isEmpty(this.groups)) { this.groups = groups.stream().filter(group -> !Strings.isEmpty(group)).collect(Collectors.toList()); @@ -117,13 +91,6 @@ public class QueryStructReq { return orders; } - public List getParams() { - if (params == null) { - return Lists.newArrayList(); - } - return params; - } - public String toCustomizedString() { StringBuilder stringBuilder = new StringBuilder("{"); stringBuilder.append("\"modelId\":") @@ -187,7 +154,7 @@ public class QueryStructReq { * @param queryStructReq * @return */ - public QueryS2SQLReq convert(QueryStructReq queryStructReq) { + public QuerySqlReq convert(QueryStructReq queryStructReq) { String sql = null; try { sql = buildSql(queryStructReq); @@ -195,10 +162,10 @@ public class QueryStructReq { log.error("buildSql error", e); } - QueryS2SQLReq result = new QueryS2SQLReq(); + QuerySqlReq result = new QuerySqlReq(); result.setSql(sql); result.setModelIds(queryStructReq.getModelIdSet()); - result.setVariables(new HashMap<>()); + result.setParams(new ArrayList<>()); return result; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SemanticQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SemanticQueryReq.java new file mode 100644 index 000000000..85c2a1652 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SemanticQueryReq.java @@ -0,0 +1,48 @@ +package com.tencent.supersonic.headless.api.request; + +import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.pojo.Cache; +import com.tencent.supersonic.headless.api.pojo.Param; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; + + +@Data +@Slf4j +public abstract class SemanticQueryReq { + + protected Set modelIds; + protected List params = new ArrayList<>(); + + protected Cache cacheInfo = new Cache(); + + public void setModelId(Long modelId) { + modelIds = new HashSet<>(); + modelIds.add(modelId); + } + + public String generateCommandMd5() { + return DigestUtils.md5Hex(this.toCustomizedString()); + } + + public abstract String toCustomizedString(); + + public List getModelIds() { + return Lists.newArrayList(modelIds); + } + + public String getModelIdStr() { + return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); + } + + public Set getModelIdSet() { + return modelIds; + } + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/QueryResultWithSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/SemanticQueryResp.java similarity index 92% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/response/QueryResultWithSchemaResp.java rename to headless/api/src/main/java/com/tencent/supersonic/headless/api/response/SemanticQueryResp.java index 7c2b20f7c..985221f47 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/QueryResultWithSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/SemanticQueryResp.java @@ -14,7 +14,7 @@ import java.util.stream.Collectors; @Data @ToString -public class QueryResultWithSchemaResp extends QueryResult> { +public class SemanticQueryResp extends QueryResult> { List columns = Lists.newArrayList(); String sql; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index ccc3cd3c1..1d99e851d 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.core.executor; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlUtils; @@ -24,7 +24,7 @@ public class JdbcExecutor implements QueryExecutor { } @Override - public QueryResultWithSchemaResp execute(QueryStatement queryStatement) { + public SemanticQueryResp execute(QueryStatement queryStatement) { if (Strings.isEmpty(queryStatement.getSourceId())) { log.warn("data base id is empty"); return null; @@ -32,9 +32,10 @@ public class JdbcExecutor implements QueryExecutor { log.info("query SQL: {}", queryStatement.getSql()); Database database = queryStatement.getSemanticModel().getDatabase(); log.info("database info:{}", database); - QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); + SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); SqlUtils sqlUtils = this.sqlUtils.init(database); sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns); + queryResultWithColumns.setSql(queryStatement.getSql()); return queryResultWithColumns; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java index a5a8c63b8..452f2eec2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java @@ -1,6 +1,6 @@ package com.tencent.supersonic.headless.core.executor; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; /** @@ -10,5 +10,5 @@ public interface QueryExecutor { boolean accept(QueryStatement queryStatement); - QueryResultWithSchemaResp execute(QueryStatement queryStatement); + SemanticQueryResp execute(QueryStatement queryStatement); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java index 9ba1041d7..6af298bdf 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.core.parser; +import com.google.common.base.Strings; import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.headless.api.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.MetricTable; @@ -26,7 +27,7 @@ import org.springframework.util.CollectionUtils; @Primary public class QueryParser { - public QueryStatement logicSql(QueryStatement queryStatement) throws Exception { + public QueryStatement parse(QueryStatement queryStatement) throws Exception { QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); if (Objects.isNull(queryStatement.getParseSqlReq())) { queryStatement.setParseSqlReq(new ParseSqlReq()); @@ -44,12 +45,16 @@ public class QueryParser { log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(), queryStatement.getMetricReq()); if (!queryStatement.getParseSqlReq().getSql().isEmpty()) { - return parser(queryStatement.getParseSqlReq(), queryStatement); + queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement); + } else { + queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery()); + queryStatement = parser(queryStatement); } - - queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery()); - return parser(queryStatement); - + if (Strings.isNullOrEmpty(queryStatement.getSql()) + || Strings.isNullOrEmpty(queryStatement.getSourceId())) { + throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg()); + } + return queryStatement; } public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQuery.java similarity index 96% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQuery.java index 5a34fe137..0877100f3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQuery.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.optimizer; +package com.tencent.supersonic.headless.core.planner; import com.google.common.base.Strings; import com.tencent.supersonic.headless.api.request.QueryStructReq; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryOptimizer.java similarity index 85% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryOptimizer.java index 49a10c962..cf5b37c81 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryOptimizer.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.optimizer; +package com.tencent.supersonic.headless.core.planner; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.core.pojo.QueryStatement; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryPlanner.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryPlanner.java new file mode 100644 index 000000000..1808cb6ad --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryPlanner.java @@ -0,0 +1,32 @@ +package com.tencent.supersonic.headless.core.planner; + +import com.tencent.supersonic.headless.core.executor.QueryExecutor; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.utils.ComponentFactory; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class QueryPlanner { + + public QueryExecutor plan(QueryStatement queryStatement) { + optimizer(queryStatement); + return route(queryStatement); + } + + public void optimizer(QueryStatement queryStatement) { + for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { + queryOptimizer.rewrite(queryStatement.getQueryStructReq(), queryStatement); + } + } + + public QueryExecutor route(QueryStatement queryStatement) { + for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) { + if (queryExecutor.accept(queryStatement)) { + return queryExecutor; + } + } + return null; + } +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java index 3768030c1..66310bca1 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java @@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.core.utils; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.core.executor.JdbcExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor; -import com.tencent.supersonic.headless.core.optimizer.DetailQuery; -import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; +import com.tencent.supersonic.headless.core.planner.DetailQuery; +import com.tencent.supersonic.headless.core.planner.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.HeadlessConverter; import com.tencent.supersonic.headless.core.parser.SqlParser; import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java index 448c88453..9a5312a99 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java @@ -5,7 +5,7 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.enums.DataType; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; import java.rmi.ServerException; @@ -89,7 +89,7 @@ public class SqlUtils { } } - public void execute(String sql, QueryResultWithSchemaResp queryResultWithColumns) { + public void execute(String sql, SemanticQueryResp queryResultWithColumns) { getResult(sql, queryResultWithColumns, jdbcTemplate()); } @@ -110,11 +110,11 @@ public class SqlUtils { return jdbcTemplate; } - public void queryInternal(String sql, QueryResultWithSchemaResp queryResultWithColumns) { + public void queryInternal(String sql, SemanticQueryResp queryResultWithColumns) { getResult(sql, queryResultWithColumns, jdbcTemplate()); } - private QueryResultWithSchemaResp getResult(String sql, QueryResultWithSchemaResp queryResultWithColumns, + private SemanticQueryResp getResult(String sql, SemanticQueryResp queryResultWithColumns, JdbcTemplate jdbcTemplate) { jdbcTemplate.query(sql, rs -> { if (null == rs) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java index b5f8dcb9e..db9ae16ea 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java @@ -19,7 +19,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; @@ -142,14 +142,14 @@ public class AuthCheckBaseAspect { return false; } - public QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns, + public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns, List modelIds, AuthorizedResourceResp authResource) { addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet()); return resultWithColumns; } - public QueryResultWithSchemaResp desensitizationData(QueryResultWithSchemaResp raw, Set need2Apply) { + public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set need2Apply) { log.debug("start desensitizationData logic"); if (CollectionUtils.isEmpty(need2Apply)) { log.info("user has all sensitiveRes"); @@ -171,7 +171,7 @@ public class AuthCheckBaseAspect { return raw; } - QueryResultWithSchemaResp queryResultWithColumns = raw; + SemanticQueryResp queryResultWithColumns = raw; try { queryResultWithColumns = deepCopyResult(raw); } catch (Exception e) { @@ -216,8 +216,8 @@ public class AuthCheckBaseAspect { } } - private QueryResultWithSchemaResp deepCopyResult(QueryResultWithSchemaResp raw) throws Exception { - QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); + private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception { + SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); BeanUtils.copyProperties(raw, queryResultWithColumns); List columns = new ArrayList<>(); @@ -241,7 +241,7 @@ public class AuthCheckBaseAspect { return queryResultWithColumns; } - public void addPromptInfoInfo(List modelIds, QueryResultWithSchemaResp queryResultWithColumns, + public void addPromptInfoInfo(List modelIds, SemanticQueryResp queryResultWithColumns, AuthorizedResourceResp authorizedResource, Set need2Apply) { List filters = authorizedResource.getFilters(); if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index 2723f7afc..ed9c29a9b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -8,11 +8,11 @@ import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.common.util.jsqlparser.FieldExpression; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.DimValueMap; import com.tencent.supersonic.headless.api.response.DimensionResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import lombok.extern.slf4j.Slf4j; @@ -51,13 +51,13 @@ public class DimValueAspect { public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable { if (!dimensionValueMapSqlEnable) { log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap"); - QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); + SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); return queryResultWithColumns; } Object[] args = joinPoint.getArgs(); - QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0]; - MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelIds())); - String sql = queryS2SQLReq.getSql(); + QuerySqlReq querySQLReq = (QuerySqlReq) args[0]; + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySQLReq.getModelIds())); + String sql = querySQLReq.getSql(); log.info("correctorSql before replacing:{}", sql); // if dimensionvalue is alias,consider the true dimensionvalue. List fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql); @@ -88,10 +88,10 @@ public class DimValueAspect { log.info("filedNameToValueMap:{}", filedNameToValueMap); sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap); log.info("correctorSql after replacing:{}", sql); - queryS2SQLReq.setSql(sql); + querySQLReq.setSql(sql); Map> techNameToBizName = getTechNameToBizName(dimensions); - QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); + SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); if (Objects.nonNull(queryResultWithColumns)) { rewriteDimValue(queryResultWithColumns, techNameToBizName); } @@ -140,7 +140,7 @@ public class DimValueAspect { if (!dimensionValueMapEnable) { log.debug("dimensionValueMapEnable is false, skip dimensionValueMap"); - QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); + SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); return queryResultWithColumns; } @@ -153,21 +153,21 @@ public class DimValueAspect { rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair); - QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); - if (Objects.nonNull(queryResultWithColumns)) { - rewriteDimValue(queryResultWithColumns, dimAndTechNameAndBizNamePair); + SemanticQueryResp semanticQueryResp = (SemanticQueryResp) joinPoint.proceed(); + if (Objects.nonNull(semanticQueryResp)) { + rewriteDimValue(semanticQueryResp, dimAndTechNameAndBizNamePair); } - return queryResultWithColumns; + return semanticQueryResp; } - private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns, + private void rewriteDimValue(SemanticQueryResp semanticQueryResp, Map> dimAndTechNameAndBizNamePair) { - if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) { + if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) { return; } log.debug("start rewriteDimValue for resultList"); - for (Map line : queryResultWithColumns.getResultList()) { + for (Map line : semanticQueryResp.getResultList()) { for (String bizName : line.keySet()) { if (dimAndTechNameAndBizNamePair.containsKey(bizName) && Objects.nonNull(line.get(bizName))) { String techName = line.get(bizName).toString(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java index 371251f3e..b288e22a0 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java @@ -1,5 +1,7 @@ package com.tencent.supersonic.headless.server.aspect; +import static com.tencent.supersonic.common.pojo.Constants.MINUS; + import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -7,15 +9,22 @@ import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResource import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.server.utils.QueryStructUtils; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.pojo.ModelFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.ModelService; +import com.tencent.supersonic.headless.server.utils.QueryStructUtils; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Expression; @@ -31,16 +40,6 @@ import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.StringJoiner; -import java.util.stream.Collectors; - -import static com.tencent.supersonic.common.pojo.Constants.MINUS; - @Component @Aspect @Order(1) @@ -64,7 +63,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { log.info("s2SQL permission check!"); Object[] objects = joinPoint.getArgs(); - QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) objects[0]; + QuerySqlReq querySQLReq = (QuerySqlReq) objects[0]; User user = (User) objects[1]; if (!permissionDataEnable) { log.info("not to check s2SQL permission!"); @@ -73,7 +72,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) { throw new RuntimeException("please provide user information"); } - List modelIds = queryS2SQLReq.getModelIds(); + List modelIds = querySQLReq.getModelIds(); //1. determine whether admin of the model if (doModelAdmin(user, modelIds)) { @@ -83,7 +82,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { // 2. determine whether the subject field is visible doModelVisible(user, modelIds); // 3. fetch data permission meta information - Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user); + Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySQLReq, user); log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege); Set sensitiveResByModel = getHighSensitiveColsByModelId(modelIds); @@ -97,13 +96,13 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { Set resAuthSet = getAuthResNameSet(authorizedResource, modelIds); // 4.if sensitive fields without permission are involved in filter, thrown an exception - doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq); + doFilterCheckLogic(querySQLReq, resAuthSet, sensitiveResReq); // 5.row permission pre-filter - doRowPermission(queryS2SQLReq, authorizedResource); + doRowPermission(querySQLReq, authorizedResource); // 6.proceed - QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed(); + SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty @@ -115,14 +114,14 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { Set need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req)) .collect(Collectors.toSet()); log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet); - QueryResultWithSchemaResp queryResultAfterDesensitization = + SemanticQueryResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns, need2Apply); addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply); return queryResultAfterDesensitization; } - private void doRowPermission(QueryS2SQLReq queryS2SQLReq, AuthorizedResourceResp authorizedResource) { + private void doRowPermission(QuerySqlReq querySQLReq, AuthorizedResourceResp authorizedResource) { log.debug("start doRowPermission logic"); StringJoiner joiner = new StringJoiner(" OR "); List dimensionFilters = new ArrayList<>(); @@ -144,10 +143,10 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { try { Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); if (StringUtils.isNotEmpty(joiner.toString())) { - String sql = SqlParserAddHelper.addWhere(queryS2SQLReq.getSql(), expression); - log.info("before doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql()); - queryS2SQLReq.setSql(sql); - log.info("after doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql()); + String sql = SqlParserAddHelper.addWhere(querySQLReq.getSql(), expression); + log.info("before doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql()); + querySQLReq.setSql(sql); + log.info("after doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql()); } } catch (JSQLParserException jsqlParserException) { log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); @@ -155,14 +154,14 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect { } - private void doFilterCheckLogic(QueryS2SQLReq queryS2SQLReq, Set resAuthName, + private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set resAuthName, Set sensitiveResReq) { - Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2SQLReq); + Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq); Set need2Apply = resFilterSet.stream() .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); Set nameCnSet = new HashSet<>(); - List modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds()); + List modelIds = Lists.newArrayList(querySQLReq.getModelIds()); ModelFilter modelFilter = new ModelFilter(); modelFilter.setModelIds(modelIds); List modelInfos = modelService.getModelList(modelFilter); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java index 59490df9b..86eae6bf1 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java @@ -10,7 +10,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; @@ -100,7 +100,7 @@ public class StructDataAspect extends AuthCheckBaseAspect { doRowPermission(queryStructReq, authorizedResource); // 6.proceed - QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) point.proceed(); + SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed(); if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) { // if sensitiveRes is empty @@ -111,7 +111,7 @@ public class StructDataAspect extends AuthCheckBaseAspect { // 6.if the column has no permission, hit * Set need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req)) .collect(Collectors.toSet()); - QueryResultWithSchemaResp queryResultAfterDesensitization = + SemanticQueryResp queryResultAfterDesensitization = desensitizationData(queryResultWithColumns, need2Apply); addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply); diff --git a/common/src/main/java/com/tencent/supersonic/common/util/cache/CacheUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CacheManager.java similarity index 66% rename from common/src/main/java/com/tencent/supersonic/common/util/cache/CacheUtils.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CacheManager.java index 598089d15..852860e13 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/cache/CacheUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CacheManager.java @@ -1,7 +1,7 @@ -package com.tencent.supersonic.common.util.cache; +package com.tencent.supersonic.headless.server.cache; -public interface CacheUtils { +public interface CacheManager { Boolean put(String key, Object value); diff --git a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CaffeineCacheManager.java similarity index 88% rename from common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CaffeineCacheManager.java index ebc41b60a..43e9e8de5 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CaffeineCacheManager.java @@ -1,8 +1,9 @@ -package com.tencent.supersonic.common.util.cache; +package com.tencent.supersonic.headless.server.cache; import com.github.benmanes.caffeine.cache.Cache; import com.google.common.base.Joiner; +import com.tencent.supersonic.headless.server.config.CacheCommonConfig; import lombok.extern.slf4j.Slf4j; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.factory.annotation.Autowired; @@ -11,8 +12,7 @@ import org.springframework.stereotype.Component; @Component @Slf4j -public class CaffeineCacheImpl implements CacheUtils { - +public class CaffeineCacheManager implements CacheManager { @Autowired private CacheCommonConfig cacheCommonConfig; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/QueryCache.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/QueryCache.java new file mode 100644 index 000000000..de213f75b --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/QueryCache.java @@ -0,0 +1,78 @@ +package com.tencent.supersonic.headless.server.cache; + + +import com.tencent.supersonic.headless.api.pojo.Cache; +import com.tencent.supersonic.headless.api.request.SemanticQueryReq; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class QueryCache { + + @Value("${query.cache.enable:true}") + private Boolean cacheEnable; + @Autowired + private CacheManager cacheManager; + + public Object query(SemanticQueryReq semanticQueryReq) { + String cacheKey = getCacheKey(semanticQueryReq); + handleGlobalCacheDisable(semanticQueryReq); + boolean isCache = isCache(semanticQueryReq); + if (isCache) { + Object result = cacheManager.get(cacheKey); + log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq); + return result; + } + return null; + } + + public Boolean put(SemanticQueryReq semanticQueryReq, Object value) { + if (cacheEnable && Objects.nonNull(value)) { + String key = getCacheKey(semanticQueryReq); + CompletableFuture.supplyAsync(() -> cacheManager.put(key, value)) + .exceptionally(exception -> { + log.warn("exception:", exception); + return null; + }); + log.info("add record to cache, key:{}", key); + return true; + } + return false; + } + + public String getCacheKey(SemanticQueryReq semanticQueryReq) { + String commandMd5 = semanticQueryReq.generateCommandMd5(); + String keyByModelIds = getKeyByModelIds(semanticQueryReq.getModelIds()); + return cacheManager.generateCacheKey(keyByModelIds, commandMd5); + } + + private void handleGlobalCacheDisable(SemanticQueryReq semanticQueryReq) { + if (!cacheEnable) { + Cache cacheInfo = new Cache(); + cacheInfo.setCache(false); + semanticQueryReq.setCacheInfo(cacheInfo); + } + } + + private String getKeyByModelIds(List modelIds) { + return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList())); + } + + private boolean isCache(SemanticQueryReq semanticQueryReq) { + if (!cacheEnable) { + return false; + } + if (semanticQueryReq.getCacheInfo() != null) { + return semanticQueryReq.getCacheInfo().getCache(); + } + return false; + } + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/cache/CacheCommonConfig.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CacheCommonConfig.java similarity index 90% rename from common/src/main/java/com/tencent/supersonic/common/util/cache/CacheCommonConfig.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CacheCommonConfig.java index 7f3d8f7eb..c6f3c1f0c 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/cache/CacheCommonConfig.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CacheCommonConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.common.util.cache; +package com.tencent.supersonic.headless.server.config; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CaffeineCacheConfig.java similarity index 85% rename from common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CaffeineCacheConfig.java index 4add68aa9..7d5640296 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CaffeineCacheConfig.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.common.util.cache; +package com.tencent.supersonic.headless.server.config; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -24,9 +24,7 @@ public class CaffeineCacheConfig { public Cache caffeineCache() { return Caffeine.newBuilder() .expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES) - // 初始的缓存空间大小 .initialCapacity(caffeineInitialCapacity) - // 缓存的最大条数 .maximumSize(caffeineMaximumSize) .build(); } @@ -35,9 +33,7 @@ public class CaffeineCacheConfig { public Cache searchCaffeineCache() { return Caffeine.newBuilder() .expireAfterWrite(10000, TimeUnit.MINUTES) - // 初始的缓存空间大小 .initialCapacity(caffeineInitialCapacity) - // 缓存的最大条数 .maximumSize(caffeineMaximumSize) .build(); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java similarity index 98% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 491e36f2f..41ae1d57e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -56,13 +56,13 @@ import org.springframework.util.CollectionUtils; @Slf4j @Service -public class HeadlessSchemaManager { +public class SemanticSchemaManager { @Autowired private LoadingCache loadingCache; private final Catalog catalog; - public HeadlessSchemaManager(Catalog catalog) { + public SemanticSchemaManager(Catalog catalog) { this.catalog = catalog; } @@ -87,7 +87,7 @@ public class HeadlessSchemaManager { semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName)); } if (!dataModelYamlTpls.isEmpty()) { - Map dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource) + Map dataSourceMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource) .collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1)); semanticModel.setDatasourceMap(dataSourceMap); } @@ -104,7 +104,6 @@ public class HeadlessSchemaManager { return semanticModel; } - //private Map semanticSchemaMap = new HashMap<>(); public SemanticModel get(String rootPath) throws Exception { rootPath = formatKey(rootPath); SemanticModel schema = loadingCache.get(rootPath); @@ -383,7 +382,7 @@ public class HeadlessSchemaManager { @Override public SemanticModel load(String key) { log.info("load SemanticSchema [{}]", key); - return HeadlessSchemaManager.this.reload(key); + return SemanticSchemaManager.this.reload(key); } } ); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java index 6b978bfb1..b8e5ef6bb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.request.DatabaseReq; import com.tencent.supersonic.headless.api.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.DatabaseParameter; import com.tencent.supersonic.headless.server.service.DatabaseService; import java.util.Map; @@ -67,7 +67,7 @@ public class DatabaseController { } @PostMapping("/executeSql") - public QueryResultWithSchemaResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq, + public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -75,18 +75,18 @@ public class DatabaseController { } @RequestMapping("/getDbNames/{id}") - public QueryResultWithSchemaResp getDbNames(@PathVariable("id") Long id) { + public SemanticQueryResp getDbNames(@PathVariable("id") Long id) { return databaseService.getDbNames(id); } @RequestMapping("/getTables/{id}/{db}") - public QueryResultWithSchemaResp getTables(@PathVariable("id") Long id, + public SemanticQueryResp getTables(@PathVariable("id") Long id, @PathVariable("db") String db) { return databaseService.getTables(id, db); } @RequestMapping("/getColumns/{id}/{db}/{table}") - public QueryResultWithSchemaResp getColumns(@PathVariable("id") Long id, + public SemanticQueryResp getColumns(@PathVariable("id") Long id, @PathVariable("db") String db, @PathVariable("table") String table) { return databaseService.getColumns(id, db, table); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index 1855c049a..c3baed954 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -12,16 +12,16 @@ import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.ExplainResp; import com.tencent.supersonic.headless.api.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.response.ItemUseResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.response.SqlParserResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.server.service.DownloadService; -import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; +import com.tencent.supersonic.headless.server.service.SemantciQueryEngine; import com.tencent.supersonic.headless.server.service.QueryService; import java.util.HashSet; import java.util.List; @@ -45,19 +45,17 @@ public class QueryController { @Autowired private QueryService queryService; @Autowired - private HeadlessQueryEngine headlessQueryEngine; + private SemantciQueryEngine semantciQueryEngine; @Autowired private DownloadService downloadService; @PostMapping("/sql") - public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq, + public Object queryBySql(@RequestBody QuerySqlReq querySQLReq, HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - Object queryBySql = queryService.queryBySql(queryS2SQLReq, user); - log.info("queryBySql:{}", queryBySql); - return queryBySql; + return queryService.queryBySql(querySQLReq, user); } @PostMapping("/struct") @@ -91,7 +89,7 @@ public class QueryController { } @PostMapping("/queryStatement") - public Object queryStatement(@RequestBody QueryStatement queryStatement) throws Exception { + public SemanticQueryResp queryStatement(@RequestBody QueryStatement queryStatement) throws Exception { return queryService.queryByQueryStatement(queryStatement); } @@ -101,7 +99,7 @@ public class QueryController { Set models = new HashSet<>(); models.add(Long.valueOf(parseSqlReq.getRootPath())); queryStructCmd.setModelIds(models); - QueryStatement queryStatement = headlessQueryEngine.physicalSql(queryStructCmd, parseSqlReq); + QueryStatement queryStatement = semantciQueryEngine.physicalSql(queryStructCmd, parseSqlReq); SqlParserResp sqlParserResp = new SqlParserResp(); BeanUtils.copyProperties(queryStatement, sqlParserResp); return sqlParserResp; @@ -130,7 +128,7 @@ public class QueryController { } @PostMapping("/queryDimValue") - public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq, + public SemanticQueryResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -147,9 +145,9 @@ public class QueryController { QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum(); if (QueryType.SQL.equals(queryTypeEnum)) { - QueryS2SQLReq queryS2SQLReq = JsonUtil.toObject(queryReqJson, QueryS2SQLReq.class); - ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() - .queryReq(queryS2SQLReq) + QuerySqlReq querySQLReq = JsonUtil.toObject(queryReqJson, QuerySqlReq.class); + ExplainSqlReq explainSqlReqNew = ExplainSqlReq.builder() + .queryReq(querySQLReq) .queryTypeEnum(queryTypeEnum).build(); return queryService.explain(explainSqlReqNew, user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java index bf6bee06d..57237e30f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.request.DatabaseReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.DatabaseParameter; import java.util.List; import java.util.Map; @@ -11,9 +11,9 @@ import java.util.Map; public interface DatabaseService { - QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp); + SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp); - QueryResultWithSchemaResp executeSql(String sql, Long id, User user); + SemanticQueryResp executeSql(String sql, Long id, User user); Map> getDatabaseParameters(); @@ -27,9 +27,9 @@ public interface DatabaseService { DatabaseResp getDatabase(Long id); - QueryResultWithSchemaResp getDbNames(Long id); + SemanticQueryResp getDbNames(Long id); - QueryResultWithSchemaResp getTables(Long id, String db); + SemanticQueryResp getTables(Long id, String db); - QueryResultWithSchemaResp getColumns(Long id, String db, String table); + SemanticQueryResp getColumns(Long id, String db, String table); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java index 257bc4cc4..8e46a7432 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java @@ -6,12 +6,12 @@ import com.tencent.supersonic.headless.api.request.ItemUseReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.ExplainResp; import com.tencent.supersonic.headless.api.response.ItemQueryResultResp; import com.tencent.supersonic.headless.api.response.ItemUseResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck; @@ -20,18 +20,17 @@ import java.util.List; public interface QueryService { - Object queryBySql(QueryS2SQLReq querySqlCmd, User user) throws Exception; + Object queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception; - QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; + SemanticQueryResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception; - QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) - throws Exception; + SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) throws Exception; - QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception; + SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception; - QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); + SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); - Object queryByQueryStatement(QueryStatement queryStatement); + SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement); List getStatInfo(ItemUseReq itemUseCommend); @@ -39,6 +38,6 @@ public interface QueryService { @ApiHeaderCheck ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq, - HttpServletRequest request) throws Exception; + HttpServletRequest request) throws Exception; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemantciQueryEngine.java similarity index 79% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemantciQueryEngine.java index 5dc3e16b8..a4baf0115 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemantciQueryEngine.java @@ -3,17 +3,17 @@ package com.tencent.supersonic.headless.server.service; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.executor.QueryExecutor; -public interface HeadlessQueryEngine { +public interface SemantciQueryEngine { QueryStatement plan(QueryStatement queryStatement) throws Exception; QueryExecutor route(QueryStatement queryStatement); - QueryResultWithSchemaResp execute(QueryStatement queryStatement); + SemanticQueryResp execute(QueryStatement queryStatement); QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index 31488c6a9..81e48422f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -4,7 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.request.DatabaseReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.core.pojo.Database; @@ -112,10 +112,10 @@ public class DatabaseServiceImpl implements DatabaseService { } @Override - public QueryResultWithSchemaResp executeSql(String sql, Long id, User user) { + public SemanticQueryResp executeSql(String sql, Long id, User user) { DatabaseResp databaseResp = getDatabase(id); if (databaseResp == null) { - return new QueryResultWithSchemaResp(); + return new SemanticQueryResp(); } List admins = databaseResp.getAdmins(); List viewers = databaseResp.getViewers(); @@ -132,7 +132,7 @@ public class DatabaseServiceImpl implements DatabaseService { } @Override - public QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp) { + public SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp) { return queryWithColumns(sql, DatabaseConverter.convert(databaseResp)); } @@ -143,8 +143,8 @@ public class DatabaseServiceImpl implements DatabaseService { LinkedHashMap::putAll); } - private QueryResultWithSchemaResp queryWithColumns(String sql, Database database) { - QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp(); + private SemanticQueryResp queryWithColumns(String sql, Database database) { + SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); SqlUtils sqlUtils = this.sqlUtils.init(database); log.info("query SQL: {}", sql); sqlUtils.queryInternal(sql, queryResultWithColumns); @@ -156,7 +156,7 @@ public class DatabaseServiceImpl implements DatabaseService { } @Override - public QueryResultWithSchemaResp getDbNames(Long id) { + public SemanticQueryResp getDbNames(Long id) { DatabaseResp databaseResp = getDatabase(id); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl(); @@ -164,7 +164,7 @@ public class DatabaseServiceImpl implements DatabaseService { } @Override - public QueryResultWithSchemaResp getTables(Long id, String db) { + public SemanticQueryResp getTables(Long id, String db) { DatabaseResp databaseResp = getDatabase(id); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl(); @@ -173,7 +173,7 @@ public class DatabaseServiceImpl implements DatabaseService { } @Override - public QueryResultWithSchemaResp getColumns(Long id, String db, String table) { + public SemanticQueryResp getColumns(Long id, String db, String table) { DatabaseResp databaseResp = getDatabase(id); DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType()); String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index cd9e58035..60f236864 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -23,7 +23,7 @@ import com.tencent.supersonic.headless.api.request.PageDimensionReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.ModelResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO; import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; @@ -266,8 +266,8 @@ public class DimensionServiceImpl implements DimensionService { String sql = "select ai_talk." + dimensionReq.getBizName() + " from (" + sqlQuery + ") as ai_talk group by ai_talk." + dimensionReq.getBizName(); - QueryResultWithSchemaResp queryResultWithSchemaResp = databaseService.executeSql(sql, database); - List> resultList = queryResultWithSchemaResp.getResultList(); + SemanticQueryResp semanticQueryResp = databaseService.executeSql(sql, database); + List> resultList = semanticQueryResp.getResultList(); List valueList = new ArrayList<>(); for (Map stringObjectMap : resultList) { String value = (String) stringObjectMap.get(dimensionReq.getBizName()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java index 76c082efb..85f8b42c8 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java @@ -21,7 +21,7 @@ import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.utils.DataTransformUtils; import com.tencent.supersonic.headless.server.pojo.DataDownload; import com.tencent.supersonic.headless.server.service.DownloadService; @@ -72,7 +72,7 @@ public class DownloadServiceImpl implements DownloadService { String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT)); File file = FileUtils.createTmpFile(fileName); try { - QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user); + SemanticQueryResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData()); } catch (RuntimeException e) { @@ -112,7 +112,7 @@ public class DownloadServiceImpl implements DownloadService { for (MetricSchemaResp metric : metrics) { try { DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq); - QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user); + SemanticQueryResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user); DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq); WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount) .head(dataDownload.getHeaders()).build(); @@ -140,9 +140,9 @@ public class DownloadServiceImpl implements DownloadService { return data; } - private List> buildHeader(QueryResultWithSchemaResp queryResultWithSchemaResp) { + private List> buildHeader(SemanticQueryResp semanticQueryResp) { List> header = Lists.newArrayList(); - for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { + for (QueryColumn column : semanticQueryResp.getColumns()) { header.add(Lists.newArrayList(column.getName())); } return header; @@ -163,11 +163,11 @@ public class DownloadServiceImpl implements DownloadService { return headers; } - private List> buildData(QueryResultWithSchemaResp queryResultWithSchemaResp) { + private List> buildData(SemanticQueryResp semanticQueryResp) { List> data = new ArrayList<>(); - for (Map row : queryResultWithSchemaResp.getResultList()) { + for (Map row : semanticQueryResp.getResultList()) { List rowData = new ArrayList<>(); - for (QueryColumn column : queryResultWithSchemaResp.getColumns()) { + for (QueryColumn column : semanticQueryResp.getColumns()) { rowData.add(String.valueOf(row.get(column.getNameEn()))); } data.add(rowData); @@ -198,7 +198,7 @@ public class DownloadServiceImpl implements DownloadService { return data; } - private DataDownload buildDataDownload(QueryResultWithSchemaResp queryResult, DownloadStructReq downloadStructReq) { + private DataDownload buildDataDownload(SemanticQueryResp queryResult, DownloadStructReq downloadStructReq) { List metricColumns = queryResult.getMetricColumns(); List dimensionColumns = queryResult.getDimensionColumns(); if (downloadStructReq.isTransform() && !CollectionUtils.isEmpty(metricColumns)) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index d77dd67ec..01901dbde 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.server.service.impl; +import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -13,20 +14,17 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.JsonUtil; -import com.tencent.supersonic.common.util.cache.CacheUtils; import com.tencent.supersonic.headless.api.enums.QueryType; -import com.tencent.supersonic.headless.api.pojo.Cache; import com.tencent.supersonic.headless.api.pojo.Dim; import com.tencent.supersonic.headless.api.pojo.Item; import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult; import com.tencent.supersonic.headless.api.request.ExplainSqlReq; import com.tencent.supersonic.headless.api.request.ItemUseReq; -import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; import com.tencent.supersonic.headless.api.request.QueryDimValueReq; import com.tencent.supersonic.headless.api.request.QueryItemReq; import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.AppDetailResp; import com.tencent.supersonic.headless.api.response.DimensionResp; @@ -36,28 +34,27 @@ import com.tencent.supersonic.headless.api.response.ItemUseResp; import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.executor.QueryExecutor; +import com.tencent.supersonic.headless.core.parser.QueryParser; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; +import com.tencent.supersonic.headless.core.planner.QueryPlanner; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission; import com.tencent.supersonic.headless.server.annotation.StructDataPermission; import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect; +import com.tencent.supersonic.headless.server.cache.CacheManager; +import com.tencent.supersonic.headless.server.cache.QueryCache; +import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import com.tencent.supersonic.headless.server.pojo.DimensionFilter; import com.tencent.supersonic.headless.server.service.AppService; import com.tencent.supersonic.headless.server.service.Catalog; -import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; import com.tencent.supersonic.headless.server.service.QueryService; import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.service.SemantciQueryEngine; import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; -import javax.servlet.http.HttpServletRequest; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -65,17 +62,24 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import javax.servlet.http.HttpServletRequest; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; @Service @Slf4j public class QueryServiceImpl implements QueryService { - protected final com.google.common.cache.Cache> itemUseCache = + protected final Cache> itemUseCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build(); private final StatUtils statUtils; - private final CacheUtils cacheUtils; + private final CacheManager cacheManager; private final QueryUtils queryUtils; private final QueryReqConverter queryReqConverter; private final Catalog catalog; @@ -84,88 +88,109 @@ public class QueryServiceImpl implements QueryService { @Value("${query.cache.enable:true}") private Boolean cacheEnable; - private final HeadlessQueryEngine headlessQueryEngine; + private final QueryCache queryCache; + + private final SemantciQueryEngine semantciQueryEngine; + private final SemanticSchemaManager semanticSchemaManager; + + private final QueryParser queryParser; + + private final QueryPlanner queryPlanner; public QueryServiceImpl( StatUtils statUtils, - CacheUtils cacheUtils, + CacheManager cacheManager, QueryUtils queryUtils, QueryReqConverter queryReqConverter, - @Lazy HeadlessQueryEngine headlessQueryEngine, + @Lazy SemantciQueryEngine semantciQueryEngine, Catalog catalog, - AppService appService) { + AppService appService, + QueryCache queryCache, + SemanticSchemaManager semanticSchemaManager, + QueryParser queryParser, + QueryPlanner queryPlanner) { this.statUtils = statUtils; - this.cacheUtils = cacheUtils; + this.cacheManager = cacheManager; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; - this.headlessQueryEngine = headlessQueryEngine; + this.semantciQueryEngine = semantciQueryEngine; this.catalog = catalog; this.appService = appService; + this.queryCache = queryCache; + this.semanticSchemaManager = semanticSchemaManager; + this.queryParser = queryParser; + this.queryPlanner = queryPlanner; } @Override @S2SQLDataPermission @SneakyThrows - public Object queryBySql(QueryS2SQLReq queryS2SQLReq, User user) { - statUtils.initStatInfo(queryS2SQLReq, user); + public Object queryBySql(QuerySqlReq querySQLReq, User user) { + statUtils.initStatInfo(querySQLReq, user); QueryStatement queryStatement = new QueryStatement(); try { - queryStatement = convertToQueryStatement(queryS2SQLReq, user); + queryStatement = convertToQueryStatement(querySQLReq, user); } catch (Exception e) { log.info("convertToQueryStatement has a exception:", e); } log.info("queryStatement:{}", queryStatement); - QueryResultWithSchemaResp results = headlessQueryEngine.execute(queryStatement); + SemanticQueryResp results = semantciQueryEngine.execute(queryStatement); statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); return results; } - public Object queryByQueryStatement(QueryStatement queryStatement) { - return headlessQueryEngine.execute(queryStatement); + public SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement) { + return semantciQueryEngine.execute(queryStatement); } - private QueryStatement convertToQueryStatement(QueryS2SQLReq queryS2SQLReq, User user) throws Exception { + private QueryStatement convertToQueryStatement(QuerySqlReq querySQLReq, User user) throws Exception { ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - filter.setModelIds(queryS2SQLReq.getModelIds()); + filter.setModelIds(querySQLReq.getModelIds()); SchemaService schemaService = ContextUtils.getBean(SchemaService.class); List modelSchemaResps = schemaService.fetchModelSchema(filter, user); - QueryStatement queryStatement = queryReqConverter.convert(queryS2SQLReq, modelSchemaResps); - queryStatement.setModelIds(queryS2SQLReq.getModelIds()); + QueryStatement queryStatement = queryReqConverter.convert(querySQLReq, modelSchemaResps); + queryStatement.setModelIds(querySQLReq.getModelIds()); return queryStatement; } @Override - public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception { - QueryResultWithSchemaResp queryResultWithColumns = null; + public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception { + SemanticQueryResp semanticQueryResp = null; log.info("[queryStructReq:{}]", queryStructReq); try { + //1.initStatInfo statUtils.initStatInfo(queryStructReq, user); - String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructReq.getModelIds()), - queryStructReq.generateCommandMd5()); - handleGlobalCacheDisable(queryStructReq); - boolean isCache = isCache(queryStructReq); - if (isCache) { - queryResultWithColumns = queryByCache(cacheKey, queryStructReq); - if (queryResultWithColumns != null) { - statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); - return queryResultWithColumns; - } + //2.query from cache + Object query = queryCache.query(queryStructReq); + if (Objects.nonNull(query)) { + statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); + return (SemanticQueryResp) query; } StatUtils.get().setUseResultCache(false); - QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryStructReq(queryStructReq); - queryStatement.setIsS2SQL(false); - queryStatement = headlessQueryEngine.plan(queryStatement); - QueryExecutor queryExecutor = headlessQueryEngine.route(queryStatement); + //3 parse + QueryStatement queryStatement = buildQueryStatement(queryStructReq); + queryStatement = queryParser.parse(queryStatement); + + //4 plan + QueryExecutor queryExecutor = queryPlanner.plan(queryStatement); + + //5 execute if (queryExecutor != null) { - queryResultWithColumns = headlessQueryEngine.execute(queryStatement); - if (isCache) { - // if queryResultWithColumns is not null, update cache data - queryUtils.cacheResultLogic(cacheKey, queryResultWithColumns); + semanticQueryResp = queryExecutor.execute(queryStatement); + if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) { + queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds()); } } - statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); - return queryResultWithColumns; + //6 reset cache and set stateInfo + Boolean putCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp); + if (putCacheSuccess) { + // if semanticQueryResp is not null, update cache data + statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq)); + } + if (Objects.nonNull(semanticQueryResp)) { + statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); + } + return semanticQueryResp; } catch (Exception e) { log.warn("exception in queryByStruct, e: ", e); statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR); @@ -173,38 +198,62 @@ public class QueryServiceImpl implements QueryService { } } + private QueryStatement buildQueryStatement(QueryStructReq queryStructReq) throws Exception { + QueryStatement queryStatement = new QueryStatement(); + queryStatement.setQueryStructReq(queryStructReq); + queryStatement.setIsS2SQL(false); + queryStatement.setEnableOptimize(queryUtils.enableOptimize()); + queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); + SemanticModel semanticModel = semanticSchemaManager.get(queryStructReq.getModelIdStr()); + queryStatement.setSemanticModel(semanticModel); + return queryStatement; + } + @Override @StructDataPermission @SneakyThrows - public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) { + public SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) { return queryByStruct(queryStructReq, user); } @Override - public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) + public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) throws Exception { - statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user); - String cacheKey = cacheUtils.generateCacheKey( - getKeyByModelIds(queryMultiStructReq.getQueryStructReqs().get(0).getModelIds()), - queryMultiStructReq.generateCommandMd5()); - boolean isCache = isCache(queryMultiStructReq); - QueryResultWithSchemaResp queryResultWithColumns; - if (isCache) { - queryResultWithColumns = queryByCache(cacheKey, queryMultiStructReq); - if (queryResultWithColumns != null) { - statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); - return queryResultWithColumns; - } - } - log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq); + try { - QueryStatement sqlParser = getQueryStatementByMultiStruct(queryMultiStructReq); - queryResultWithColumns = headlessQueryEngine.execute(sqlParser); - if (queryResultWithColumns != null) { + //1.initStatInfo + statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user); + //2.query from cache + Object query = queryCache.query(queryMultiStructReq); + if (Objects.nonNull(query)) { statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); - queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructReq); + return (SemanticQueryResp) query; } - return queryResultWithColumns; + StatUtils.get().setUseResultCache(false); + log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq); + List sqlParsers = new ArrayList<>(); + + for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { + QueryStatement queryStatement = buildQueryStatement(queryStructReq); + queryStatement = queryParser.parse(queryStatement); + queryPlanner.optimizer(queryStatement); + sqlParsers.add(queryStatement); + } + log.info("multi sqlParser:{}", sqlParsers); + QueryStatement queryStatement = queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); + QueryExecutor executor = queryPlanner.route(queryStatement); + + SemanticQueryResp semanticQueryResp = null; + if (executor != null) { + semanticQueryResp = executor.execute(queryStatement); + if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) { + queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds()); + } + } + if (Objects.nonNull(semanticQueryResp)) { + statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); + } + return semanticQueryResp; } catch (Exception e) { log.warn("exception in queryByMultiStruct, e: ", e); statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR); @@ -212,13 +261,11 @@ public class QueryServiceImpl implements QueryService { } } - private QueryStatement getQueryStatementByMultiStruct(QueryMultiStructReq queryMultiStructReq) throws Exception { + private QueryStatement buildQueryStatement(QueryMultiStructReq queryMultiStructReq) throws Exception { List sqlParsers = new ArrayList<>(); for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { - QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryStructReq(queryStructReq); - queryStatement.setIsS2SQL(false); - queryStatement = headlessQueryEngine.plan(queryStatement); + QueryStatement queryStatement = buildQueryStatement(queryStructReq); + queryStatement = semantciQueryEngine.plan(queryStatement); queryUtils.checkSqlParse(queryStatement); sqlParsers.add(queryStatement); } @@ -228,17 +275,9 @@ public class QueryServiceImpl implements QueryService { @Override @SneakyThrows - public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { - QueryS2SQLReq queryS2SQLReq = generateDimValueQuerySql(queryDimValueReq); - return (QueryResultWithSchemaResp) queryBySql(queryS2SQLReq, user); - } - - private void handleGlobalCacheDisable(QueryStructReq queryStructReq) { - if (!cacheEnable) { - Cache cacheInfo = new Cache(); - cacheInfo.setCache(false); - queryStructReq.setCacheInfo(cacheInfo); - } + public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { + QuerySqlReq querySQLReq = buildQuerySqlReq(queryDimValueReq); + return (SemanticQueryResp) queryBySql(querySQLReq, user); } @Override @@ -259,20 +298,18 @@ public class QueryServiceImpl implements QueryService { QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum(); T queryReq = explainSqlReq.getQueryReq(); - if (QueryType.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2SQLReq) { - QueryStatement queryStatement = convertToQueryStatement((QueryS2SQLReq) queryReq, user); + if (QueryType.SQL.equals(queryTypeEnum) && queryReq instanceof QuerySqlReq) { + QueryStatement queryStatement = convertToQueryStatement((QuerySqlReq) queryReq, user); return getExplainResp(queryStatement); } if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) { - QueryStatement queryStatement = new QueryStatement(); - queryStatement.setQueryStructReq((QueryStructReq) queryReq); - queryStatement.setIsS2SQL(false); - queryStatement = headlessQueryEngine.plan(queryStatement); + QueryStatement queryStatement = buildQueryStatement((QueryStructReq) queryReq); + queryStatement = semantciQueryEngine.plan(queryStatement); return getExplainResp(queryStatement); } if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryMultiStructReq) { QueryMultiStructReq queryMultiStructReq = (QueryMultiStructReq) queryReq; - QueryStatement queryStatement = getQueryStatementByMultiStruct(queryMultiStructReq); + QueryStatement queryStatement = buildQueryStatement(queryMultiStructReq); return getExplainResp(queryStatement); } @@ -310,11 +347,11 @@ public class QueryServiceImpl implements QueryService { dimensionResps = catalog.getDimensions(dimensionFilter); } QueryStructReq queryStructReq = buildQueryStructReq(dimensionResps, metricResp, dateConf, limit); - QueryResultWithSchemaResp queryResultWithSchemaResp = + SemanticQueryResp semanticQueryResp = queryByStruct(queryStructReq, User.getAppUser(appId)); SingleItemQueryResult apiQuerySingleResult = new SingleItemQueryResult(); apiQuerySingleResult.setItem(item); - apiQuerySingleResult.setResult(queryResultWithSchemaResp); + apiQuerySingleResult.setResult(semanticQueryResp); return apiQuerySingleResult; } @@ -366,21 +403,6 @@ public class QueryServiceImpl implements QueryService { return ExplainResp.builder().sql(sql).build(); } - public QueryStatement parseMetricReq(MetricQueryReq metricReq) throws Exception { - QueryStructReq queryStructReq = new QueryStructReq(); - return headlessQueryEngine.physicalSql(queryStructReq, metricReq); - } - - private boolean isCache(QueryStructReq queryStructReq) { - if (!cacheEnable) { - return false; - } - if (queryStructReq.getCacheInfo() != null) { - return queryStructReq.getCacheInfo().getCache(); - } - return false; - } - private boolean isCache(QueryMultiStructReq queryStructReq) { if (!cacheEnable) { return false; @@ -392,19 +414,19 @@ public class QueryServiceImpl implements QueryService { return false; } - private QueryResultWithSchemaResp queryByCache(String key, Object queryCmd) { + private SemanticQueryResp queryByCache(String key, Object queryCmd) { - Object resultObject = cacheUtils.get(key); + Object resultObject = cacheManager.get(key); if (Objects.nonNull(resultObject)) { log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString()); statUtils.updateResultCacheKey(key); - return (QueryResultWithSchemaResp) resultObject; + return (SemanticQueryResp) resultObject; } return null; } - private QueryS2SQLReq generateDimValueQuerySql(QueryDimValueReq queryDimValueReq) { - QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq(); + private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) { + QuerySqlReq querySQLReq = new QuerySqlReq(); List modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(), queryDimValueReq.getModelId()); @@ -416,9 +438,9 @@ public class QueryServiceImpl implements QueryService { queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate()); } - queryS2SQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); - queryS2SQLReq.setSql(sql); - return queryS2SQLReq; + querySQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); + querySQLReq.setSql(sql); + return querySQLReq; } private String getKeyByModelIds(List modelIds) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemantciQueryEngineImpl.java similarity index 77% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemantciQueryEngineImpl.java index 7a4308993..4406682bb 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemantciQueryEngineImpl.java @@ -3,16 +3,15 @@ package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.headless.api.request.MetricQueryReq; import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.executor.QueryExecutor; -import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; +import com.tencent.supersonic.headless.core.planner.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.QueryParser; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.ComponentFactory; -import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; -import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; -import com.tencent.supersonic.headless.server.utils.QueryStructUtils; +import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; +import com.tencent.supersonic.headless.server.service.SemantciQueryEngine; import com.tencent.supersonic.headless.server.utils.QueryUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -20,24 +19,21 @@ import org.springframework.util.CollectionUtils; @Slf4j @Component -public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { +public class SemantciQueryEngineImpl implements SemantciQueryEngine { private final QueryParser queryParser; private final QueryUtils queryUtils; - private final QueryStructUtils queryStructUtils; - private final HeadlessSchemaManager headlessSchemaManager; + private final SemanticSchemaManager semanticSchemaManager; - public HeadlessQueryEngineImpl(QueryParser queryParser, - QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager, - QueryStructUtils queryStructUtils) { + public SemantciQueryEngineImpl(QueryParser queryParser, + QueryUtils queryUtils, SemanticSchemaManager semanticSchemaManager) { this.queryParser = queryParser; this.queryUtils = queryUtils; - this.headlessSchemaManager = headlessSchemaManager; - this.queryStructUtils = queryStructUtils; + this.semanticSchemaManager = semanticSchemaManager; } - public QueryResultWithSchemaResp execute(QueryStatement queryStatement) { - QueryResultWithSchemaResp queryResultWithColumns = null; + public SemanticQueryResp execute(QueryStatement queryStatement) { + SemanticQueryResp queryResultWithColumns = null; QueryExecutor queryExecutor = route(queryStatement); if (queryExecutor != null) { queryResultWithColumns = queryExecutor.execute(queryStatement); @@ -52,7 +48,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { public QueryStatement plan(QueryStatement queryStatement) throws Exception { queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setSemanticModel(getSemanticModel(queryStatement)); - queryStatement = queryParser.logicSql(queryStatement); + queryStatement = queryParser.parse(queryStatement); queryUtils.checkSqlParse(queryStatement); queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds()); log.info("queryStatement:{}", queryStatement); @@ -97,7 +93,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception { QueryStructReq queryStructReq = queryStatement.getQueryStructReq(); - return headlessSchemaManager.get(queryStructReq.getModelIdStr()); + return semanticSchemaManager.get(queryStructReq.getModelIdStr()); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 85db71323..15cc8805b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -16,7 +16,7 @@ import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.ParseSqlReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; @@ -29,7 +29,7 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; -import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; +import com.tencent.supersonic.headless.server.service.SemantciQueryEngine; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -56,7 +56,7 @@ public class QueryReqConverter { private Boolean limitWrapper; @Autowired - private HeadlessQueryEngine headlessQueryEngine; + private SemantciQueryEngine semantciQueryEngine; @Autowired private QueryStructUtils queryStructUtils; @@ -66,7 +66,7 @@ public class QueryReqConverter { @Autowired private Catalog catalog; - public QueryStatement convert(QueryS2SQLReq queryS2SQLReq, + public QueryStatement convert(QuerySqlReq querySQLReq, List modelSchemaResps) throws Exception { if (CollectionUtils.isEmpty(modelSchemaResps)) { @@ -75,18 +75,18 @@ public class QueryReqConverter { Map modelSchemaRespMap = modelSchemaResps.stream() .collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp)); //1.convert name to bizName - convertNameToBizName(queryS2SQLReq, modelSchemaResps); + convertNameToBizName(querySQLReq, modelSchemaResps); //2.functionName corrector - functionNameCorrector(queryS2SQLReq); + functionNameCorrector(querySQLReq); //3.correct tableName - correctTableName(queryS2SQLReq); + correctTableName(querySQLReq); - String tableName = SqlParserSelectHelper.getTableName(queryS2SQLReq.getSql()); + String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql()); if (StringUtils.isEmpty(tableName)) { return new QueryStatement(); } //4.build MetricTables - List allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql()); + List allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); List metrics = getMetrics(modelSchemaResps, allFields); QueryStructReq queryStructReq = new QueryStructReq(); MetricTable metricTable = new MetricTable(); @@ -100,7 +100,7 @@ public class QueryReqConverter { // if metric empty , fill model default if (CollectionUtils.isEmpty(metricTable.getMetrics())) { metricTable.setMetrics(new ArrayList<>()); - for (Long modelId : queryS2SQLReq.getModelIds()) { + for (Long modelId : querySQLReq.getModelIds()) { ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId); metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName())); } @@ -109,27 +109,27 @@ public class QueryReqConverter { metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( Collectors.toList())); } - AggOption aggOption = getAggOption(queryS2SQLReq); + AggOption aggOption = getAggOption(querySQLReq); metricTable.setAggOption(aggOption); List tables = new ArrayList<>(); tables.add(metricTable); //4.build ParseSqlReq ParseSqlReq result = new ParseSqlReq(); - BeanUtils.copyProperties(queryS2SQLReq, result); + BeanUtils.copyProperties(querySQLReq, result); - result.setRootPath(queryS2SQLReq.getModelIdStr()); + result.setRootPath(querySQLReq.getModelIdStr()); result.setTables(tables); - DatabaseResp database = catalog.getDatabaseByModelId(queryS2SQLReq.getModelIds().get(0)); + DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0)); if (!sqlGenerateUtils.isSupportWith(EngineType.valueOf(database.getType().toUpperCase()), database.getVersion())) { result.setSupportWith(false); result.setWithAlias(false); } //5. do deriveMetric - generateDerivedMetric(queryS2SQLReq.getModelIds(), modelSchemaResps, result); + generateDerivedMetric(querySQLReq.getModelIds(), modelSchemaResps, result); //6.physicalSql by ParseSqlReq - queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql())); - queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds())); + queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); + queryStructReq.setModelIds(new HashSet<>(querySQLReq.getModelIds())); queryStructReq.setQueryType(getQueryType(aggOption)); log.info("QueryReqConverter queryStructReq[{}]", queryStructReq); QueryStatement queryStatement = new QueryStatement(); @@ -137,14 +137,14 @@ public class QueryReqConverter { queryStatement.setParseSqlReq(result); queryStatement.setIsS2SQL(true); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - queryStatement.setModelIds(queryS2SQLReq.getModelIds()); - queryStatement = headlessQueryEngine.plan(queryStatement); + queryStatement.setModelIds(querySQLReq.getModelIds()); + queryStatement = semantciQueryEngine.plan(queryStatement); queryStatement.setSql(limitWrapper ? String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql()) : queryStatement.getSql()); return queryStatement; } - private AggOption getAggOption(QueryS2SQLReq databaseReq) { + private AggOption getAggOption(QuerySqlReq databaseReq) { // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" String sql = databaseReq.getSql(); @@ -156,7 +156,7 @@ public class QueryReqConverter { return AggOption.DEFAULT; } - private void convertNameToBizName(QueryS2SQLReq databaseReq, List modelSchemaResps) { + private void convertNameToBizName(QuerySqlReq databaseReq, List modelSchemaResps) { Map fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps); String sql = databaseReq.getSql(); log.info("convert name to bizName before:{}", sql); @@ -186,7 +186,7 @@ public class QueryReqConverter { .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList()); } - private void functionNameCorrector(QueryS2SQLReq databaseReq) { + private void functionNameCorrector(QuerySqlReq databaseReq) { DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0)); if (Objects.isNull(database) || Objects.isNull(database.getType())) { return; @@ -231,7 +231,7 @@ public class QueryReqConverter { return elements.stream(); } - public void correctTableName(QueryS2SQLReq databaseReq) { + public void correctTableName(QuerySqlReq databaseReq) { String sql = databaseReq.getSql(); for (Long modelId : databaseReq.getModelIds()) { sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index 2f9ba1940..e1b058377 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -1,5 +1,10 @@ package com.tencent.supersonic.headless.server.utils; +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; + import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; @@ -14,7 +19,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.pojo.ItemDateFilter; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimSchemaResp; import com.tencent.supersonic.headless.api.response.DimensionResp; @@ -24,13 +29,6 @@ import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.SchemaService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -44,11 +42,12 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; - -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Slf4j @@ -130,8 +129,8 @@ public class QueryStructUtils { return resNameEnSet; } - public Set getResName(QueryS2SQLReq queryS2SQLReq) { - Set resNameSet = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql()) + public Set getResName(QuerySqlReq querySQLReq) { + Set resNameSet = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()) .stream().collect(Collectors.toSet()); return resNameSet; } @@ -141,11 +140,11 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public Set getResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq, User user) { - Set resNameSet = getResName(queryS2SQLReq); + public Set getResNameEnExceptInternalCol(QuerySqlReq querySQLReq, User user) { + Set resNameSet = getResName(querySQLReq); Set resNameEnSet = new HashSet<>(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - List modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds()); + List modelIds = Lists.newArrayList(querySQLReq.getModelIds()); filter.setModelIds(modelIds); List modelSchemaRespList = schemaService.fetchModelSchema(filter, user); if (!CollectionUtils.isEmpty(modelSchemaRespList)) { @@ -176,8 +175,8 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public Set getFilterResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq) { - String sql = queryS2SQLReq.getSql(); + public Set getFilterResNameEnExceptInternalCol(QuerySqlReq querySQLReq) { + String sql = querySQLReq.getSql(); Set resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 6e47dd672..5da568d31 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -1,40 +1,37 @@ package com.tencent.supersonic.headless.server.utils; +import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; + import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.cache.CacheUtils; -import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.enums.SemanticType; +import com.tencent.supersonic.headless.api.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; import com.tencent.supersonic.headless.api.response.MetricResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import com.tencent.supersonic.headless.server.cache.CacheManager; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import javax.annotation.PostConstruct; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; - -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.UNIONALL; +import javax.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Slf4j @@ -52,15 +49,15 @@ public class QueryUtils { @Value("${query.optimizer.enable:true}") private Boolean optimizeEnable; - private final CacheUtils cacheUtils; + private final CacheManager cacheManager; private final StatUtils statUtils; private final Catalog catalog; public QueryUtils( - CacheUtils cacheUtils, StatUtils statUtils, Catalog catalog) { + CacheManager cacheManager, StatUtils statUtils, Catalog catalog) { - this.cacheUtils = cacheUtils; + this.cacheManager = cacheManager; this.statUtils = statUtils; this.catalog = catalog; } @@ -74,7 +71,7 @@ public class QueryUtils { } } - public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, List modelIds) { + public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns, List modelIds) { MetaFilter metaFilter = new MetaFilter(modelIds); List metricDescList = catalog.getMetrics(metaFilter); List dimensionDescList = catalog.getDimensions(metaFilter); @@ -125,7 +122,7 @@ public class QueryUtils { }); } - public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, + public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns, QueryMultiStructReq queryMultiStructCmd) { List aggregators = queryMultiStructCmd.getQueryStructReqs().stream() .flatMap(queryStructCmd -> queryStructCmd.getAggregators().stream()) @@ -244,21 +241,6 @@ public class QueryUtils { return sqlParser; } - public void cacheResultLogic(String key, QueryResultWithSchemaResp queryResultWithColumns) { - if (cacheEnable && Objects.nonNull(queryResultWithColumns) && !CollectionUtils.isEmpty( - queryResultWithColumns.getResultList())) { - QueryResultWithSchemaResp finalQueryResultWithColumns = queryResultWithColumns; - CompletableFuture.supplyAsync(() -> cacheUtils.put(key, finalQueryResultWithColumns)) - .exceptionally(exception -> { - log.warn("exception:", exception); - return null; - }); - statUtils.updateResultCacheKey(key); - log.info("add record to cache, key:{}", key); - } - - } - public Boolean enableOptimize() { return optimizeEnable; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index 45c379bd3..652f1288d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -13,7 +13,7 @@ import com.tencent.supersonic.headless.api.enums.QueryTypeBack; import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.ItemUseReq; -import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; +import com.tencent.supersonic.headless.api.request.QuerySqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.ItemUseResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; @@ -85,11 +85,11 @@ public class StatUtils { return true; } - public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) { + public void initStatInfo(QuerySqlReq querySQLReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); - List allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql()); - queryStatInfo.setModelId(queryS2SQLReq.getModelIds().get(0)); - ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelIds().get(0)); + List allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); + queryStatInfo.setModelId(querySQLReq.getModelIds().get(0)); + ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySQLReq.getModelIds().get(0)); List dimensions = new ArrayList<>(); List metrics = new ArrayList<>(); @@ -101,12 +101,12 @@ public class StatUtils { String userName = getUserName(facadeUser); try { queryStatInfo.setTraceId("") - .setModelId(queryS2SQLReq.getModelIds().get(0)) + .setModelId(querySQLReq.getModelIds().get(0)) .setUser(userName) .setQueryType(QueryType.SQL.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) - .setQuerySqlCmd(queryS2SQLReq.toString()) - .setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2SQLReq.toString())) + .setQuerySqlCmd(querySQLReq.toString()) + .setQuerySqlCmdMd5(DigestUtils.md5Hex(querySQLReq.toString())) .setStartTime(System.currentTimeMillis()) .setUseResultCache(true) .setUseSqlCache(true) diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index 826fd723d..7c24128ca 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -14,7 +14,7 @@ import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; -import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; +import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -124,7 +124,7 @@ class HeadlessParserServiceTest { SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build(); - HeadlessSchemaManager.update(semanticSchema, HeadlessSchemaManager.getDatasource(datasource)); + SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getDatasource(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("page"); @@ -133,8 +133,8 @@ class HeadlessParserServiceTest { List dimensionYamlTpls = new ArrayList<>(); dimensionYamlTpls.add(dimension1); - HeadlessSchemaManager.update(semanticSchema, "s2_pv_uv_statis", - HeadlessSchemaManager.getDimensions(dimensionYamlTpls)); + SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis", + SemanticSchemaManager.getDimensions(dimensionYamlTpls)); MetricYamlTpl metric1 = new MetricYamlTpl(); metric1.setName("pv"); @@ -240,7 +240,7 @@ class HeadlessParserServiceTest { identifies.add(identify); datasource.setIdentifiers(identifies); - semanticSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource)); + semanticSchema.getDatasource().put("user_department", SemanticSchemaManager.getDatasource(datasource)); DimensionYamlTpl dimension1 = new DimensionYamlTpl(); dimension1.setExpr("department"); @@ -250,6 +250,6 @@ class HeadlessParserServiceTest { dimensionYamlTpls.add(dimension1); semanticSchema.getDimension() - .put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls)); + .put("user_department", SemanticSchemaManager.getDimensions(dimensionYamlTpls)); } } diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java index 6574e57e9..c6bd40db6 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java @@ -1,5 +1,8 @@ package com.tencent.supersonic.headless.server.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + import com.alibaba.excel.util.FileUtils; import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; @@ -11,20 +14,16 @@ import com.tencent.supersonic.headless.api.request.BatchDownloadReq; import com.tencent.supersonic.headless.api.response.DimSchemaResp; import com.tencent.supersonic.headless.api.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; -import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.headless.api.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.service.impl.DownloadServiceImpl; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - import java.io.File; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; class DownloadServiceImplTest { @@ -98,16 +97,16 @@ class DownloadServiceImplTest { return dateConf; } - private QueryResultWithSchemaResp mockQueryResult() { - QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp(); + private SemanticQueryResp mockQueryResult() { + SemanticQueryResp semanticQueryResp = new SemanticQueryResp(); List> resultList = Lists.newArrayList(); resultList.add(createMap("2023-10-11", "tom", "hr", "1")); resultList.add(createMap("2023-10-12", "alice", "sales", "2")); resultList.add(createMap("2023-10-13", "jack", "sales", "3")); resultList.add(createMap("2023-10-14", "luck", "market", "4")); resultList.add(createMap("2023-10-15", "tom", "hr", "5")); - queryResultWithSchemaResp.setResultList(resultList); - return queryResultWithSchemaResp; + semanticQueryResp.setResultList(resultList); + return semanticQueryResp; } private static Map createMap(String sysImpDate, String d1, String d2, String m1) {