topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Headless has preliminarily completed the abstraction of QueryCache, QueryParser, QueryPlanner, and QueryExecutor. (#651)

Browse Source
This commit is contained in:
lexluo09
2024-01-18 22:39:58 +08:00
committed by GitHub
parent b019f4d9bb
commit 3e77fc3069
56 changed files with 699 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.DomainResp;
@@ -19,12 +19,12 @@ import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import java.util.HashMap;
import java.util.ArrayList;
import java.util.List;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@@ -40,20 +40,20 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@SneakyThrows
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet());
queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2SQL(queryS2SQLReq, user);
QuerySqlReq querySQLReq = new QuerySqlReq();
querySQLReq.setSql(queryStructReq.getCorrectS2SQL());
querySQLReq.setModelIds(queryStructReq.getModelIdSet());
querySQLReq.setParams(new ArrayList<>());
return queryByS2SQL(querySQLReq, user);
}
queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryByStructWithAuth(queryStructReq, user);
}
@Override
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
try {
queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryByMultiStruct(queryMultiStructReq, user);
@@ -65,15 +65,15 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@Override
@SneakyThrows
public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) {
queryService = ContextUtils.getBean(QueryService.class);
Object object = queryService.queryBySql(queryS2SQLReq, user);
return JsonUtil.toObject(JsonUtil.toString(object), QueryResultWithSchemaResp.class);
Object object = queryService.queryBySql(querySQLReq, user);
return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class);
}
@Override
@SneakyThrows
public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryDimValue(queryDimValueReq, user);
}

48
chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java
View File

@@ -20,24 +20,24 @@ import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.S2ThreadContext;
import com.tencent.supersonic.common.util.ThreadContext;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.request.PageMetricReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.DomainResp;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import java.net.URI;
import java.net.URL;
import java.util.HashMap;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
@@ -61,8 +61,8 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
private AuthenticationConfig authenticationConfig;
private ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>> structTypeRef =
new ParameterizedTypeReference<ResultData<QueryResultWithSchemaResp>>() {
private ParameterizedTypeReference<ResultData<SemanticQueryResp>> structTypeRef =
new ParameterizedTypeReference<ResultData<SemanticQueryResp>>() {
};
private ParameterizedTypeReference<ResultData<ExplainResp>> explainTypeRef =
@@ -70,13 +70,13 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
};
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) {
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
queryS2SQLReq.setSql(queryStructReq.getCorrectS2SQL());
queryS2SQLReq.setModelIds(queryStructReq.getModelIdSet());
queryS2SQLReq.setVariables(new HashMap<>());
return queryByS2SQL(queryS2SQLReq, user);
QuerySqlReq querySQLReq = new QuerySqlReq();
querySQLReq.setSql(queryStructReq.getCorrectS2SQL());
querySQLReq.setModelIds(queryStructReq.getModelIdSet());
querySQLReq.setParams(new ArrayList<>());
return queryByS2SQL(querySQLReq, user);
}
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
@@ -86,7 +86,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
}
@Override
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(
defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchByMultiStructPath(),
@@ -94,30 +94,30 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
}
@Override
public QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user) {
public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
new Gson().toJson(queryS2SQLReq));
new Gson().toJson(querySQLReq));
}
public QueryResultWithSchemaResp searchByRestTemplate(String url, String jsonReq) {
public SemanticQueryResp searchByRestTemplate(String url, String jsonReq) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
fillToken(headers);
URI requestUrl = UriComponentsBuilder.fromHttpUrl(url).build().encode().toUri();
HttpEntity<String> entity = new HttpEntity<>(jsonReq, headers);
log.info("url:{},searchByRestTemplate:{}", url, entity.getBody());
ResultData<QueryResultWithSchemaResp> responseBody;
ResultData<SemanticQueryResp> responseBody;
try {
RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
ResponseEntity<ResultData<QueryResultWithSchemaResp>> responseEntity = restTemplate.exchange(
ResponseEntity<ResultData<SemanticQueryResp>> responseEntity = restTemplate.exchange(
requestUrl, HttpMethod.POST, entity, structTypeRef);
responseBody = responseEntity.getBody();
log.info("ApiResponse<QueryResultWithColumns> responseBody:{}", responseBody);
QueryResultWithSchemaResp schemaResp = new QueryResultWithSchemaResp();
SemanticQueryResp schemaResp = new SemanticQueryResp();
if (ReturnCode.SUCCESS.getCode() == responseBody.getCode()) {
QueryResultWithSchemaResp data = responseBody.getData();
SemanticQueryResp data = responseBody.getData();
schemaResp.setColumns(data.getColumns());
schemaResp.setResultList(data.getResultList());
schemaResp.setSql(data.getSql());
@@ -131,7 +131,7 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
}
@Override
public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl()
+ defaultSemanticConfig.getQueryDimValuePath(),

12
chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java
View File

@@ -12,10 +12,10 @@ import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
@@ -34,13 +34,13 @@ import java.util.List;
*/
public interface SemanticInterpreter {
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user);
SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user);
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
QueryResultWithSchemaResp queryByS2SQL(QueryS2SQLReq queryS2SQLReq, User user);
SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user);
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
List<ModelSchema> getModelSchema();

8
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java
View File

@@ -16,7 +16,7 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import java.io.Serializable;
@@ -121,9 +121,9 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
}
QueryStructReq queryStructReq = convertQueryStruct();
convertBizNameToName(semanticSchema, queryStructReq);
QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
parseInfo.getSqlInfo().setS2SQL(queryS2SQLReq.getSql());
parseInfo.getSqlInfo().setCorrectS2SQL(queryS2SQLReq.getSql());
QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq);
parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql());
parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql());
}
}

8
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java
View File

@@ -20,7 +20,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -59,8 +59,8 @@ public class MetricAnalyzeQuery extends LLMSemanticQuery {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getQuerySQL());
}
QueryResultWithSchemaResp queryResultWithSchemaResp = semanticInterpreter.queryByStruct(queryStructReq, user);
String text = generateTableText(queryResultWithSchemaResp);
SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
String text = generateTableText(semanticQueryResp);
Map<String, Object> properties = parseInfo.getProperties();
Map<String, String> replacedMap = new HashMap<>();
String textReplaced = replaceText((String) properties.get("queryText"),
@@ -130,7 +130,7 @@ public class MetricAnalyzeQuery extends LLMSemanticQuery {
return text;
}
public static String generateTableText(QueryResultWithSchemaResp result) {
public static String generateTableText(SemanticQueryResp result) {
StringBuilder tableBuilder = new StringBuilder();
for (QueryColumn column : result.getColumns()) {
tableBuilder.append(column.getName()).append("\t");

8
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java
View File

@@ -11,8 +11,8 @@ import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.chat.core.query.QueryManager;
import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -42,8 +42,8 @@ public class LLMSqlQuery extends LLMSemanticQuery {
long startTime = System.currentTimeMillis();
String querySql = parseInfo.getSqlInfo().getCorrectS2SQL();
QueryS2SQLReq queryS2SQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds());
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByS2SQL(queryS2SQLReq, user);
QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds());
SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySQLReq, user);
log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql);

6
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java
View File

@@ -22,7 +22,7 @@ import com.tencent.supersonic.common.pojo.ModelCluster;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import lombok.ToString;
@@ -207,7 +207,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
queryStructReq.setS2SQL(parseInfo.getSqlInfo().getS2SQL());
queryStructReq.setCorrectS2SQL(parseInfo.getSqlInfo().getCorrectS2SQL());
}
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
if (queryResp != null) {
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
@@ -242,7 +242,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery {
QueryResult queryResult = new QueryResult();
QueryMultiStructReq queryMultiStructReq = convertQueryMultiStruct();
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByMultiStruct(queryMultiStructReq, user);
SemanticQueryResp queryResp = semanticInterpreter.queryByMultiStruct(queryMultiStructReq, user);
if (queryResp != null) {
queryResult.setQueryAuthorization(queryResp.getQueryAuthorization());
}

12
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java
View File

@@ -36,7 +36,7 @@ import com.tencent.supersonic.common.pojo.enums.RatioOverType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import java.text.DecimalFormat;
import java.time.DayOfWeek;
import java.time.LocalDate;
@@ -105,7 +105,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
public void fillAggregateInfo(User user, QueryResult queryResult) {
if (Objects.nonNull(queryResult)) {
QueryResultWithSchemaResp queryResp = new QueryResultWithSchemaResp();
SemanticQueryResp queryResp = new SemanticQueryResp();
queryResp.setColumns(queryResult.getQueryColumns());
queryResp.setResultList(queryResult.getQueryResults());
AggregateInfo aggregateInfo = getAggregateInfo(user, parseInfo, queryResp);
@@ -114,7 +114,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
}
public AggregateInfo getAggregateInfo(User user, SemanticParseInfo semanticParseInfo,
QueryResultWithSchemaResp result) {
SemanticQueryResp result) {
AggregatorConfig aggregatorConfig = ContextUtils.getBean(AggregatorConfig.class);
if (CollectionUtils.isEmpty(semanticParseInfo.getMetrics()) || !aggregatorConfig.getEnableRatio()) {
@@ -170,7 +170,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
}
private MetricInfo queryRatio(User user, SemanticParseInfo semanticParseInfo, SchemaElement metric,
AggOperatorEnum aggOperatorEnum, QueryResultWithSchemaResp results) {
AggOperatorEnum aggOperatorEnum, SemanticQueryResp results) {
MetricInfo metricInfo = new MetricInfo();
metricInfo.setStatistics(new HashMap<>());
QueryStructReq queryStructReq = QueryReqBuilder.buildStructRatioReq(semanticParseInfo, metric, aggOperatorEnum);
@@ -180,7 +180,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
queryStructReq.setGroups(new ArrayList<>(Arrays.asList(dateField)));
queryStructReq.setDateInfo(getRatioDateConf(aggOperatorEnum, semanticParseInfo, results));
QueryResultWithSchemaResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
SemanticQueryResp queryResp = semanticInterpreter.queryByStruct(queryStructReq, user);
if (Objects.nonNull(queryResp) && !CollectionUtils.isEmpty(queryResp.getResultList())) {
@@ -220,7 +220,7 @@ public abstract class MetricSemanticQuery extends RuleSemanticQuery {
}
private DateConf getRatioDateConf(AggOperatorEnum aggOperatorEnum, SemanticParseInfo semanticParseInfo,
QueryResultWithSchemaResp results) {
SemanticQueryResp results) {
String dateField = QueryReqBuilder.getDateField(semanticParseInfo.getDateInfo());
Optional<String> lastDayOp = results.getResultList().stream()
.map(r -> r.get(dateField).toString())

26
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/CacheUtils.java
View File

@@ -1,26 +0,0 @@
package com.tencent.supersonic.chat.core.utils;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.tencent.supersonic.chat.core.pojo.ChatContext;
import com.tencent.supersonic.chat.core.pojo.QueryContext;
import java.util.concurrent.TimeUnit;
public class CacheUtils {
private static final Cache<String, Object> cache = Caffeine.newBuilder()
.expireAfterWrite(1200, TimeUnit.SECONDS)
.expireAfterAccess(1200, TimeUnit.SECONDS)
.maximumSize(1000)
.build();
public static void put(QueryContext queryContext, ChatContext chatCtx, Object v) {
String key = chatCtx.getUser() + "_" + chatCtx.getChatId() + "_" + queryContext.getRequest().getQueryText();
cache.put(key, v);
}
public static Object get(QueryContext queryContext, ChatContext chatCtx) {
String key = chatCtx.getUser() + "_" + chatCtx.getChatId() + "_" + queryContext.getRequest().getQueryText();
return cache.getIfPresent(key);
}
}

14
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java
View File

@@ -18,7 +18,7 @@ import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import java.util.ArrayList;
import java.util.Arrays;
@@ -53,20 +53,20 @@ public class DictQueryHelper {
List<String> data = new ArrayList<>();
QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict);
try {
QueryResultWithSchemaResp queryResultWithColumns = semanticInterpreter.queryByStruct(queryStructCmd, user);
SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructCmd, user);
log.info("fetchDimValueSingle sql:{}", queryResultWithColumns.getSql());
log.info("fetchDimValueSingle sql:{}", semanticQueryResp.getSql());
String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId());
String dimNameRewrite = rewriteDimName(queryResultWithColumns.getColumns(), dim4Dict.getBizName());
data = generateFileData(queryResultWithColumns.getResultList(), nature, dimNameRewrite,
String dimNameRewrite = rewriteDimName(semanticQueryResp.getColumns(), dim4Dict.getBizName());
data = generateFileData(semanticQueryResp.getResultList(), nature, dimNameRewrite,
defaultMetricDesc.getBizName(), dim4Dict);
if (!CollectionUtils.isEmpty(data)) {
int size = (data.size() > printDataShow) ? printDataShow : data.size();
log.info("data:{}", data.subList(0, size));
} else {
log.warn("data is empty. nature:{}", nature);
if (Objects.nonNull(queryResultWithColumns)) {
log.warn("sql:{}", queryResultWithColumns.getSql());
if (Objects.nonNull(semanticQueryResp)) {
log.warn("sql:{}", semanticQueryResp.getSql());
}
}

12
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java
View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -133,13 +133,13 @@ public class QueryReqBuilder {
* @param modelIds
* @return
*/
public static QueryS2SQLReq buildS2SQLReq(String querySql, Set<Long> modelIds) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
public static QuerySqlReq buildS2SQLReq(String querySql, Set<Long> modelIds) {
QuerySqlReq querySQLReq = new QuerySqlReq();
if (Objects.nonNull(querySql)) {
queryS2SQLReq.setSql(querySql);
querySQLReq.setSql(querySql);
}
queryS2SQLReq.setModelIds(modelIds);
return queryS2SQLReq;
querySQLReq.setModelIds(modelIds);
return querySQLReq;
}
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {

8
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java
View File

@@ -27,7 +27,7 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collections;
@@ -162,7 +162,7 @@ public class SemanticService {
List<String> entities = Collections.singletonList(entity);
QueryResultWithSchemaResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, parseInfo, entities,
SemanticQueryResp queryResultWithColumns = getQueryResultWithSchemaResp(modelInfo, parseInfo, entities,
user);
if (queryResultWithColumns != null) {
@@ -183,7 +183,7 @@ public class SemanticService {
}
}
public QueryResultWithSchemaResp getQueryResultWithSchemaResp(EntityInfo modelInfo, SemanticParseInfo parseInfo,
public SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo modelInfo, SemanticParseInfo parseInfo,
List<String> entities, User user) {
if (CollectionUtils.isEmpty(entities)) {
return null;
@@ -219,7 +219,7 @@ public class SemanticService {
chatFilters.add(chatFilter);
semanticParseInfo.setDimensionFilters(chatFilters);
QueryResultWithSchemaResp queryResultWithColumns = null;
SemanticQueryResp queryResultWithColumns = null;
try {
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(semanticParseInfo);
queryResultWithColumns = semanticInterpreter.queryByStruct(queryStructReq, user);

16
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java
View File

@@ -64,7 +64,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserRemoveHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -628,7 +628,7 @@ public class QueryServiceImpl implements QueryService {
@Override
public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception {
QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp();
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
SemanticService semanticService = ContextUtils.getBean(SemanticService.class);
SemanticSchema semanticSchema = semanticService.getSemanticSchema();
SchemaElement schemaElement = semanticSchema.getDimensions(dimensionValueReq.getElementID());
@@ -638,8 +638,8 @@ public class QueryServiceImpl implements QueryService {
List<String> dimensionValues = getDimensionValues(dimensionValueReq, detectModelIds);
// if the search results is null,search dimensionValue from database
if (CollectionUtils.isEmpty(dimensionValues)) {
queryResultWithSchemaResp = queryDatabase(dimensionValueReq, user);
return queryResultWithSchemaResp;
semanticQueryResp = queryDatabase(dimensionValueReq, user);
return semanticQueryResp;
}
List<QueryColumn> columns = new ArrayList<>();
QueryColumn queryColumn = new QueryColumn();
@@ -654,9 +654,9 @@ public class QueryServiceImpl implements QueryService {
map.put(dimensionValueReq.getBizName(), o);
resultList.add(map);
});
queryResultWithSchemaResp.setColumns(columns);
queryResultWithSchemaResp.setResultList(resultList);
return queryResultWithSchemaResp;
semanticQueryResp.setColumns(columns);
semanticQueryResp.setResultList(resultList);
return semanticQueryResp;
}
private List<String> getDimensionValues(DimensionValueReq dimensionValueReq, Set<Long> detectModelIds) {
@@ -682,7 +682,7 @@ public class QueryServiceImpl implements QueryService {
.collect(Collectors.toList());
}
private QueryResultWithSchemaResp queryDatabase(DimensionValueReq dimensionValueReq, User user) {
private SemanticQueryResp queryDatabase(DimensionValueReq dimensionValueReq, User user) {
QueryStructReq queryStructReq = new QueryStructReq();
DateConf dateConf = new DateConf();

10
chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java
View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import java.util.ArrayList;
import java.util.Arrays;
@@ -51,18 +51,18 @@ class QueryReqBuilderTest {
orders.add(order);
queryStructReq.setOrders(orders);
QueryS2SQLReq queryS2SQLReq = queryStructReq.convert(queryStructReq);
QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals(
"SELECT department, SUM(pv) AS pv FROM 内容库 "
+ "WHERE (sys_imp_date IN ('2023-08-01')) GROUP "
+ "BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
+ "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql());
queryStructReq.setQueryType(QueryType.TAG);
queryS2SQLReq = queryStructReq.convert(queryStructReq);
querySQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals(
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "ORDER BY uv LIMIT 2000",
queryS2SQLReq.getSql());
querySQLReq.getSql());
}

4
common/src/main/META-INF/spring.factories
View File

@@ -1,2 +1,2 @@
com.tencent.supersonic.common.util.cache.CacheUtils=\
com.tencent.supersonic.common.util.cache.CaffeineCacheImpl
com.tencent.supersonic.headless.server.cache.CacheManager=\
com.tencent.supersonic.headless.server.cache.CaffeineCacheManager

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SingleItemQueryResult.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.api.pojo;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import lombok.Data;
@Data
@@ -9,6 +9,6 @@ public class SingleItemQueryResult {
private Item item;
private QueryResultWithSchemaResp result;
private SemanticQueryResp result;
}

20
headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryMultiStructReq.java
View File

@@ -1,15 +1,17 @@
package com.tencent.supersonic.headless.api.request;
import com.alibaba.fastjson.JSONObject;
import com.tencent.supersonic.headless.api.pojo.Cache;
import java.util.ArrayList;
import java.util.List;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.codec.digest.DigestUtils;
import org.springframework.util.CollectionUtils;
@Data
@ToString
public class QueryMultiStructReq {
public class QueryMultiStructReq extends SemanticQueryReq {
List<QueryStructReq> queryStructReqs;
@@ -21,4 +23,18 @@ public class QueryMultiStructReq {
return DigestUtils.md5Hex(this.toCustomizedString());
}
public List<Long> getModelIds() {
if (CollectionUtils.isEmpty(this.getQueryStructReqs())) {
return new ArrayList<>();
}
return this.getQueryStructReqs().get(0).getModelIds();
}
public Cache getCacheInfo() {
if (CollectionUtils.isEmpty(this.getQueryStructReqs())) {
return getCacheInfo();
}
return this.getQueryStructReqs().get(0).getCacheInfo();
}
}

35
headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryS2SQLReq.java
View File

@@ -1,35 +0,0 @@
package com.tencent.supersonic.headless.api.request;
import com.google.common.collect.Lists;
import lombok.Data;
import lombok.ToString;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Data
@ToString
public class QueryS2SQLReq {
private Set<Long> modelIds;
private String sql;
private Map<String, String> variables;
public void setModelId(Long modelId) {
modelIds = new HashSet<>();
modelIds.add(modelId);
}
public List<Long> getModelIds() {
return Lists.newArrayList(modelIds);
}
public String getModelIdStr() {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
}

27
headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QuerySqlReq.java Normal file
View File

@@ -0,0 +1,27 @@
package com.tencent.supersonic.headless.api.request;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class QuerySqlReq extends SemanticQueryReq {
private String sql;
@Override
public String toCustomizedString() {
StringBuilder stringBuilder = new StringBuilder("{");
stringBuilder.append("\"modelId\":")
.append(modelIds);
stringBuilder.append(",\"params\":")
.append(params);
stringBuilder.append(",\"cacheInfo\":")
.append(cacheInfo);
stringBuilder.append(",\"sql\":")
.append(sql);
stringBuilder.append('}');
return stringBuilder.toString();
}
}

53
headless/api/src/main/java/com/tencent/supersonic/headless/api/request/QueryStructReq.java
View File

@@ -6,14 +6,16 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
import com.tencent.supersonic.common.util.SqlFilterUtils;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Param;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
@@ -37,20 +39,10 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.util.Strings;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Data
@Slf4j
public class QueryStructReq {
private Set<Long> modelIds;
public class QueryStructReq extends SemanticQueryReq {
private String modelName;
private List<String> groups = new ArrayList<>();
@@ -58,11 +50,10 @@ public class QueryStructReq {
private List<Order> orders = new ArrayList<>();
private List<Filter> dimensionFilters = new ArrayList<>();
private List<Filter> metricFilters = new ArrayList<>();
private List<Param> params = new ArrayList<>();
private DateConf dateInfo;
private Long limit = 2000L;
private QueryType queryType = QueryType.ID;
private Cache cacheInfo;
/**
* Later deleted for compatibility only
@@ -73,23 +64,6 @@ public class QueryStructReq {
*/
private String correctS2SQL;
public void setModelId(Long modelId) {
modelIds = new HashSet<>();
modelIds.add(modelId);
}
public List<Long> getModelIds() {
return Lists.newArrayList(modelIds);
}
public String getModelIdStr() {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
public Set<Long> getModelIdSet() {
return modelIds;
}
public List<String> getGroups() {
if (!CollectionUtils.isEmpty(this.groups)) {
this.groups = groups.stream().filter(group -> !Strings.isEmpty(group)).collect(Collectors.toList());
@@ -117,13 +91,6 @@ public class QueryStructReq {
return orders;
}
public List<Param> getParams() {
if (params == null) {
return Lists.newArrayList();
}
return params;
}
public String toCustomizedString() {
StringBuilder stringBuilder = new StringBuilder("{");
stringBuilder.append("\"modelId\":")
@@ -187,7 +154,7 @@ public class QueryStructReq {
* @param queryStructReq
* @return
*/
public QueryS2SQLReq convert(QueryStructReq queryStructReq) {
public QuerySqlReq convert(QueryStructReq queryStructReq) {
String sql = null;
try {
sql = buildSql(queryStructReq);
@@ -195,10 +162,10 @@ public class QueryStructReq {
log.error("buildSql error", e);
}
QueryS2SQLReq result = new QueryS2SQLReq();
QuerySqlReq result = new QuerySqlReq();
result.setSql(sql);
result.setModelIds(queryStructReq.getModelIdSet());
result.setVariables(new HashMap<>());
result.setParams(new ArrayList<>());
return result;
}

48
headless/api/src/main/java/com/tencent/supersonic/headless/api/request/SemanticQueryReq.java Normal file
View File

@@ -0,0 +1,48 @@
package com.tencent.supersonic.headless.api.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Param;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
@Data
@Slf4j
public abstract class SemanticQueryReq {
protected Set<Long> modelIds;
protected List<Param> params = new ArrayList<>();
protected Cache cacheInfo = new Cache();
public void setModelId(Long modelId) {
modelIds = new HashSet<>();
modelIds.add(modelId);
}
public String generateCommandMd5() {
return DigestUtils.md5Hex(this.toCustomizedString());
}
public abstract String toCustomizedString();
public List<Long> getModelIds() {
return Lists.newArrayList(modelIds);
}
public String getModelIdStr() {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
public Set<Long> getModelIdSet() {
return modelIds;
}
}

2
headless/api/src/main/java/com/tencent/supersonic/headless/api/response/QueryResultWithSchemaResp.java → headless/api/src/main/java/com/tencent/supersonic/headless/api/response/SemanticQueryResp.java
View File

@@ -14,7 +14,7 @@ import java.util.stream.Collectors;
@Data
@ToString
public class QueryResultWithSchemaResp extends QueryResult<Map<String, Object>> {
public class SemanticQueryResp extends QueryResult<Map<String, Object>> {
List<QueryColumn> columns = Lists.newArrayList();
String sql;

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.executor;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlUtils;
@@ -24,7 +24,7 @@ public class JdbcExecutor implements QueryExecutor {
}
@Override
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
public SemanticQueryResp execute(QueryStatement queryStatement) {
if (Strings.isEmpty(queryStatement.getSourceId())) {
log.warn("data base id is empty");
return null;
@@ -32,9 +32,10 @@ public class JdbcExecutor implements QueryExecutor {
log.info("query SQL: {}", queryStatement.getSql());
Database database = queryStatement.getSemanticModel().getDatabase();
log.info("database info:{}", database);
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
SqlUtils sqlUtils = this.sqlUtils.init(database);
sqlUtils.queryInternal(queryStatement.getSql(), queryResultWithColumns);
queryResultWithColumns.setSql(queryStatement.getSql());
return queryResultWithColumns;
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/QueryExecutor.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.executor;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/**
@@ -10,5 +10,5 @@ public interface QueryExecutor {
boolean accept(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(QueryStatement queryStatement);
SemanticQueryResp execute(QueryStatement queryStatement);
}

17
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.core.parser;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
@@ -26,7 +27,7 @@ import org.springframework.util.CollectionUtils;
@Primary
public class QueryParser {
public QueryStatement logicSql(QueryStatement queryStatement) throws Exception {
public QueryStatement parse(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
if (Objects.isNull(queryStatement.getParseSqlReq())) {
queryStatement.setParseSqlReq(new ParseSqlReq());
@@ -44,12 +45,16 @@ public class QueryParser {
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
queryStatement.getMetricReq());
if (!queryStatement.getParseSqlReq().getSql().isEmpty()) {
return parser(queryStatement.getParseSqlReq(), queryStatement);
queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement);
} else {
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
}
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
return parser(queryStatement);
if (Strings.isNullOrEmpty(queryStatement.getSql())
|| Strings.isNullOrEmpty(queryStatement.getSourceId())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
}
return queryStatement;
}
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/DetailQuery.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQuery.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.optimizer;
package com.tencent.supersonic.headless.core.planner;
import com.google.common.base.Strings;
import com.tencent.supersonic.headless.api.request.QueryStructReq;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/optimizer/QueryOptimizer.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryOptimizer.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.optimizer;
package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;

32
headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryPlanner.java Normal file
View File

@@ -0,0 +1,32 @@
package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class QueryPlanner {
public QueryExecutor plan(QueryStatement queryStatement) {
optimizer(queryStatement);
return route(queryStatement);
}
public void optimizer(QueryStatement queryStatement) {
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement.getQueryStructReq(), queryStatement);
}
}
public QueryExecutor route(QueryStatement queryStatement) {
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
if (queryExecutor.accept(queryStatement)) {
return queryExecutor;
}
}
return null;
}
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.optimizer.DetailQuery;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.planner.DetailQuery;
import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.SqlParser;
import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser;

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java
View File

@@ -5,7 +5,7 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.enums.DataType;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.JdbcDataSource;
import java.rmi.ServerException;
@@ -89,7 +89,7 @@ public class SqlUtils {
}
}
public void execute(String sql, QueryResultWithSchemaResp queryResultWithColumns) {
public void execute(String sql, SemanticQueryResp queryResultWithColumns) {
getResult(sql, queryResultWithColumns, jdbcTemplate());
}
@@ -110,11 +110,11 @@ public class SqlUtils {
return jdbcTemplate;
}
public void queryInternal(String sql, QueryResultWithSchemaResp queryResultWithColumns) {
public void queryInternal(String sql, SemanticQueryResp queryResultWithColumns) {
getResult(sql, queryResultWithColumns, jdbcTemplate());
}
private QueryResultWithSchemaResp getResult(String sql, QueryResultWithSchemaResp queryResultWithColumns,
private SemanticQueryResp getResult(String sql, SemanticQueryResp queryResultWithColumns,
JdbcTemplate jdbcTemplate) {
jdbcTemplate.query(sql, rs -> {
if (null == rs) {

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java
View File

@@ -19,7 +19,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
@@ -142,14 +142,14 @@ public class AuthCheckBaseAspect {
return false;
}
public QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns,
public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns,
List<Long> modelIds,
AuthorizedResourceResp authResource) {
addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
public QueryResultWithSchemaResp desensitizationData(QueryResultWithSchemaResp raw, Set<String> need2Apply) {
public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set<String> need2Apply) {
log.debug("start desensitizationData logic");
if (CollectionUtils.isEmpty(need2Apply)) {
log.info("user has all sensitiveRes");
@@ -171,7 +171,7 @@ public class AuthCheckBaseAspect {
return raw;
}
QueryResultWithSchemaResp queryResultWithColumns = raw;
SemanticQueryResp queryResultWithColumns = raw;
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
@@ -216,8 +216,8 @@ public class AuthCheckBaseAspect {
}
}
private QueryResultWithSchemaResp deepCopyResult(QueryResultWithSchemaResp raw) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
BeanUtils.copyProperties(raw, queryResultWithColumns);
List<QueryColumn> columns = new ArrayList<>();
@@ -241,7 +241,7 @@ public class AuthCheckBaseAspect {
return queryResultWithColumns;
}
public void addPromptInfoInfo(List<Long> modelIds, QueryResultWithSchemaResp queryResultWithColumns,
public void addPromptInfoInfo(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -8,11 +8,11 @@ import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import lombok.extern.slf4j.Slf4j;
@@ -51,13 +51,13 @@ public class DimValueAspect {
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapSqlEnable) {
log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap");
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
return queryResultWithColumns;
}
Object[] args = joinPoint.getArgs();
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelIds()));
String sql = queryS2SQLReq.getSql();
QuerySqlReq querySQLReq = (QuerySqlReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySQLReq.getModelIds()));
String sql = querySQLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
@@ -88,10 +88,10 @@ public class DimValueAspect {
log.info("filedNameToValueMap:{}", filedNameToValueMap);
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql);
queryS2SQLReq.setSql(sql);
querySQLReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
if (Objects.nonNull(queryResultWithColumns)) {
rewriteDimValue(queryResultWithColumns, techNameToBizName);
}
@@ -140,7 +140,7 @@ public class DimValueAspect {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
return queryResultWithColumns;
}
@@ -153,21 +153,21 @@ public class DimValueAspect {
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
if (Objects.nonNull(queryResultWithColumns)) {
rewriteDimValue(queryResultWithColumns, dimAndTechNameAndBizNamePair);
SemanticQueryResp semanticQueryResp = (SemanticQueryResp) joinPoint.proceed();
if (Objects.nonNull(semanticQueryResp)) {
rewriteDimValue(semanticQueryResp, dimAndTechNameAndBizNamePair);
}
return queryResultWithColumns;
return semanticQueryResp;
}
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns,
private void rewriteDimValue(SemanticQueryResp semanticQueryResp,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) {
if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) {
return;
}
log.debug("start rewriteDimValue for resultList");
for (Map<String, Object> line : queryResultWithColumns.getResultList()) {
for (Map<String, Object> line : semanticQueryResp.getResultList()) {
for (String bizName : line.keySet()) {
if (dimAndTechNameAndBizNamePair.containsKey(bizName) && Objects.nonNull(line.get(bizName))) {
String techName = line.get(bizName).toString();

55
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java
View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.headless.server.aspect;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -7,15 +9,22 @@ import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResource
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -31,16 +40,6 @@ import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@Order(1)
@@ -64,7 +63,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("s2SQL permission check!");
Object[] objects = joinPoint.getArgs();
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) objects[0];
QuerySqlReq querySQLReq = (QuerySqlReq) objects[0];
User user = (User) objects[1];
if (!permissionDataEnable) {
log.info("not to check s2SQL permission!");
@@ -73,7 +72,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
List<Long> modelIds = queryS2SQLReq.getModelIds();
List<Long> modelIds = querySQLReq.getModelIds();
//1. determine whether admin of the model
if (doModelAdmin(user, modelIds)) {
@@ -83,7 +82,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
// 2. determine whether the subject field is visible
doModelVisible(user, modelIds);
// 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user);
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySQLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
@@ -97,13 +96,13 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds);
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq);
doFilterCheckLogic(querySQLReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter
doRowPermission(queryS2SQLReq, authorizedResource);
doRowPermission(querySQLReq, authorizedResource);
// 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
@@ -115,14 +114,14 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
QueryResultWithSchemaResp queryResultAfterDesensitization =
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
private void doRowPermission(QueryS2SQLReq queryS2SQLReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QuerySqlReq querySQLReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -144,10 +143,10 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(queryS2SQLReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
queryS2SQLReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
String sql = SqlParserAddHelper.addWhere(querySQLReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql());
querySQLReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql());
}
} catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
@@ -155,14 +154,14 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
}
private void doFilterCheckLogic(QueryS2SQLReq queryS2SQLReq, Set<String> resAuthName,
private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2SQLReq);
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java
View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -100,7 +100,7 @@ public class StructDataAspect extends AuthCheckBaseAspect {
doRowPermission(queryStructReq, authorizedResource);
// 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) point.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
@@ -111,7 +111,7 @@ public class StructDataAspect extends AuthCheckBaseAspect {
// 6.if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
QueryResultWithSchemaResp queryResultAfterDesensitization =
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);

4
common/src/main/java/com/tencent/supersonic/common/util/cache/CacheUtils.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CacheManager.java vendored
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.common.util.cache;
package com.tencent.supersonic.headless.server.cache;
public interface CacheUtils {
public interface CacheManager {
Boolean put(String key, Object value);

6
common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CaffeineCacheManager.java vendored
View File

@@ -1,8 +1,9 @@
package com.tencent.supersonic.common.util.cache;
package com.tencent.supersonic.headless.server.cache;
import com.github.benmanes.caffeine.cache.Cache;
import com.google.common.base.Joiner;
import com.tencent.supersonic.headless.server.config.CacheCommonConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
@@ -11,8 +12,7 @@ import org.springframework.stereotype.Component;
@Component
@Slf4j
public class CaffeineCacheImpl implements CacheUtils {
public class CaffeineCacheManager implements CacheManager {
@Autowired
private CacheCommonConfig cacheCommonConfig;

78
headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/QueryCache.java vendored Normal file
View File

@@ -0,0 +1,78 @@
package com.tencent.supersonic.headless.server.cache;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.request.SemanticQueryReq;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class QueryCache {
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
@Autowired
private CacheManager cacheManager;
public Object query(SemanticQueryReq semanticQueryReq) {
String cacheKey = getCacheKey(semanticQueryReq);
handleGlobalCacheDisable(semanticQueryReq);
boolean isCache = isCache(semanticQueryReq);
if (isCache) {
Object result = cacheManager.get(cacheKey);
log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq);
return result;
}
return null;
}
public Boolean put(SemanticQueryReq semanticQueryReq, Object value) {
if (cacheEnable && Objects.nonNull(value)) {
String key = getCacheKey(semanticQueryReq);
CompletableFuture.supplyAsync(() -> cacheManager.put(key, value))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
log.info("add record to cache, key:{}", key);
return true;
}
return false;
}
public String getCacheKey(SemanticQueryReq semanticQueryReq) {
String commandMd5 = semanticQueryReq.generateCommandMd5();
String keyByModelIds = getKeyByModelIds(semanticQueryReq.getModelIds());
return cacheManager.generateCacheKey(keyByModelIds, commandMd5);
}
private void handleGlobalCacheDisable(SemanticQueryReq semanticQueryReq) {
if (!cacheEnable) {
Cache cacheInfo = new Cache();
cacheInfo.setCache(false);
semanticQueryReq.setCacheInfo(cacheInfo);
}
}
private String getKeyByModelIds(List<Long> modelIds) {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
private boolean isCache(SemanticQueryReq semanticQueryReq) {
if (!cacheEnable) {
return false;
}
if (semanticQueryReq.getCacheInfo() != null) {
return semanticQueryReq.getCacheInfo().getCache();
}
return false;
}
}

2
common/src/main/java/com/tencent/supersonic/common/util/cache/CacheCommonConfig.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CacheCommonConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.cache;
package com.tencent.supersonic.headless.server.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;

6
common/src/main/java/com/tencent/supersonic/common/util/cache/CaffeineCacheConfig.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CaffeineCacheConfig.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.common.util.cache;
package com.tencent.supersonic.headless.server.config;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
@@ -24,9 +24,7 @@ public class CaffeineCacheConfig {
public Cache<String, Object> caffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES)
// 初始的缓存空间大小
.initialCapacity(caffeineInitialCapacity)
// 缓存的最大条数
.maximumSize(caffeineMaximumSize)
.build();
}
@@ -35,9 +33,7 @@ public class CaffeineCacheConfig {
public Cache<Long, Object> searchCaffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(10000, TimeUnit.MINUTES)
// 初始的缓存空间大小
.initialCapacity(caffeineInitialCapacity)
// 缓存的最大条数
.maximumSize(caffeineMaximumSize)
.build();
}

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -56,13 +56,13 @@ import org.springframework.util.CollectionUtils;
@Slf4j
@Service
public class HeadlessSchemaManager {
public class SemanticSchemaManager {
@Autowired
private LoadingCache<String, SemanticModel> loadingCache;
private final Catalog catalog;
public HeadlessSchemaManager(Catalog catalog) {
public SemanticSchemaManager(Catalog catalog) {
this.catalog = catalog;
}
@@ -87,7 +87,7 @@ public class HeadlessSchemaManager {
semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource)
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource)
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap);
}
@@ -104,7 +104,6 @@ public class HeadlessSchemaManager {
return semanticModel;
}
//private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
public SemanticModel get(String rootPath) throws Exception {
rootPath = formatKey(rootPath);
SemanticModel schema = loadingCache.get(rootPath);
@@ -383,7 +382,7 @@ public class HeadlessSchemaManager {
@Override
public SemanticModel load(String key) {
log.info("load SemanticSchema [{}]", key);
return HeadlessSchemaManager.this.reload(key);
return SemanticSchemaManager.this.reload(key);
}
}
);

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import java.util.Map;
@@ -67,7 +67,7 @@ public class DatabaseController {
}
@PostMapping("/executeSql")
public QueryResultWithSchemaResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
@@ -75,18 +75,18 @@ public class DatabaseController {
}
@RequestMapping("/getDbNames/{id}")
public QueryResultWithSchemaResp getDbNames(@PathVariable("id") Long id) {
public SemanticQueryResp getDbNames(@PathVariable("id") Long id) {
return databaseService.getDbNames(id);
}
@RequestMapping("/getTables/{id}/{db}")
public QueryResultWithSchemaResp getTables(@PathVariable("id") Long id,
public SemanticQueryResp getTables(@PathVariable("id") Long id,
@PathVariable("db") String db) {
return databaseService.getTables(id, db);
}
@RequestMapping("/getColumns/{id}/{db}/{table}")
public QueryResultWithSchemaResp getColumns(@PathVariable("id") Long id,
public SemanticQueryResp getColumns(@PathVariable("id") Long id,
@PathVariable("db") String db,
@PathVariable("table") String table) {
return databaseService.getColumns(id, db, table);

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java
View File

@@ -12,16 +12,16 @@ import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.DownloadService;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import com.tencent.supersonic.headless.server.service.QueryService;
import java.util.HashSet;
import java.util.List;
@@ -45,19 +45,17 @@ public class QueryController {
@Autowired
private QueryService queryService;
@Autowired
private HeadlessQueryEngine headlessQueryEngine;
private SemantciQueryEngine semantciQueryEngine;
@Autowired
private DownloadService downloadService;
@PostMapping("/sql")
public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq,
public Object queryBySql(@RequestBody QuerySqlReq querySQLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
Object queryBySql = queryService.queryBySql(queryS2SQLReq, user);
log.info("queryBySql:{}", queryBySql);
return queryBySql;
return queryService.queryBySql(querySQLReq, user);
}
@PostMapping("/struct")
@@ -91,7 +89,7 @@ public class QueryController {
}
@PostMapping("/queryStatement")
public Object queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
public SemanticQueryResp queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
return queryService.queryByQueryStatement(queryStatement);
}
@@ -101,7 +99,7 @@ public class QueryController {
Set<Long> models = new HashSet<>();
models.add(Long.valueOf(parseSqlReq.getRootPath()));
queryStructCmd.setModelIds(models);
QueryStatement queryStatement = headlessQueryEngine.physicalSql(queryStructCmd, parseSqlReq);
QueryStatement queryStatement = semantciQueryEngine.physicalSql(queryStructCmd, parseSqlReq);
SqlParserResp sqlParserResp = new SqlParserResp();
BeanUtils.copyProperties(queryStatement, sqlParserResp);
return sqlParserResp;
@@ -130,7 +128,7 @@ public class QueryController {
}
@PostMapping("/queryDimValue")
public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
public SemanticQueryResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
@@ -147,9 +145,9 @@ public class QueryController {
QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryType.SQL.equals(queryTypeEnum)) {
QueryS2SQLReq queryS2SQLReq = JsonUtil.toObject(queryReqJson, QueryS2SQLReq.class);
ExplainSqlReq<QueryS2SQLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2SQLReq>builder()
.queryReq(queryS2SQLReq)
QuerySqlReq querySQLReq = JsonUtil.toObject(queryReqJson, QuerySqlReq.class);
ExplainSqlReq<QuerySqlReq> explainSqlReqNew = ExplainSqlReq.<QuerySqlReq>builder()
.queryReq(querySQLReq)
.queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user);
}

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import java.util.List;
import java.util.Map;
@@ -11,9 +11,9 @@ import java.util.Map;
public interface DatabaseService {
QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp);
SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp);
QueryResultWithSchemaResp executeSql(String sql, Long id, User user);
SemanticQueryResp executeSql(String sql, Long id, User user);
Map<String, List<DatabaseParameter>> getDatabaseParameters();
@@ -27,9 +27,9 @@ public interface DatabaseService {
DatabaseResp getDatabase(Long id);
QueryResultWithSchemaResp getDbNames(Long id);
SemanticQueryResp getDbNames(Long id);
QueryResultWithSchemaResp getTables(Long id, String db);
SemanticQueryResp getTables(Long id, String db);
QueryResultWithSchemaResp getColumns(Long id, String db, String table);
SemanticQueryResp getColumns(Long id, String db, String table);
}

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
View File

@@ -6,12 +6,12 @@ import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck;
@@ -20,18 +20,17 @@ import java.util.List;
public interface QueryService {
Object queryBySql(QueryS2SQLReq querySqlCmd, User user) throws Exception;
Object queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;
SemanticQueryResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user)
throws Exception;
SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception;
SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception;
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
Object queryByQueryStatement(QueryStatement queryStatement);
SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
@@ -39,6 +38,6 @@ public interface QueryService {
@ApiHeaderCheck
ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq,
HttpServletRequest request) throws Exception;
HttpServletRequest request) throws Exception;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemantciQueryEngine.java
View File

@@ -3,17 +3,17 @@ package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
public interface HeadlessQueryEngine {
public interface SemantciQueryEngine {
QueryStatement plan(QueryStatement queryStatement) throws Exception;
QueryExecutor route(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(QueryStatement queryStatement);
SemanticQueryResp execute(QueryStatement queryStatement);
QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception;

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -4,7 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.Database;
@@ -112,10 +112,10 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp executeSql(String sql, Long id, User user) {
public SemanticQueryResp executeSql(String sql, Long id, User user) {
DatabaseResp databaseResp = getDatabase(id);
if (databaseResp == null) {
return new QueryResultWithSchemaResp();
return new SemanticQueryResp();
}
List<String> admins = databaseResp.getAdmins();
List<String> viewers = databaseResp.getViewers();
@@ -132,7 +132,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp) {
public SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp) {
return queryWithColumns(sql, DatabaseConverter.convert(databaseResp));
}
@@ -143,8 +143,8 @@ public class DatabaseServiceImpl implements DatabaseService {
LinkedHashMap::putAll);
}
private QueryResultWithSchemaResp queryWithColumns(String sql, Database database) {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
private SemanticQueryResp queryWithColumns(String sql, Database database) {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
SqlUtils sqlUtils = this.sqlUtils.init(database);
log.info("query SQL: {}", sql);
sqlUtils.queryInternal(sql, queryResultWithColumns);
@@ -156,7 +156,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp getDbNames(Long id) {
public SemanticQueryResp getDbNames(Long id) {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl();
@@ -164,7 +164,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp getTables(Long id, String db) {
public SemanticQueryResp getTables(Long id, String db) {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl();
@@ -173,7 +173,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp getColumns(Long id, String db, String table) {
public SemanticQueryResp getColumns(Long id, String db, String table) {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl();

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -23,7 +23,7 @@ import com.tencent.supersonic.headless.api.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
@@ -266,8 +266,8 @@ public class DimensionServiceImpl implements DimensionService {
String sql = "select ai_talk." + dimensionReq.getBizName() + " from (" + sqlQuery
+ ") as ai_talk group by ai_talk." + dimensionReq.getBizName();
QueryResultWithSchemaResp queryResultWithSchemaResp = databaseService.executeSql(sql, database);
List<Map<String, Object>> resultList = queryResultWithSchemaResp.getResultList();
SemanticQueryResp semanticQueryResp = databaseService.executeSql(sql, database);
List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
List<String> valueList = new ArrayList<>();
for (Map<String, Object> stringObjectMap : resultList) {
String value = (String) stringObjectMap.get(dimensionReq.getBizName());

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -21,7 +21,7 @@ import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.utils.DataTransformUtils;
import com.tencent.supersonic.headless.server.pojo.DataDownload;
import com.tencent.supersonic.headless.server.service.DownloadService;
@@ -72,7 +72,7 @@ public class DownloadServiceImpl implements DownloadService {
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
File file = FileUtils.createTmpFile(fileName);
try {
QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
SemanticQueryResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
} catch (RuntimeException e) {
@@ -112,7 +112,7 @@ public class DownloadServiceImpl implements DownloadService {
for (MetricSchemaResp metric : metrics) {
try {
DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq);
QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
SemanticQueryResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
.head(dataDownload.getHeaders()).build();
@@ -140,9 +140,9 @@ public class DownloadServiceImpl implements DownloadService {
return data;
}
private List<List<String>> buildHeader(QueryResultWithSchemaResp queryResultWithSchemaResp) {
private List<List<String>> buildHeader(SemanticQueryResp semanticQueryResp) {
List<List<String>> header = Lists.newArrayList();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
for (QueryColumn column : semanticQueryResp.getColumns()) {
header.add(Lists.newArrayList(column.getName()));
}
return header;
@@ -163,11 +163,11 @@ public class DownloadServiceImpl implements DownloadService {
return headers;
}
private List<List<String>> buildData(QueryResultWithSchemaResp queryResultWithSchemaResp) {
private List<List<String>> buildData(SemanticQueryResp semanticQueryResp) {
List<List<String>> data = new ArrayList<>();
for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
for (Map<String, Object> row : semanticQueryResp.getResultList()) {
List<String> rowData = new ArrayList<>();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
for (QueryColumn column : semanticQueryResp.getColumns()) {
rowData.add(String.valueOf(row.get(column.getNameEn())));
}
data.add(rowData);
@@ -198,7 +198,7 @@ public class DownloadServiceImpl implements DownloadService {
return data;
}
private DataDownload buildDataDownload(QueryResultWithSchemaResp queryResult, DownloadStructReq downloadStructReq) {
private DataDownload buildDataDownload(SemanticQueryResp queryResult, DownloadStructReq downloadStructReq) {
List<QueryColumn> metricColumns = queryResult.getMetricColumns();
List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
if (downloadStructReq.isTransform() && !CollectionUtils.isEmpty(metricColumns)) {

268
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -13,20 +14,17 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.AppDetailResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
@@ -36,28 +34,27 @@ import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.planner.QueryPlanner;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.server.annotation.StructDataPermission;
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.server.cache.CacheManager;
import com.tencent.supersonic.headless.server.cache.QueryCache;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.service.AppService;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -65,17 +62,24 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class QueryServiceImpl implements QueryService {
protected final com.google.common.cache.Cache<String, List<ItemUseResp>> itemUseCache =
protected final Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
private final StatUtils statUtils;
private final CacheUtils cacheUtils;
private final CacheManager cacheManager;
private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter;
private final Catalog catalog;
@@ -84,88 +88,109 @@ public class QueryServiceImpl implements QueryService {
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
private final HeadlessQueryEngine headlessQueryEngine;
private final QueryCache queryCache;
private final SemantciQueryEngine semantciQueryEngine;
private final SemanticSchemaManager semanticSchemaManager;
private final QueryParser queryParser;
private final QueryPlanner queryPlanner;
public QueryServiceImpl(
StatUtils statUtils,
CacheUtils cacheUtils,
CacheManager cacheManager,
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
@Lazy HeadlessQueryEngine headlessQueryEngine,
@Lazy SemantciQueryEngine semantciQueryEngine,
Catalog catalog,
AppService appService) {
AppService appService,
QueryCache queryCache,
SemanticSchemaManager semanticSchemaManager,
QueryParser queryParser,
QueryPlanner queryPlanner) {
this.statUtils = statUtils;
this.cacheUtils = cacheUtils;
this.cacheManager = cacheManager;
this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter;
this.headlessQueryEngine = headlessQueryEngine;
this.semantciQueryEngine = semantciQueryEngine;
this.catalog = catalog;
this.appService = appService;
this.queryCache = queryCache;
this.semanticSchemaManager = semanticSchemaManager;
this.queryParser = queryParser;
this.queryPlanner = queryPlanner;
}
@Override
@S2SQLDataPermission
@SneakyThrows
public Object queryBySql(QueryS2SQLReq queryS2SQLReq, User user) {
statUtils.initStatInfo(queryS2SQLReq, user);
public Object queryBySql(QuerySqlReq querySQLReq, User user) {
statUtils.initStatInfo(querySQLReq, user);
QueryStatement queryStatement = new QueryStatement();
try {
queryStatement = convertToQueryStatement(queryS2SQLReq, user);
queryStatement = convertToQueryStatement(querySQLReq, user);
} catch (Exception e) {
log.info("convertToQueryStatement has a exception:", e);
}
log.info("queryStatement:{}", queryStatement);
QueryResultWithSchemaResp results = headlessQueryEngine.execute(queryStatement);
SemanticQueryResp results = semantciQueryEngine.execute(queryStatement);
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return results;
}
public Object queryByQueryStatement(QueryStatement queryStatement) {
return headlessQueryEngine.execute(queryStatement);
public SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement) {
return semantciQueryEngine.execute(queryStatement);
}
private QueryStatement convertToQueryStatement(QueryS2SQLReq queryS2SQLReq, User user) throws Exception {
private QueryStatement convertToQueryStatement(QuerySqlReq querySQLReq, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(queryS2SQLReq.getModelIds());
filter.setModelIds(querySQLReq.getModelIds());
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(queryS2SQLReq, modelSchemaResps);
queryStatement.setModelIds(queryS2SQLReq.getModelIds());
QueryStatement queryStatement = queryReqConverter.convert(querySQLReq, modelSchemaResps);
queryStatement.setModelIds(querySQLReq.getModelIds());
return queryStatement;
}
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns = null;
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
SemanticQueryResp semanticQueryResp = null;
log.info("[queryStructReq:{}]", queryStructReq);
try {
//1.initStatInfo
statUtils.initStatInfo(queryStructReq, user);
String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructReq.getModelIds()),
queryStructReq.generateCommandMd5());
handleGlobalCacheDisable(queryStructReq);
boolean isCache = isCache(queryStructReq);
if (isCache) {
queryResultWithColumns = queryByCache(cacheKey, queryStructReq);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
}
//2.query from cache
Object query = queryCache.query(queryStructReq);
if (Objects.nonNull(query)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return (SemanticQueryResp) query;
}
StatUtils.get().setUseResultCache(false);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement = headlessQueryEngine.plan(queryStatement);
QueryExecutor queryExecutor = headlessQueryEngine.route(queryStatement);
//3 parse
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = queryParser.parse(queryStatement);
//4 plan
QueryExecutor queryExecutor = queryPlanner.plan(queryStatement);
//5 execute
if (queryExecutor != null) {
queryResultWithColumns = headlessQueryEngine.execute(queryStatement);
if (isCache) {
// if queryResultWithColumns is not null, update cache data
queryUtils.cacheResultLogic(cacheKey, queryResultWithColumns);
semanticQueryResp = queryExecutor.execute(queryStatement);
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
}
}
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
//6 reset cache and set stateInfo
Boolean putCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp);
if (putCacheSuccess) {
// if semanticQueryResp is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq));
}
if (Objects.nonNull(semanticQueryResp)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
}
return semanticQueryResp;
} catch (Exception e) {
log.warn("exception in queryByStruct, e: ", e);
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
@@ -173,38 +198,62 @@ public class QueryServiceImpl implements QueryService {
}
}
private QueryStatement buildQueryStatement(QueryStructReq queryStructReq) throws Exception {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
SemanticModel semanticModel = semanticSchemaManager.get(queryStructReq.getModelIdStr());
queryStatement.setSemanticModel(semanticModel);
return queryStatement;
}
@Override
@StructDataPermission
@SneakyThrows
public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) {
public SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) {
return queryByStruct(queryStructReq, user);
}
@Override
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user)
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user)
throws Exception {
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
String cacheKey = cacheUtils.generateCacheKey(
getKeyByModelIds(queryMultiStructReq.getQueryStructReqs().get(0).getModelIds()),
queryMultiStructReq.generateCommandMd5());
boolean isCache = isCache(queryMultiStructReq);
QueryResultWithSchemaResp queryResultWithColumns;
if (isCache) {
queryResultWithColumns = queryByCache(cacheKey, queryMultiStructReq);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
}
}
log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq);
try {
QueryStatement sqlParser = getQueryStatementByMultiStruct(queryMultiStructReq);
queryResultWithColumns = headlessQueryEngine.execute(sqlParser);
if (queryResultWithColumns != null) {
//1.initStatInfo
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
//2.query from cache
Object query = queryCache.query(queryMultiStructReq);
if (Objects.nonNull(query)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructReq);
return (SemanticQueryResp) query;
}
return queryResultWithColumns;
StatUtils.get().setUseResultCache(false);
log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq);
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = queryParser.parse(queryStatement);
queryPlanner.optimizer(queryStatement);
sqlParsers.add(queryStatement);
}
log.info("multi sqlParser:{}", sqlParsers);
QueryStatement queryStatement = queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
QueryExecutor executor = queryPlanner.route(queryStatement);
SemanticQueryResp semanticQueryResp = null;
if (executor != null) {
semanticQueryResp = executor.execute(queryStatement);
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
}
}
if (Objects.nonNull(semanticQueryResp)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
}
return semanticQueryResp;
} catch (Exception e) {
log.warn("exception in queryByMultiStruct, e: ", e);
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
@@ -212,13 +261,11 @@ public class QueryServiceImpl implements QueryService {
}
}
private QueryStatement getQueryStatementByMultiStruct(QueryMultiStructReq queryMultiStructReq) throws Exception {
private QueryStatement buildQueryStatement(QueryMultiStructReq queryMultiStructReq) throws Exception {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement = headlessQueryEngine.plan(queryStatement);
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = semantciQueryEngine.plan(queryStatement);
queryUtils.checkSqlParse(queryStatement);
sqlParsers.add(queryStatement);
}
@@ -228,17 +275,9 @@ public class QueryServiceImpl implements QueryService {
@Override
@SneakyThrows
public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
QueryS2SQLReq queryS2SQLReq = generateDimValueQuerySql(queryDimValueReq);
return (QueryResultWithSchemaResp) queryBySql(queryS2SQLReq, user);
}
private void handleGlobalCacheDisable(QueryStructReq queryStructReq) {
if (!cacheEnable) {
Cache cacheInfo = new Cache();
cacheInfo.setCache(false);
queryStructReq.setCacheInfo(cacheInfo);
}
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
QuerySqlReq querySQLReq = buildQuerySqlReq(queryDimValueReq);
return (SemanticQueryResp) queryBySql(querySQLReq, user);
}
@Override
@@ -259,20 +298,18 @@ public class QueryServiceImpl implements QueryService {
QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum();
T queryReq = explainSqlReq.getQueryReq();
if (QueryType.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2SQLReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryS2SQLReq) queryReq, user);
if (QueryType.SQL.equals(queryTypeEnum) && queryReq instanceof QuerySqlReq) {
QueryStatement queryStatement = convertToQueryStatement((QuerySqlReq) queryReq, user);
return getExplainResp(queryStatement);
}
if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq((QueryStructReq) queryReq);
queryStatement.setIsS2SQL(false);
queryStatement = headlessQueryEngine.plan(queryStatement);
QueryStatement queryStatement = buildQueryStatement((QueryStructReq) queryReq);
queryStatement = semantciQueryEngine.plan(queryStatement);
return getExplainResp(queryStatement);
}
if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryMultiStructReq) {
QueryMultiStructReq queryMultiStructReq = (QueryMultiStructReq) queryReq;
QueryStatement queryStatement = getQueryStatementByMultiStruct(queryMultiStructReq);
QueryStatement queryStatement = buildQueryStatement(queryMultiStructReq);
return getExplainResp(queryStatement);
}
@@ -310,11 +347,11 @@ public class QueryServiceImpl implements QueryService {
dimensionResps = catalog.getDimensions(dimensionFilter);
}
QueryStructReq queryStructReq = buildQueryStructReq(dimensionResps, metricResp, dateConf, limit);
QueryResultWithSchemaResp queryResultWithSchemaResp =
SemanticQueryResp semanticQueryResp =
queryByStruct(queryStructReq, User.getAppUser(appId));
SingleItemQueryResult apiQuerySingleResult = new SingleItemQueryResult();
apiQuerySingleResult.setItem(item);
apiQuerySingleResult.setResult(queryResultWithSchemaResp);
apiQuerySingleResult.setResult(semanticQueryResp);
return apiQuerySingleResult;
}
@@ -366,21 +403,6 @@ public class QueryServiceImpl implements QueryService {
return ExplainResp.builder().sql(sql).build();
}
public QueryStatement parseMetricReq(MetricQueryReq metricReq) throws Exception {
QueryStructReq queryStructReq = new QueryStructReq();
return headlessQueryEngine.physicalSql(queryStructReq, metricReq);
}
private boolean isCache(QueryStructReq queryStructReq) {
if (!cacheEnable) {
return false;
}
if (queryStructReq.getCacheInfo() != null) {
return queryStructReq.getCacheInfo().getCache();
}
return false;
}
private boolean isCache(QueryMultiStructReq queryStructReq) {
if (!cacheEnable) {
return false;
@@ -392,19 +414,19 @@ public class QueryServiceImpl implements QueryService {
return false;
}
private QueryResultWithSchemaResp queryByCache(String key, Object queryCmd) {
private SemanticQueryResp queryByCache(String key, Object queryCmd) {
Object resultObject = cacheUtils.get(key);
Object resultObject = cacheManager.get(key);
if (Objects.nonNull(resultObject)) {
log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
statUtils.updateResultCacheKey(key);
return (QueryResultWithSchemaResp) resultObject;
return (SemanticQueryResp) resultObject;
}
return null;
}
private QueryS2SQLReq generateDimValueQuerySql(QueryDimValueReq queryDimValueReq) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
QuerySqlReq querySQLReq = new QuerySqlReq();
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(),
queryDimValueReq.getModelId());
@@ -416,9 +438,9 @@ public class QueryServiceImpl implements QueryService {
queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate());
}
queryS2SQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
queryS2SQLReq.setSql(sql);
return queryS2SQLReq;
querySQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySQLReq.setSql(sql);
return querySQLReq;
}
private String getKeyByModelIds(List<Long> modelIds) {

30
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemantciQueryEngineImpl.java
View File

@@ -3,16 +3,15 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -20,24 +19,21 @@ import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
public class SemantciQueryEngineImpl implements SemantciQueryEngine {
private final QueryParser queryParser;
private final QueryUtils queryUtils;
private final QueryStructUtils queryStructUtils;
private final HeadlessSchemaManager headlessSchemaManager;
private final SemanticSchemaManager semanticSchemaManager;
public HeadlessQueryEngineImpl(QueryParser queryParser,
QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager,
QueryStructUtils queryStructUtils) {
public SemantciQueryEngineImpl(QueryParser queryParser,
QueryUtils queryUtils, SemanticSchemaManager semanticSchemaManager) {
this.queryParser = queryParser;
this.queryUtils = queryUtils;
this.headlessSchemaManager = headlessSchemaManager;
this.queryStructUtils = queryStructUtils;
this.semanticSchemaManager = semanticSchemaManager;
}
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
QueryResultWithSchemaResp queryResultWithColumns = null;
public SemanticQueryResp execute(QueryStatement queryStatement) {
SemanticQueryResp queryResultWithColumns = null;
QueryExecutor queryExecutor = route(queryStatement);
if (queryExecutor != null) {
queryResultWithColumns = queryExecutor.execute(queryStatement);
@@ -52,7 +48,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
public QueryStatement plan(QueryStatement queryStatement) throws Exception {
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setSemanticModel(getSemanticModel(queryStatement));
queryStatement = queryParser.logicSql(queryStatement);
queryStatement = queryParser.parse(queryStatement);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
log.info("queryStatement:{}", queryStatement);
@@ -97,7 +93,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
return headlessSchemaManager.get(queryStructReq.getModelIdStr());
return semanticSchemaManager.get(queryStructReq.getModelIdStr());
}
}

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -16,7 +16,7 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
@@ -29,7 +29,7 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -56,7 +56,7 @@ public class QueryReqConverter {
private Boolean limitWrapper;
@Autowired
private HeadlessQueryEngine headlessQueryEngine;
private SemantciQueryEngine semantciQueryEngine;
@Autowired
private QueryStructUtils queryStructUtils;
@@ -66,7 +66,7 @@ public class QueryReqConverter {
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryS2SQLReq queryS2SQLReq,
public QueryStatement convert(QuerySqlReq querySQLReq,
List<ModelSchemaResp> modelSchemaResps) throws Exception {
if (CollectionUtils.isEmpty(modelSchemaResps)) {
@@ -75,18 +75,18 @@ public class QueryReqConverter {
Map<Long, ModelSchemaResp> modelSchemaRespMap = modelSchemaResps.stream()
.collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp));
//1.convert name to bizName
convertNameToBizName(queryS2SQLReq, modelSchemaResps);
convertNameToBizName(querySQLReq, modelSchemaResps);
//2.functionName corrector
functionNameCorrector(queryS2SQLReq);
functionNameCorrector(querySQLReq);
//3.correct tableName
correctTableName(queryS2SQLReq);
correctTableName(querySQLReq);
String tableName = SqlParserSelectHelper.getTableName(queryS2SQLReq.getSql());
String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql());
if (StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
//4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
List<String> metrics = getMetrics(modelSchemaResps, allFields);
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
@@ -100,7 +100,7 @@ public class QueryReqConverter {
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>());
for (Long modelId : queryS2SQLReq.getModelIds()) {
for (Long modelId : querySQLReq.getModelIds()) {
ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId);
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName()));
}
@@ -109,27 +109,27 @@ public class QueryReqConverter {
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
Collectors.toList()));
}
AggOption aggOption = getAggOption(queryS2SQLReq);
AggOption aggOption = getAggOption(querySQLReq);
metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(queryS2SQLReq, result);
BeanUtils.copyProperties(querySQLReq, result);
result.setRootPath(queryS2SQLReq.getModelIdStr());
result.setRootPath(querySQLReq.getModelIdStr());
result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(queryS2SQLReq.getModelIds().get(0));
DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0));
if (!sqlGenerateUtils.isSupportWith(EngineType.valueOf(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);
}
//5. do deriveMetric
generateDerivedMetric(queryS2SQLReq.getModelIds(), modelSchemaResps, result);
generateDerivedMetric(querySQLReq.getModelIds(), modelSchemaResps, result);
//6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds()));
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(querySQLReq.getModelIds()));
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryStatement queryStatement = new QueryStatement();
@@ -137,14 +137,14 @@ public class QueryReqConverter {
queryStatement.setParseSqlReq(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setModelIds(queryS2SQLReq.getModelIds());
queryStatement = headlessQueryEngine.plan(queryStatement);
queryStatement.setModelIds(querySQLReq.getModelIds());
queryStatement = semantciQueryEngine.plan(queryStatement);
queryStatement.setSql(limitWrapper ? String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql())
: queryStatement.getSql());
return queryStatement;
}
private AggOption getAggOption(QueryS2SQLReq databaseReq) {
private AggOption getAggOption(QuerySqlReq databaseReq) {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql();
@@ -156,7 +156,7 @@ public class QueryReqConverter {
return AggOption.DEFAULT;
}
private void convertNameToBizName(QueryS2SQLReq databaseReq, List<ModelSchemaResp> modelSchemaResps) {
private void convertNameToBizName(QuerySqlReq databaseReq, List<ModelSchemaResp> modelSchemaResps) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
@@ -186,7 +186,7 @@ public class QueryReqConverter {
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
}
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
private void functionNameCorrector(QuerySqlReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
@@ -231,7 +231,7 @@ public class QueryReqConverter {
return elements.stream();
}
public void correctTableName(QueryS2SQLReq databaseReq) {
public void correctTableName(QuerySqlReq databaseReq) {
String sql = databaseReq.getSql();
for (Long modelId : databaseReq.getModelIds()) {
sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId);

39
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -1,5 +1,10 @@
package com.tencent.supersonic.headless.server.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
@@ -14,7 +19,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
@@ -24,13 +29,6 @@ import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -44,11 +42,12 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -130,8 +129,8 @@ public class QueryStructUtils {
return resNameEnSet;
}
public Set<String> getResName(QueryS2SQLReq queryS2SQLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql())
public Set<String> getResName(QuerySqlReq querySQLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(querySQLReq.getSql())
.stream().collect(Collectors.toSet());
return resNameSet;
}
@@ -141,11 +140,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq, User user) {
Set<String> resNameSet = getResName(queryS2SQLReq);
public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySQLReq, User user) {
Set<String> resNameSet = getResName(querySQLReq);
Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -176,8 +175,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getFilterResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq) {
String sql = queryS2SQLReq.getSql();
public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySQLReq) {
String sql = querySQLReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}

52
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -1,40 +1,37 @@
package com.tencent.supersonic.headless.server.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.enums.SemanticType;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.cache.CacheManager;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import javax.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -52,15 +49,15 @@ public class QueryUtils {
@Value("${query.optimizer.enable:true}")
private Boolean optimizeEnable;
private final CacheUtils cacheUtils;
private final CacheManager cacheManager;
private final StatUtils statUtils;
private final Catalog catalog;
public QueryUtils(
CacheUtils cacheUtils, StatUtils statUtils, Catalog catalog) {
CacheManager cacheManager, StatUtils statUtils, Catalog catalog) {
this.cacheUtils = cacheUtils;
this.cacheManager = cacheManager;
this.statUtils = statUtils;
this.catalog = catalog;
}
@@ -74,7 +71,7 @@ public class QueryUtils {
}
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, List<Long> modelIds) {
public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns, List<Long> modelIds) {
MetaFilter metaFilter = new MetaFilter(modelIds);
List<MetricResp> metricDescList = catalog.getMetrics(metaFilter);
List<DimensionResp> dimensionDescList = catalog.getDimensions(metaFilter);
@@ -125,7 +122,7 @@ public class QueryUtils {
});
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns,
public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns,
QueryMultiStructReq queryMultiStructCmd) {
List<Aggregator> aggregators = queryMultiStructCmd.getQueryStructReqs().stream()
.flatMap(queryStructCmd -> queryStructCmd.getAggregators().stream())
@@ -244,21 +241,6 @@ public class QueryUtils {
return sqlParser;
}
public void cacheResultLogic(String key, QueryResultWithSchemaResp queryResultWithColumns) {
if (cacheEnable && Objects.nonNull(queryResultWithColumns) && !CollectionUtils.isEmpty(
queryResultWithColumns.getResultList())) {
QueryResultWithSchemaResp finalQueryResultWithColumns = queryResultWithColumns;
CompletableFuture.supplyAsync(() -> cacheUtils.put(key, finalQueryResultWithColumns))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
statUtils.updateResultCacheKey(key);
log.info("add record to cache, key:{}", key);
}
}
public Boolean enableOptimize() {
return optimizeEnable;
}

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.headless.api.enums.QueryTypeBack;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
@@ -85,11 +85,11 @@ public class StatUtils {
return true;
}
public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) {
public void initStatInfo(QuerySqlReq querySQLReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
queryStatInfo.setModelId(queryS2SQLReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelIds().get(0));
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
queryStatInfo.setModelId(querySQLReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySQLReq.getModelIds().get(0));
List<String> dimensions = new ArrayList<>();
List<String> metrics = new ArrayList<>();
@@ -101,12 +101,12 @@ public class StatUtils {
String userName = getUserName(facadeUser);
try {
queryStatInfo.setTraceId("")
.setModelId(queryS2SQLReq.getModelIds().get(0))
.setModelId(querySQLReq.getModelIds().get(0))
.setUser(userName)
.setQueryType(QueryType.SQL.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQuerySqlCmd(queryS2SQLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2SQLReq.toString()))
.setQuerySqlCmd(querySQLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(querySQLReq.toString()))
.setStartTime(System.currentTimeMillis())
.setUseResultCache(true)
.setUseSqlCache(true)

12
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
@@ -124,7 +124,7 @@ class HeadlessParserServiceTest {
SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build();
HeadlessSchemaManager.update(semanticSchema, HeadlessSchemaManager.getDatasource(datasource));
SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("page");
@@ -133,8 +133,8 @@ class HeadlessParserServiceTest {
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
HeadlessSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
MetricYamlTpl metric1 = new MetricYamlTpl();
metric1.setName("pv");
@@ -240,7 +240,7 @@ class HeadlessParserServiceTest {
identifies.add(identify);
datasource.setIdentifiers(identifies);
semanticSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource));
semanticSchema.getDatasource().put("user_department", SemanticSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department");
@@ -250,6 +250,6 @@ class HeadlessParserServiceTest {
dimensionYamlTpls.add(dimension1);
semanticSchema.getDimension()
.put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
.put("user_department", SemanticSchemaManager.getDimensions(dimensionYamlTpls));
}
}

21
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java
View File

@@ -1,5 +1,8 @@
package com.tencent.supersonic.headless.server.service;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import com.alibaba.excel.util.FileUtils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -11,20 +14,16 @@ import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.impl.DownloadServiceImpl;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.io.File;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
class DownloadServiceImplTest {
@@ -98,16 +97,16 @@ class DownloadServiceImplTest {
return dateConf;
}
private QueryResultWithSchemaResp mockQueryResult() {
QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp();
private SemanticQueryResp mockQueryResult() {
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
List<Map<String, Object>> resultList = Lists.newArrayList();
resultList.add(createMap("2023-10-11", "tom", "hr", "1"));
resultList.add(createMap("2023-10-12", "alice", "sales", "2"));
resultList.add(createMap("2023-10-13", "jack", "sales", "3"));
resultList.add(createMap("2023-10-14", "luck", "market", "4"));
resultList.add(createMap("2023-10-15", "tom", "hr", "5"));
queryResultWithSchemaResp.setResultList(resultList);
return queryResultWithSchemaResp;
semanticQueryResp.setResultList(resultList);
return semanticQueryResp;
}
private static Map<String, Object> createMap(String sysImpDate, String d1, String d2, String m1) {