topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 22:25:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Headless has preliminarily completed the abstraction of QueryCache, QueryParser, QueryPlanner, and QueryExecutor. (#651)

Browse Source
This commit is contained in:
lexluo09
2024-01-18 22:39:58 +08:00
committed by GitHub
parent b019f4d9bb
commit 3e77fc3069
56 changed files with 699 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/AuthCheckBaseAspect.java
View File

@@ -19,7 +19,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
@@ -142,14 +142,14 @@ public class AuthCheckBaseAspect {
return false;
}
public QueryResultWithSchemaResp getQueryResultWithColumns(QueryResultWithSchemaResp resultWithColumns,
public SemanticQueryResp getQueryResultWithColumns(SemanticQueryResp resultWithColumns,
List<Long> modelIds,
AuthorizedResourceResp authResource) {
addPromptInfoInfo(modelIds, resultWithColumns, authResource, Sets.newHashSet());
return resultWithColumns;
}
public QueryResultWithSchemaResp desensitizationData(QueryResultWithSchemaResp raw, Set<String> need2Apply) {
public SemanticQueryResp desensitizationData(SemanticQueryResp raw, Set<String> need2Apply) {
log.debug("start desensitizationData logic");
if (CollectionUtils.isEmpty(need2Apply)) {
log.info("user has all sensitiveRes");
@@ -171,7 +171,7 @@ public class AuthCheckBaseAspect {
return raw;
}
QueryResultWithSchemaResp queryResultWithColumns = raw;
SemanticQueryResp queryResultWithColumns = raw;
try {
queryResultWithColumns = deepCopyResult(raw);
} catch (Exception e) {
@@ -216,8 +216,8 @@ public class AuthCheckBaseAspect {
}
}
private QueryResultWithSchemaResp deepCopyResult(QueryResultWithSchemaResp raw) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
private SemanticQueryResp deepCopyResult(SemanticQueryResp raw) throws Exception {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
BeanUtils.copyProperties(raw, queryResultWithColumns);
List<QueryColumn> columns = new ArrayList<>();
@@ -241,7 +241,7 @@ public class AuthCheckBaseAspect {
return queryResultWithColumns;
}
public void addPromptInfoInfo(List<Long> modelIds, QueryResultWithSchemaResp queryResultWithColumns,
public void addPromptInfoInfo(List<Long> modelIds, SemanticQueryResp queryResultWithColumns,
AuthorizedResourceResp authorizedResource, Set<String> need2Apply) {
List<DimensionFilter> filters = authorizedResource.getFilters();
if (CollectionUtils.isEmpty(need2Apply) && CollectionUtils.isEmpty(filters)) {

32
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -8,11 +8,11 @@ import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.jsqlparser.FieldExpression;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import lombok.extern.slf4j.Slf4j;
@@ -51,13 +51,13 @@ public class DimValueAspect {
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
if (!dimensionValueMapSqlEnable) {
log.debug("sql dimensionValueMapEnable is false, skip dimensionValueMap");
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
return queryResultWithColumns;
}
Object[] args = joinPoint.getArgs();
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(queryS2SQLReq.getModelIds()));
String sql = queryS2SQLReq.getSql();
QuerySqlReq querySQLReq = (QuerySqlReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySQLReq.getModelIds()));
String sql = querySQLReq.getSql();
log.info("correctorSql before replacing:{}", sql);
// if dimensionvalue is alias,consider the true dimensionvalue.
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
@@ -88,10 +88,10 @@ public class DimValueAspect {
log.info("filedNameToValueMap:{}", filedNameToValueMap);
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql);
queryS2SQLReq.setSql(sql);
querySQLReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
if (Objects.nonNull(queryResultWithColumns)) {
rewriteDimValue(queryResultWithColumns, techNameToBizName);
}
@@ -140,7 +140,7 @@ public class DimValueAspect {
if (!dimensionValueMapEnable) {
log.debug("dimensionValueMapEnable is false, skip dimensionValueMap");
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
return queryResultWithColumns;
}
@@ -153,21 +153,21 @@ public class DimValueAspect {
rewriteFilter(queryStructReq.getDimensionFilters(), dimAndAliasAndTechNamePair);
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
if (Objects.nonNull(queryResultWithColumns)) {
rewriteDimValue(queryResultWithColumns, dimAndTechNameAndBizNamePair);
SemanticQueryResp semanticQueryResp = (SemanticQueryResp) joinPoint.proceed();
if (Objects.nonNull(semanticQueryResp)) {
rewriteDimValue(semanticQueryResp, dimAndTechNameAndBizNamePair);
}
return queryResultWithColumns;
return semanticQueryResp;
}
private void rewriteDimValue(QueryResultWithSchemaResp queryResultWithColumns,
private void rewriteDimValue(SemanticQueryResp semanticQueryResp,
Map<String, Map<String, String>> dimAndTechNameAndBizNamePair) {
if (!selectDimValueMap(queryResultWithColumns.getColumns(), dimAndTechNameAndBizNamePair)) {
if (!selectDimValueMap(semanticQueryResp.getColumns(), dimAndTechNameAndBizNamePair)) {
return;
}
log.debug("start rewriteDimValue for resultList");
for (Map<String, Object> line : queryResultWithColumns.getResultList()) {
for (Map<String, Object> line : semanticQueryResp.getResultList()) {
for (String bizName : line.keySet()) {
if (dimAndTechNameAndBizNamePair.containsKey(bizName) && Objects.nonNull(line.get(bizName))) {
String techName = line.get(bizName).toString();

55
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2SQLDataAspect.java
View File

@@ -1,5 +1,7 @@
package com.tencent.supersonic.headless.server.aspect;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -7,15 +9,22 @@ import com.tencent.supersonic.auth.api.authorization.response.AuthorizedResource
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserAddHelper;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.ModelFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
@@ -31,16 +40,6 @@ import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.MINUS;
@Component
@Aspect
@Order(1)
@@ -64,7 +63,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
log.info("s2SQL permission check!");
Object[] objects = joinPoint.getArgs();
QueryS2SQLReq queryS2SQLReq = (QueryS2SQLReq) objects[0];
QuerySqlReq querySQLReq = (QuerySqlReq) objects[0];
User user = (User) objects[1];
if (!permissionDataEnable) {
log.info("not to check s2SQL permission!");
@@ -73,7 +72,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
if (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) {
throw new RuntimeException("please provide user information");
}
List<Long> modelIds = queryS2SQLReq.getModelIds();
List<Long> modelIds = querySQLReq.getModelIds();
//1. determine whether admin of the model
if (doModelAdmin(user, modelIds)) {
@@ -83,7 +82,7 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
// 2. determine whether the subject field is visible
doModelVisible(user, modelIds);
// 3. fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(queryS2SQLReq, user);
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySQLReq, user);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
@@ -97,13 +96,13 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds);
// 4.if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(queryS2SQLReq, resAuthSet, sensitiveResReq);
doFilterCheckLogic(querySQLReq, resAuthSet, sensitiveResReq);
// 5.row permission pre-filter
doRowPermission(queryS2SQLReq, authorizedResource);
doRowPermission(querySQLReq, authorizedResource);
// 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) joinPoint.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
@@ -115,14 +114,14 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
log.info("need2Apply:{},sensitiveResReq:{},resAuthSet:{}", need2Apply, sensitiveResReq, resAuthSet);
QueryResultWithSchemaResp queryResultAfterDesensitization =
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);
return queryResultAfterDesensitization;
}
private void doRowPermission(QueryS2SQLReq queryS2SQLReq, AuthorizedResourceResp authorizedResource) {
private void doRowPermission(QuerySqlReq querySQLReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>();
@@ -144,10 +143,10 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(queryS2SQLReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
queryS2SQLReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", queryS2SQLReq.getSql());
String sql = SqlParserAddHelper.addWhere(querySQLReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql());
querySQLReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql());
}
} catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString());
@@ -155,14 +154,14 @@ public class S2SQLDataAspect extends AuthCheckBaseAspect {
}
private void doFilterCheckLogic(QueryS2SQLReq queryS2SQLReq, Set<String> resAuthName,
private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set<String> resAuthName,
Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(queryS2SQLReq);
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq);
Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/StructDataAspect.java
View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.DimensionService;
@@ -100,7 +100,7 @@ public class StructDataAspect extends AuthCheckBaseAspect {
doRowPermission(queryStructReq, authorizedResource);
// 6.proceed
QueryResultWithSchemaResp queryResultWithColumns = (QueryResultWithSchemaResp) point.proceed();
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) point.proceed();
if (CollectionUtils.isEmpty(sensitiveResReq) || allSensitiveResReqIsOk(sensitiveResReq, resAuthSet)) {
// if sensitiveRes is empty
@@ -111,7 +111,7 @@ public class StructDataAspect extends AuthCheckBaseAspect {
// 6.if the column has no permission, hit *
Set<String> need2Apply = sensitiveResReq.stream().filter(req -> !resAuthSet.contains(req))
.collect(Collectors.toSet());
QueryResultWithSchemaResp queryResultAfterDesensitization =
SemanticQueryResp queryResultAfterDesensitization =
desensitizationData(queryResultWithColumns, need2Apply);
addPromptInfoInfo(modelIds, queryResultAfterDesensitization, authorizedResource, need2Apply);

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CacheManager.java vendored Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.headless.server.cache;
public interface CacheManager {
Boolean put(String key, Object value);
Object get(String key);
String generateCacheKey(String prefix, String body);
Boolean removeCache(String key);
}

52
headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/CaffeineCacheManager.java vendored Normal file
View File

@@ -0,0 +1,52 @@
package com.tencent.supersonic.headless.server.cache;
import com.github.benmanes.caffeine.cache.Cache;
import com.google.common.base.Joiner;
import com.tencent.supersonic.headless.server.config.CacheCommonConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class CaffeineCacheManager implements CacheManager {
@Autowired
private CacheCommonConfig cacheCommonConfig;
@Autowired
@Qualifier("caffeineCache")
private Cache<String, Object> caffeineCache;
@Override
public Boolean put(String key, Object value) {
log.info("[put caffeineCache] key:{}, value:{}", key, value);
caffeineCache.put(key, value);
return true;
}
@Override
public Object get(String key) {
Object value = caffeineCache.asMap().get(key);
log.info("[get caffeineCache] key:{}, value:{}", key, value);
return value;
}
@Override
public String generateCacheKey(String prefix, String body) {
if (Strings.isEmpty(prefix)) {
prefix = "-1";
}
return Joiner.on(":").join(cacheCommonConfig.getCacheCommonApp(), cacheCommonConfig.getCacheCommonEnv(),
cacheCommonConfig.getCacheCommonVersion(), prefix, body);
}
@Override
public Boolean removeCache(String key) {
caffeineCache.asMap().remove(key);
return true;
}
}

78
headless/server/src/main/java/com/tencent/supersonic/headless/server/cache/QueryCache.java vendored Normal file
View File

@@ -0,0 +1,78 @@
package com.tencent.supersonic.headless.server.cache;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.request.SemanticQueryReq;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class QueryCache {
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
@Autowired
private CacheManager cacheManager;
public Object query(SemanticQueryReq semanticQueryReq) {
String cacheKey = getCacheKey(semanticQueryReq);
handleGlobalCacheDisable(semanticQueryReq);
boolean isCache = isCache(semanticQueryReq);
if (isCache) {
Object result = cacheManager.get(cacheKey);
log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq);
return result;
}
return null;
}
public Boolean put(SemanticQueryReq semanticQueryReq, Object value) {
if (cacheEnable && Objects.nonNull(value)) {
String key = getCacheKey(semanticQueryReq);
CompletableFuture.supplyAsync(() -> cacheManager.put(key, value))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
log.info("add record to cache, key:{}", key);
return true;
}
return false;
}
public String getCacheKey(SemanticQueryReq semanticQueryReq) {
String commandMd5 = semanticQueryReq.generateCommandMd5();
String keyByModelIds = getKeyByModelIds(semanticQueryReq.getModelIds());
return cacheManager.generateCacheKey(keyByModelIds, commandMd5);
}
private void handleGlobalCacheDisable(SemanticQueryReq semanticQueryReq) {
if (!cacheEnable) {
Cache cacheInfo = new Cache();
cacheInfo.setCache(false);
semanticQueryReq.setCacheInfo(cacheInfo);
}
}
private String getKeyByModelIds(List<Long> modelIds) {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
private boolean isCache(SemanticQueryReq semanticQueryReq) {
if (!cacheEnable) {
return false;
}
if (semanticQueryReq.getCacheInfo() != null) {
return semanticQueryReq.getCacheInfo().getCache();
}
return false;
}
}

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CacheCommonConfig.java Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.headless.server.config;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
@Data
public class CacheCommonConfig {
@Value("${cache.common.app:supersonic}")
private String cacheCommonApp;
@Value("${cache.common.env:dev}")
private String cacheCommonEnv;
@Value("${cache.common.version:0}")
private Integer cacheCommonVersion;
@Value("${cache.common.expire.after.write:10}")
private Integer cacheCommonExpireAfterWrite;
}

40
headless/server/src/main/java/com/tencent/supersonic/headless/server/config/CaffeineCacheConfig.java Normal file
View File

@@ -0,0 +1,40 @@
package com.tencent.supersonic.headless.server.config;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class CaffeineCacheConfig {
@Autowired
private CacheCommonConfig cacheCommonConfig;
@Value("${caffeine.initial.capacity:500}")
private Integer caffeineInitialCapacity;
@Value("${caffeine.max.size:5000}")
private Integer caffeineMaximumSize;
@Bean(name = "caffeineCache")
public Cache<String, Object> caffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity)
.maximumSize(caffeineMaximumSize)
.build();
}
@Bean(name = "searchCaffeineCache")
public Cache<Long, Object> searchCaffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(10000, TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity)
.maximumSize(caffeineMaximumSize)
.build();
}
}

9
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
View File

@@ -56,13 +56,13 @@ import org.springframework.util.CollectionUtils;
@Slf4j
@Service
public class HeadlessSchemaManager {
public class SemanticSchemaManager {
@Autowired
private LoadingCache<String, SemanticModel> loadingCache;
private final Catalog catalog;
public HeadlessSchemaManager(Catalog catalog) {
public SemanticSchemaManager(Catalog catalog) {
this.catalog = catalog;
}
@@ -87,7 +87,7 @@ public class HeadlessSchemaManager {
semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
}
if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource)
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource)
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap);
}
@@ -104,7 +104,6 @@ public class HeadlessSchemaManager {
return semanticModel;
}
//private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
public SemanticModel get(String rootPath) throws Exception {
rootPath = formatKey(rootPath);
SemanticModel schema = loadingCache.get(rootPath);
@@ -383,7 +382,7 @@ public class HeadlessSchemaManager {
@Override
public SemanticModel load(String key) {
log.info("load SemanticSchema [{}]", key);
return HeadlessSchemaManager.this.reload(key);
return SemanticSchemaManager.this.reload(key);
}
}
);

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java
View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import java.util.Map;
@@ -67,7 +67,7 @@ public class DatabaseController {
}
@PostMapping("/executeSql")
public QueryResultWithSchemaResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
public SemanticQueryResp executeSql(@RequestBody SqlExecuteReq sqlExecuteReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
@@ -75,18 +75,18 @@ public class DatabaseController {
}
@RequestMapping("/getDbNames/{id}")
public QueryResultWithSchemaResp getDbNames(@PathVariable("id") Long id) {
public SemanticQueryResp getDbNames(@PathVariable("id") Long id) {
return databaseService.getDbNames(id);
}
@RequestMapping("/getTables/{id}/{db}")
public QueryResultWithSchemaResp getTables(@PathVariable("id") Long id,
public SemanticQueryResp getTables(@PathVariable("id") Long id,
@PathVariable("db") String db) {
return databaseService.getTables(id, db);
}
@RequestMapping("/getColumns/{id}/{db}/{table}")
public QueryResultWithSchemaResp getColumns(@PathVariable("id") Long id,
public SemanticQueryResp getColumns(@PathVariable("id") Long id,
@PathVariable("db") String db,
@PathVariable("table") String table) {
return databaseService.getColumns(id, db, table);

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java
View File

@@ -12,16 +12,16 @@ import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.response.SqlParserResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.service.DownloadService;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import com.tencent.supersonic.headless.server.service.QueryService;
import java.util.HashSet;
import java.util.List;
@@ -45,19 +45,17 @@ public class QueryController {
@Autowired
private QueryService queryService;
@Autowired
private HeadlessQueryEngine headlessQueryEngine;
private SemantciQueryEngine semantciQueryEngine;
@Autowired
private DownloadService downloadService;
@PostMapping("/sql")
public Object queryBySql(@RequestBody QueryS2SQLReq queryS2SQLReq,
public Object queryBySql(@RequestBody QuerySqlReq querySQLReq,
HttpServletRequest request,
HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response);
Object queryBySql = queryService.queryBySql(queryS2SQLReq, user);
log.info("queryBySql:{}", queryBySql);
return queryBySql;
return queryService.queryBySql(querySQLReq, user);
}
@PostMapping("/struct")
@@ -91,7 +89,7 @@ public class QueryController {
}
@PostMapping("/queryStatement")
public Object queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
public SemanticQueryResp queryStatement(@RequestBody QueryStatement queryStatement) throws Exception {
return queryService.queryByQueryStatement(queryStatement);
}
@@ -101,7 +99,7 @@ public class QueryController {
Set<Long> models = new HashSet<>();
models.add(Long.valueOf(parseSqlReq.getRootPath()));
queryStructCmd.setModelIds(models);
QueryStatement queryStatement = headlessQueryEngine.physicalSql(queryStructCmd, parseSqlReq);
QueryStatement queryStatement = semantciQueryEngine.physicalSql(queryStructCmd, parseSqlReq);
SqlParserResp sqlParserResp = new SqlParserResp();
BeanUtils.copyProperties(queryStatement, sqlParserResp);
return sqlParserResp;
@@ -130,7 +128,7 @@ public class QueryController {
}
@PostMapping("/queryDimValue")
public QueryResultWithSchemaResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
public SemanticQueryResp queryDimValue(@RequestBody QueryDimValueReq queryDimValueReq,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
@@ -147,9 +145,9 @@ public class QueryController {
QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum();
if (QueryType.SQL.equals(queryTypeEnum)) {
QueryS2SQLReq queryS2SQLReq = JsonUtil.toObject(queryReqJson, QueryS2SQLReq.class);
ExplainSqlReq<QueryS2SQLReq> explainSqlReqNew = ExplainSqlReq.<QueryS2SQLReq>builder()
.queryReq(queryS2SQLReq)
QuerySqlReq querySQLReq = JsonUtil.toObject(queryReqJson, QuerySqlReq.class);
ExplainSqlReq<QuerySqlReq> explainSqlReqNew = ExplainSqlReq.<QuerySqlReq>builder()
.queryReq(querySQLReq)
.queryTypeEnum(queryTypeEnum).build();
return queryService.explain(explainSqlReqNew, user);
}

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
import java.util.List;
import java.util.Map;
@@ -11,9 +11,9 @@ import java.util.Map;
public interface DatabaseService {
QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp);
SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp);
QueryResultWithSchemaResp executeSql(String sql, Long id, User user);
SemanticQueryResp executeSql(String sql, Long id, User user);
Map<String, List<DatabaseParameter>> getDatabaseParameters();
@@ -27,9 +27,9 @@ public interface DatabaseService {
DatabaseResp getDatabase(Long id);
QueryResultWithSchemaResp getDbNames(Long id);
SemanticQueryResp getDbNames(Long id);
QueryResultWithSchemaResp getTables(Long id, String db);
SemanticQueryResp getTables(Long id, String db);
QueryResultWithSchemaResp getColumns(Long id, String db, String table);
SemanticQueryResp getColumns(Long id, String db, String table);
}

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
View File

@@ -6,12 +6,12 @@ import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ExplainResp;
import com.tencent.supersonic.headless.api.response.ItemQueryResultResp;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.ApiHeaderCheck;
@@ -20,18 +20,17 @@ import java.util.List;
public interface QueryService {
Object queryBySql(QueryS2SQLReq querySqlCmd, User user) throws Exception;
Object queryBySql(QuerySqlReq querySqlCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;
SemanticQueryResp queryByStruct(QueryStructReq queryStructCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user)
throws Exception;
SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructCmd, User user) throws Exception;
QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception;
SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructCmd, User user) throws Exception;
QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);
Object queryByQueryStatement(QueryStatement queryStatement);
SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement);
List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
@@ -39,6 +38,6 @@ public interface QueryService {
@ApiHeaderCheck
ItemQueryResultResp queryMetricDataById(QueryItemReq queryApiReq,
HttpServletRequest request) throws Exception;
HttpServletRequest request) throws Exception;
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/HeadlessQueryEngine.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/SemantciQueryEngine.java
View File

@@ -3,17 +3,17 @@ package com.tencent.supersonic.headless.server.service;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
public interface HeadlessQueryEngine {
public interface SemantciQueryEngine {
QueryStatement plan(QueryStatement queryStatement) throws Exception;
QueryExecutor route(QueryStatement queryStatement);
QueryResultWithSchemaResp execute(QueryStatement queryStatement);
SemanticQueryResp execute(QueryStatement queryStatement);
QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception;

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java
View File

@@ -4,7 +4,7 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.request.DatabaseReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.Database;
@@ -112,10 +112,10 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp executeSql(String sql, Long id, User user) {
public SemanticQueryResp executeSql(String sql, Long id, User user) {
DatabaseResp databaseResp = getDatabase(id);
if (databaseResp == null) {
return new QueryResultWithSchemaResp();
return new SemanticQueryResp();
}
List<String> admins = databaseResp.getAdmins();
List<String> viewers = databaseResp.getViewers();
@@ -132,7 +132,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp executeSql(String sql, DatabaseResp databaseResp) {
public SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp) {
return queryWithColumns(sql, DatabaseConverter.convert(databaseResp));
}
@@ -143,8 +143,8 @@ public class DatabaseServiceImpl implements DatabaseService {
LinkedHashMap::putAll);
}
private QueryResultWithSchemaResp queryWithColumns(String sql, Database database) {
QueryResultWithSchemaResp queryResultWithColumns = new QueryResultWithSchemaResp();
private SemanticQueryResp queryWithColumns(String sql, Database database) {
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
SqlUtils sqlUtils = this.sqlUtils.init(database);
log.info("query SQL: {}", sql);
sqlUtils.queryInternal(sql, queryResultWithColumns);
@@ -156,7 +156,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp getDbNames(Long id) {
public SemanticQueryResp getDbNames(Long id) {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getDbMetaQueryTpl();
@@ -164,7 +164,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp getTables(Long id, String db) {
public SemanticQueryResp getTables(Long id, String db) {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getTableMetaQueryTpl();
@@ -173,7 +173,7 @@ public class DatabaseServiceImpl implements DatabaseService {
}
@Override
public QueryResultWithSchemaResp getColumns(Long id, String db, String table) {
public SemanticQueryResp getColumns(Long id, String db, String table) {
DatabaseResp databaseResp = getDatabase(id);
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
String metaQueryTpl = engineAdaptor.getColumnMetaQueryTpl();

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View File

@@ -23,7 +23,7 @@ import com.tencent.supersonic.headless.api.request.PageDimensionReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
@@ -266,8 +266,8 @@ public class DimensionServiceImpl implements DimensionService {
String sql = "select ai_talk." + dimensionReq.getBizName() + " from (" + sqlQuery
+ ") as ai_talk group by ai_talk." + dimensionReq.getBizName();
QueryResultWithSchemaResp queryResultWithSchemaResp = databaseService.executeSql(sql, database);
List<Map<String, Object>> resultList = queryResultWithSchemaResp.getResultList();
SemanticQueryResp semanticQueryResp = databaseService.executeSql(sql, database);
List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
List<String> valueList = new ArrayList<>();
for (Map<String, Object> stringObjectMap : resultList) {
String value = (String) stringObjectMap.get(dimensionReq.getBizName());

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DownloadServiceImpl.java
View File

@@ -21,7 +21,7 @@ import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.utils.DataTransformUtils;
import com.tencent.supersonic.headless.server.pojo.DataDownload;
import com.tencent.supersonic.headless.server.service.DownloadService;
@@ -72,7 +72,7 @@ public class DownloadServiceImpl implements DownloadService {
String fileName = String.format("%s_%s.xlsx", "supersonic", DateUtils.format(new Date(), DateUtils.FORMAT));
File file = FileUtils.createTmpFile(fileName);
try {
QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
SemanticQueryResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
EasyExcel.write(file).sheet("Sheet1").head(dataDownload.getHeaders()).doWrite(dataDownload.getData());
} catch (RuntimeException e) {
@@ -112,7 +112,7 @@ public class DownloadServiceImpl implements DownloadService {
for (MetricSchemaResp metric : metrics) {
try {
DownloadStructReq downloadStructReq = buildDownloadStructReq(dimensions, metric, batchDownloadReq);
QueryResultWithSchemaResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
SemanticQueryResp queryResult = queryService.queryByStructWithAuth(downloadStructReq, user);
DataDownload dataDownload = buildDataDownload(queryResult, downloadStructReq);
WriteSheet writeSheet = EasyExcel.writerSheet("Sheet" + sheetCount)
.head(dataDownload.getHeaders()).build();
@@ -140,9 +140,9 @@ public class DownloadServiceImpl implements DownloadService {
return data;
}
private List<List<String>> buildHeader(QueryResultWithSchemaResp queryResultWithSchemaResp) {
private List<List<String>> buildHeader(SemanticQueryResp semanticQueryResp) {
List<List<String>> header = Lists.newArrayList();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
for (QueryColumn column : semanticQueryResp.getColumns()) {
header.add(Lists.newArrayList(column.getName()));
}
return header;
@@ -163,11 +163,11 @@ public class DownloadServiceImpl implements DownloadService {
return headers;
}
private List<List<String>> buildData(QueryResultWithSchemaResp queryResultWithSchemaResp) {
private List<List<String>> buildData(SemanticQueryResp semanticQueryResp) {
List<List<String>> data = new ArrayList<>();
for (Map<String, Object> row : queryResultWithSchemaResp.getResultList()) {
for (Map<String, Object> row : semanticQueryResp.getResultList()) {
List<String> rowData = new ArrayList<>();
for (QueryColumn column : queryResultWithSchemaResp.getColumns()) {
for (QueryColumn column : semanticQueryResp.getColumns()) {
rowData.add(String.valueOf(row.get(column.getNameEn())));
}
data.add(rowData);
@@ -198,7 +198,7 @@ public class DownloadServiceImpl implements DownloadService {
return data;
}
private DataDownload buildDataDownload(QueryResultWithSchemaResp queryResult, DownloadStructReq downloadStructReq) {
private DataDownload buildDataDownload(SemanticQueryResp queryResult, DownloadStructReq downloadStructReq) {
List<QueryColumn> metricColumns = queryResult.getMetricColumns();
List<QueryColumn> dimensionColumns = queryResult.getDimensionColumns();
if (downloadStructReq.isTransform() && !CollectionUtils.isEmpty(metricColumns)) {

268
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.service.impl;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -13,20 +14,17 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.headless.api.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Item;
import com.tencent.supersonic.headless.api.pojo.SingleItemQueryResult;
import com.tencent.supersonic.headless.api.request.ExplainSqlReq;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.request.QueryDimValueReq;
import com.tencent.supersonic.headless.api.request.QueryItemReq;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.AppDetailResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
@@ -36,28 +34,27 @@ import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.planner.QueryPlanner;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.server.annotation.S2SQLDataPermission;
import com.tencent.supersonic.headless.server.annotation.StructDataPermission;
import com.tencent.supersonic.headless.server.aspect.ApiHeaderCheckAspect;
import com.tencent.supersonic.headless.server.cache.CacheManager;
import com.tencent.supersonic.headless.server.cache.QueryCache;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.service.AppService;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.QueryService;
import com.tencent.supersonic.headless.server.service.SchemaService;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryReqConverter;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import com.tencent.supersonic.headless.server.utils.StatUtils;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -65,17 +62,24 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class QueryServiceImpl implements QueryService {
protected final com.google.common.cache.Cache<String, List<ItemUseResp>> itemUseCache =
protected final Cache<String, List<ItemUseResp>> itemUseCache =
CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.DAYS).build();
private final StatUtils statUtils;
private final CacheUtils cacheUtils;
private final CacheManager cacheManager;
private final QueryUtils queryUtils;
private final QueryReqConverter queryReqConverter;
private final Catalog catalog;
@@ -84,88 +88,109 @@ public class QueryServiceImpl implements QueryService {
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
private final HeadlessQueryEngine headlessQueryEngine;
private final QueryCache queryCache;
private final SemantciQueryEngine semantciQueryEngine;
private final SemanticSchemaManager semanticSchemaManager;
private final QueryParser queryParser;
private final QueryPlanner queryPlanner;
public QueryServiceImpl(
StatUtils statUtils,
CacheUtils cacheUtils,
CacheManager cacheManager,
QueryUtils queryUtils,
QueryReqConverter queryReqConverter,
@Lazy HeadlessQueryEngine headlessQueryEngine,
@Lazy SemantciQueryEngine semantciQueryEngine,
Catalog catalog,
AppService appService) {
AppService appService,
QueryCache queryCache,
SemanticSchemaManager semanticSchemaManager,
QueryParser queryParser,
QueryPlanner queryPlanner) {
this.statUtils = statUtils;
this.cacheUtils = cacheUtils;
this.cacheManager = cacheManager;
this.queryUtils = queryUtils;
this.queryReqConverter = queryReqConverter;
this.headlessQueryEngine = headlessQueryEngine;
this.semantciQueryEngine = semantciQueryEngine;
this.catalog = catalog;
this.appService = appService;
this.queryCache = queryCache;
this.semanticSchemaManager = semanticSchemaManager;
this.queryParser = queryParser;
this.queryPlanner = queryPlanner;
}
@Override
@S2SQLDataPermission
@SneakyThrows
public Object queryBySql(QueryS2SQLReq queryS2SQLReq, User user) {
statUtils.initStatInfo(queryS2SQLReq, user);
public Object queryBySql(QuerySqlReq querySQLReq, User user) {
statUtils.initStatInfo(querySQLReq, user);
QueryStatement queryStatement = new QueryStatement();
try {
queryStatement = convertToQueryStatement(queryS2SQLReq, user);
queryStatement = convertToQueryStatement(querySQLReq, user);
} catch (Exception e) {
log.info("convertToQueryStatement has a exception:", e);
}
log.info("queryStatement:{}", queryStatement);
QueryResultWithSchemaResp results = headlessQueryEngine.execute(queryStatement);
SemanticQueryResp results = semantciQueryEngine.execute(queryStatement);
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return results;
}
public Object queryByQueryStatement(QueryStatement queryStatement) {
return headlessQueryEngine.execute(queryStatement);
public SemanticQueryResp queryByQueryStatement(QueryStatement queryStatement) {
return semantciQueryEngine.execute(queryStatement);
}
private QueryStatement convertToQueryStatement(QueryS2SQLReq queryS2SQLReq, User user) throws Exception {
private QueryStatement convertToQueryStatement(QuerySqlReq querySQLReq, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(queryS2SQLReq.getModelIds());
filter.setModelIds(querySQLReq.getModelIds());
SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(queryS2SQLReq, modelSchemaResps);
queryStatement.setModelIds(queryS2SQLReq.getModelIds());
QueryStatement queryStatement = queryReqConverter.convert(querySQLReq, modelSchemaResps);
queryStatement.setModelIds(querySQLReq.getModelIds());
return queryStatement;
}
@Override
public QueryResultWithSchemaResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
QueryResultWithSchemaResp queryResultWithColumns = null;
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) throws Exception {
SemanticQueryResp semanticQueryResp = null;
log.info("[queryStructReq:{}]", queryStructReq);
try {
//1.initStatInfo
statUtils.initStatInfo(queryStructReq, user);
String cacheKey = cacheUtils.generateCacheKey(getKeyByModelIds(queryStructReq.getModelIds()),
queryStructReq.generateCommandMd5());
handleGlobalCacheDisable(queryStructReq);
boolean isCache = isCache(queryStructReq);
if (isCache) {
queryResultWithColumns = queryByCache(cacheKey, queryStructReq);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
}
//2.query from cache
Object query = queryCache.query(queryStructReq);
if (Objects.nonNull(query)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return (SemanticQueryResp) query;
}
StatUtils.get().setUseResultCache(false);
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement = headlessQueryEngine.plan(queryStatement);
QueryExecutor queryExecutor = headlessQueryEngine.route(queryStatement);
//3 parse
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = queryParser.parse(queryStatement);
//4 plan
QueryExecutor queryExecutor = queryPlanner.plan(queryStatement);
//5 execute
if (queryExecutor != null) {
queryResultWithColumns = headlessQueryEngine.execute(queryStatement);
if (isCache) {
// if queryResultWithColumns is not null, update cache data
queryUtils.cacheResultLogic(cacheKey, queryResultWithColumns);
semanticQueryResp = queryExecutor.execute(queryStatement);
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
}
}
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
//6 reset cache and set stateInfo
Boolean putCacheSuccess = queryCache.put(queryStructReq, semanticQueryResp);
if (putCacheSuccess) {
// if semanticQueryResp is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(queryStructReq));
}
if (Objects.nonNull(semanticQueryResp)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
}
return semanticQueryResp;
} catch (Exception e) {
log.warn("exception in queryByStruct, e: ", e);
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
@@ -173,38 +198,62 @@ public class QueryServiceImpl implements QueryService {
}
}
private QueryStatement buildQueryStatement(QueryStructReq queryStructReq) throws Exception {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
SemanticModel semanticModel = semanticSchemaManager.get(queryStructReq.getModelIdStr());
queryStatement.setSemanticModel(semanticModel);
return queryStatement;
}
@Override
@StructDataPermission
@SneakyThrows
public QueryResultWithSchemaResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) {
public SemanticQueryResp queryByStructWithAuth(QueryStructReq queryStructReq, User user) {
return queryByStruct(queryStructReq, user);
}
@Override
public QueryResultWithSchemaResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user)
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user)
throws Exception {
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
String cacheKey = cacheUtils.generateCacheKey(
getKeyByModelIds(queryMultiStructReq.getQueryStructReqs().get(0).getModelIds()),
queryMultiStructReq.generateCommandMd5());
boolean isCache = isCache(queryMultiStructReq);
QueryResultWithSchemaResp queryResultWithColumns;
if (isCache) {
queryResultWithColumns = queryByCache(cacheKey, queryMultiStructReq);
if (queryResultWithColumns != null) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
return queryResultWithColumns;
}
}
log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq);
try {
QueryStatement sqlParser = getQueryStatementByMultiStruct(queryMultiStructReq);
queryResultWithColumns = headlessQueryEngine.execute(sqlParser);
if (queryResultWithColumns != null) {
//1.initStatInfo
statUtils.initStatInfo(queryMultiStructReq.getQueryStructReqs().get(0), user);
//2.query from cache
Object query = queryCache.query(queryMultiStructReq);
if (Objects.nonNull(query)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
queryUtils.fillItemNameInfo(queryResultWithColumns, queryMultiStructReq);
return (SemanticQueryResp) query;
}
return queryResultWithColumns;
StatUtils.get().setUseResultCache(false);
log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq);
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = queryParser.parse(queryStatement);
queryPlanner.optimizer(queryStatement);
sqlParsers.add(queryStatement);
}
log.info("multi sqlParser:{}", sqlParsers);
QueryStatement queryStatement = queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
QueryExecutor executor = queryPlanner.route(queryStatement);
SemanticQueryResp semanticQueryResp = null;
if (executor != null) {
semanticQueryResp = executor.execute(queryStatement);
if (!CollectionUtils.isEmpty(queryStatement.getModelIds())) {
queryUtils.fillItemNameInfo(semanticQueryResp, queryStatement.getModelIds());
}
}
if (Objects.nonNull(semanticQueryResp)) {
statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS);
}
return semanticQueryResp;
} catch (Exception e) {
log.warn("exception in queryByMultiStruct, e: ", e);
statUtils.statInfo2DbAsync(TaskStatusEnum.ERROR);
@@ -212,13 +261,11 @@ public class QueryServiceImpl implements QueryService {
}
}
private QueryStatement getQueryStatementByMultiStruct(QueryMultiStructReq queryMultiStructReq) throws Exception {
private QueryStatement buildQueryStatement(QueryMultiStructReq queryMultiStructReq) throws Exception {
List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq(queryStructReq);
queryStatement.setIsS2SQL(false);
queryStatement = headlessQueryEngine.plan(queryStatement);
QueryStatement queryStatement = buildQueryStatement(queryStructReq);
queryStatement = semantciQueryEngine.plan(queryStatement);
queryUtils.checkSqlParse(queryStatement);
sqlParsers.add(queryStatement);
}
@@ -228,17 +275,9 @@ public class QueryServiceImpl implements QueryService {
@Override
@SneakyThrows
public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
QueryS2SQLReq queryS2SQLReq = generateDimValueQuerySql(queryDimValueReq);
return (QueryResultWithSchemaResp) queryBySql(queryS2SQLReq, user);
}
private void handleGlobalCacheDisable(QueryStructReq queryStructReq) {
if (!cacheEnable) {
Cache cacheInfo = new Cache();
cacheInfo.setCache(false);
queryStructReq.setCacheInfo(cacheInfo);
}
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
QuerySqlReq querySQLReq = buildQuerySqlReq(queryDimValueReq);
return (SemanticQueryResp) queryBySql(querySQLReq, user);
}
@Override
@@ -259,20 +298,18 @@ public class QueryServiceImpl implements QueryService {
QueryType queryTypeEnum = explainSqlReq.getQueryTypeEnum();
T queryReq = explainSqlReq.getQueryReq();
if (QueryType.SQL.equals(queryTypeEnum) && queryReq instanceof QueryS2SQLReq) {
QueryStatement queryStatement = convertToQueryStatement((QueryS2SQLReq) queryReq, user);
if (QueryType.SQL.equals(queryTypeEnum) && queryReq instanceof QuerySqlReq) {
QueryStatement queryStatement = convertToQueryStatement((QuerySqlReq) queryReq, user);
return getExplainResp(queryStatement);
}
if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryStructReq) {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryStructReq((QueryStructReq) queryReq);
queryStatement.setIsS2SQL(false);
queryStatement = headlessQueryEngine.plan(queryStatement);
QueryStatement queryStatement = buildQueryStatement((QueryStructReq) queryReq);
queryStatement = semantciQueryEngine.plan(queryStatement);
return getExplainResp(queryStatement);
}
if (QueryType.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryMultiStructReq) {
QueryMultiStructReq queryMultiStructReq = (QueryMultiStructReq) queryReq;
QueryStatement queryStatement = getQueryStatementByMultiStruct(queryMultiStructReq);
QueryStatement queryStatement = buildQueryStatement(queryMultiStructReq);
return getExplainResp(queryStatement);
}
@@ -310,11 +347,11 @@ public class QueryServiceImpl implements QueryService {
dimensionResps = catalog.getDimensions(dimensionFilter);
}
QueryStructReq queryStructReq = buildQueryStructReq(dimensionResps, metricResp, dateConf, limit);
QueryResultWithSchemaResp queryResultWithSchemaResp =
SemanticQueryResp semanticQueryResp =
queryByStruct(queryStructReq, User.getAppUser(appId));
SingleItemQueryResult apiQuerySingleResult = new SingleItemQueryResult();
apiQuerySingleResult.setItem(item);
apiQuerySingleResult.setResult(queryResultWithSchemaResp);
apiQuerySingleResult.setResult(semanticQueryResp);
return apiQuerySingleResult;
}
@@ -366,21 +403,6 @@ public class QueryServiceImpl implements QueryService {
return ExplainResp.builder().sql(sql).build();
}
public QueryStatement parseMetricReq(MetricQueryReq metricReq) throws Exception {
QueryStructReq queryStructReq = new QueryStructReq();
return headlessQueryEngine.physicalSql(queryStructReq, metricReq);
}
private boolean isCache(QueryStructReq queryStructReq) {
if (!cacheEnable) {
return false;
}
if (queryStructReq.getCacheInfo() != null) {
return queryStructReq.getCacheInfo().getCache();
}
return false;
}
private boolean isCache(QueryMultiStructReq queryStructReq) {
if (!cacheEnable) {
return false;
@@ -392,19 +414,19 @@ public class QueryServiceImpl implements QueryService {
return false;
}
private QueryResultWithSchemaResp queryByCache(String key, Object queryCmd) {
private SemanticQueryResp queryByCache(String key, Object queryCmd) {
Object resultObject = cacheUtils.get(key);
Object resultObject = cacheManager.get(key);
if (Objects.nonNull(resultObject)) {
log.info("queryByStructWithCache, key:{}, queryCmd:{}", key, queryCmd.toString());
statUtils.updateResultCacheKey(key);
return (QueryResultWithSchemaResp) resultObject;
return (SemanticQueryResp) resultObject;
}
return null;
}
private QueryS2SQLReq generateDimValueQuerySql(QueryDimValueReq queryDimValueReq) {
QueryS2SQLReq queryS2SQLReq = new QueryS2SQLReq();
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
QuerySqlReq querySQLReq = new QuerySqlReq();
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(),
queryDimValueReq.getModelId());
@@ -416,9 +438,9 @@ public class QueryServiceImpl implements QueryService {
queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate());
}
queryS2SQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
queryS2SQLReq.setSql(sql);
return queryS2SQLReq;
querySQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySQLReq.setSql(sql);
return querySQLReq;
}
private String getKeyByModelIds(List<Long> modelIds) {

30
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SemantciQueryEngineImpl.java
View File

@@ -3,16 +3,15 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.QueryParser;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import com.tencent.supersonic.headless.server.utils.QueryUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -20,24 +19,21 @@ import org.springframework.util.CollectionUtils;
@Slf4j
@Component
public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
public class SemantciQueryEngineImpl implements SemantciQueryEngine {
private final QueryParser queryParser;
private final QueryUtils queryUtils;
private final QueryStructUtils queryStructUtils;
private final HeadlessSchemaManager headlessSchemaManager;
private final SemanticSchemaManager semanticSchemaManager;
public HeadlessQueryEngineImpl(QueryParser queryParser,
QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager,
QueryStructUtils queryStructUtils) {
public SemantciQueryEngineImpl(QueryParser queryParser,
QueryUtils queryUtils, SemanticSchemaManager semanticSchemaManager) {
this.queryParser = queryParser;
this.queryUtils = queryUtils;
this.headlessSchemaManager = headlessSchemaManager;
this.queryStructUtils = queryStructUtils;
this.semanticSchemaManager = semanticSchemaManager;
}
public QueryResultWithSchemaResp execute(QueryStatement queryStatement) {
QueryResultWithSchemaResp queryResultWithColumns = null;
public SemanticQueryResp execute(QueryStatement queryStatement) {
SemanticQueryResp queryResultWithColumns = null;
QueryExecutor queryExecutor = route(queryStatement);
if (queryExecutor != null) {
queryResultWithColumns = queryExecutor.execute(queryStatement);
@@ -52,7 +48,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
public QueryStatement plan(QueryStatement queryStatement) throws Exception {
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setSemanticModel(getSemanticModel(queryStatement));
queryStatement = queryParser.logicSql(queryStatement);
queryStatement = queryParser.parse(queryStatement);
queryUtils.checkSqlParse(queryStatement);
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
log.info("queryStatement:{}", queryStatement);
@@ -97,7 +93,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
return headlessSchemaManager.get(queryStructReq.getModelIdStr());
return semanticSchemaManager.get(queryStructReq.getModelIdStr());
}
}

46
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -16,7 +16,7 @@ import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
@@ -29,7 +29,7 @@ import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
import com.tencent.supersonic.headless.server.service.SemantciQueryEngine;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -56,7 +56,7 @@ public class QueryReqConverter {
private Boolean limitWrapper;
@Autowired
private HeadlessQueryEngine headlessQueryEngine;
private SemantciQueryEngine semantciQueryEngine;
@Autowired
private QueryStructUtils queryStructUtils;
@@ -66,7 +66,7 @@ public class QueryReqConverter {
@Autowired
private Catalog catalog;
public QueryStatement convert(QueryS2SQLReq queryS2SQLReq,
public QueryStatement convert(QuerySqlReq querySQLReq,
List<ModelSchemaResp> modelSchemaResps) throws Exception {
if (CollectionUtils.isEmpty(modelSchemaResps)) {
@@ -75,18 +75,18 @@ public class QueryReqConverter {
Map<Long, ModelSchemaResp> modelSchemaRespMap = modelSchemaResps.stream()
.collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp));
//1.convert name to bizName
convertNameToBizName(queryS2SQLReq, modelSchemaResps);
convertNameToBizName(querySQLReq, modelSchemaResps);
//2.functionName corrector
functionNameCorrector(queryS2SQLReq);
functionNameCorrector(querySQLReq);
//3.correct tableName
correctTableName(queryS2SQLReq);
correctTableName(querySQLReq);
String tableName = SqlParserSelectHelper.getTableName(queryS2SQLReq.getSql());
String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql());
if (StringUtils.isEmpty(tableName)) {
return new QueryStatement();
}
//4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
List<String> metrics = getMetrics(modelSchemaResps, allFields);
QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable();
@@ -100,7 +100,7 @@ public class QueryReqConverter {
// if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>());
for (Long modelId : queryS2SQLReq.getModelIds()) {
for (Long modelId : querySQLReq.getModelIds()) {
ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId);
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName()));
}
@@ -109,27 +109,27 @@ public class QueryReqConverter {
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
Collectors.toList()));
}
AggOption aggOption = getAggOption(queryS2SQLReq);
AggOption aggOption = getAggOption(querySQLReq);
metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable);
//4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(queryS2SQLReq, result);
BeanUtils.copyProperties(querySQLReq, result);
result.setRootPath(queryS2SQLReq.getModelIdStr());
result.setRootPath(querySQLReq.getModelIdStr());
result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(queryS2SQLReq.getModelIds().get(0));
DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0));
if (!sqlGenerateUtils.isSupportWith(EngineType.valueOf(database.getType().toUpperCase()),
database.getVersion())) {
result.setSupportWith(false);
result.setWithAlias(false);
}
//5. do deriveMetric
generateDerivedMetric(queryS2SQLReq.getModelIds(), modelSchemaResps, result);
generateDerivedMetric(querySQLReq.getModelIds(), modelSchemaResps, result);
//6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds()));
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(querySQLReq.getModelIds()));
queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryStatement queryStatement = new QueryStatement();
@@ -137,14 +137,14 @@ public class QueryReqConverter {
queryStatement.setParseSqlReq(result);
queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setModelIds(queryS2SQLReq.getModelIds());
queryStatement = headlessQueryEngine.plan(queryStatement);
queryStatement.setModelIds(querySQLReq.getModelIds());
queryStatement = semantciQueryEngine.plan(queryStatement);
queryStatement.setSql(limitWrapper ? String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql())
: queryStatement.getSql());
return queryStatement;
}
private AggOption getAggOption(QueryS2SQLReq databaseReq) {
private AggOption getAggOption(QuerySqlReq databaseReq) {
// if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE"
// if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE"
String sql = databaseReq.getSql();
@@ -156,7 +156,7 @@ public class QueryReqConverter {
return AggOption.DEFAULT;
}
private void convertNameToBizName(QueryS2SQLReq databaseReq, List<ModelSchemaResp> modelSchemaResps) {
private void convertNameToBizName(QuerySqlReq databaseReq, List<ModelSchemaResp> modelSchemaResps) {
Map<String, String> fieldNameToBizNameMap = getFieldNameToBizNameMap(modelSchemaResps);
String sql = databaseReq.getSql();
log.info("convert name to bizName before:{}", sql);
@@ -186,7 +186,7 @@ public class QueryReqConverter {
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
}
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
private void functionNameCorrector(QuerySqlReq databaseReq) {
DatabaseResp database = catalog.getDatabaseByModelId(databaseReq.getModelIds().get(0));
if (Objects.isNull(database) || Objects.isNull(database.getType())) {
return;
@@ -231,7 +231,7 @@ public class QueryReqConverter {
return elements.stream();
}
public void correctTableName(QueryS2SQLReq databaseReq) {
public void correctTableName(QuerySqlReq databaseReq) {
String sql = databaseReq.getSql();
for (Long modelId : databaseReq.getModelIds()) {
sql = SqlParserReplaceHelper.replaceTable(sql, Constants.TABLE_PREFIX + modelId);

39
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -1,5 +1,10 @@
package com.tencent.supersonic.headless.server.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
@@ -14,7 +19,7 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
import com.tencent.supersonic.headless.api.pojo.ItemDateFilter;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ModelSchemaFilterReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.response.DimensionResp;
@@ -24,13 +29,6 @@ import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -44,11 +42,12 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -130,8 +129,8 @@ public class QueryStructUtils {
return resNameEnSet;
}
public Set<String> getResName(QueryS2SQLReq queryS2SQLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql())
public Set<String> getResName(QuerySqlReq querySQLReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(querySQLReq.getSql())
.stream().collect(Collectors.toSet());
return resNameSet;
}
@@ -141,11 +140,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq, User user) {
Set<String> resNameSet = getResName(queryS2SQLReq);
public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySQLReq, User user) {
Set<String> resNameSet = getResName(querySQLReq);
Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(queryS2SQLReq.getModelIds());
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds());
filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -176,8 +175,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}
public Set<String> getFilterResNameEnExceptInternalCol(QueryS2SQLReq queryS2SQLReq) {
String sql = queryS2SQLReq.getSql();
public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySQLReq) {
String sql = querySQLReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
}

52
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -1,40 +1,37 @@
package com.tencent.supersonic.headless.server.utils;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.cache.CacheUtils;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.enums.SemanticType;
import com.tencent.supersonic.headless.api.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.cache.CacheManager;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.service.Catalog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.UNIONALL;
import javax.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -52,15 +49,15 @@ public class QueryUtils {
@Value("${query.optimizer.enable:true}")
private Boolean optimizeEnable;
private final CacheUtils cacheUtils;
private final CacheManager cacheManager;
private final StatUtils statUtils;
private final Catalog catalog;
public QueryUtils(
CacheUtils cacheUtils, StatUtils statUtils, Catalog catalog) {
CacheManager cacheManager, StatUtils statUtils, Catalog catalog) {
this.cacheUtils = cacheUtils;
this.cacheManager = cacheManager;
this.statUtils = statUtils;
this.catalog = catalog;
}
@@ -74,7 +71,7 @@ public class QueryUtils {
}
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns, List<Long> modelIds) {
public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns, List<Long> modelIds) {
MetaFilter metaFilter = new MetaFilter(modelIds);
List<MetricResp> metricDescList = catalog.getMetrics(metaFilter);
List<DimensionResp> dimensionDescList = catalog.getDimensions(metaFilter);
@@ -125,7 +122,7 @@ public class QueryUtils {
});
}
public void fillItemNameInfo(QueryResultWithSchemaResp queryResultWithColumns,
public void fillItemNameInfo(SemanticQueryResp queryResultWithColumns,
QueryMultiStructReq queryMultiStructCmd) {
List<Aggregator> aggregators = queryMultiStructCmd.getQueryStructReqs().stream()
.flatMap(queryStructCmd -> queryStructCmd.getAggregators().stream())
@@ -244,21 +241,6 @@ public class QueryUtils {
return sqlParser;
}
public void cacheResultLogic(String key, QueryResultWithSchemaResp queryResultWithColumns) {
if (cacheEnable && Objects.nonNull(queryResultWithColumns) && !CollectionUtils.isEmpty(
queryResultWithColumns.getResultList())) {
QueryResultWithSchemaResp finalQueryResultWithColumns = queryResultWithColumns;
CompletableFuture.supplyAsync(() -> cacheUtils.put(key, finalQueryResultWithColumns))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
statUtils.updateResultCacheKey(key);
log.info("add record to cache, key:{}", key);
}
}
public Boolean enableOptimize() {
return optimizeEnable;
}

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -13,7 +13,7 @@ import com.tencent.supersonic.headless.api.enums.QueryTypeBack;
import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.request.ItemUseReq;
import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
import com.tencent.supersonic.headless.api.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.response.ItemUseResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
@@ -85,11 +85,11 @@ public class StatUtils {
return true;
}
public void initStatInfo(QueryS2SQLReq queryS2SQLReq, User facadeUser) {
public void initStatInfo(QuerySqlReq querySQLReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(queryS2SQLReq.getSql());
queryStatInfo.setModelId(queryS2SQLReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(queryS2SQLReq.getModelIds().get(0));
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql());
queryStatInfo.setModelId(querySQLReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySQLReq.getModelIds().get(0));
List<String> dimensions = new ArrayList<>();
List<String> metrics = new ArrayList<>();
@@ -101,12 +101,12 @@ public class StatUtils {
String userName = getUserName(facadeUser);
try {
queryStatInfo.setTraceId("")
.setModelId(queryS2SQLReq.getModelIds().get(0))
.setModelId(querySQLReq.getModelIds().get(0))
.setUser(userName)
.setQueryType(QueryType.SQL.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQuerySqlCmd(queryS2SQLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(queryS2SQLReq.toString()))
.setQuerySqlCmd(querySQLReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(querySQLReq.toString()))
.setStartTime(System.currentTimeMillis())
.setUseResultCache(true)
.setUseSqlCache(true)

12
headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java
View File

@@ -14,7 +14,7 @@ import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import lombok.extern.slf4j.Slf4j;
import java.util.ArrayList;
@@ -124,7 +124,7 @@ class HeadlessParserServiceTest {
SemanticSchema semanticSchema = SemanticSchema.newBuilder("s2").build();
HeadlessSchemaManager.update(semanticSchema, HeadlessSchemaManager.getDatasource(datasource));
SemanticSchemaManager.update(semanticSchema, SemanticSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("page");
@@ -133,8 +133,8 @@ class HeadlessParserServiceTest {
List<DimensionYamlTpl> dimensionYamlTpls = new ArrayList<>();
dimensionYamlTpls.add(dimension1);
HeadlessSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
SemanticSchemaManager.update(semanticSchema, "s2_pv_uv_statis",
SemanticSchemaManager.getDimensions(dimensionYamlTpls));
MetricYamlTpl metric1 = new MetricYamlTpl();
metric1.setName("pv");
@@ -240,7 +240,7 @@ class HeadlessParserServiceTest {
identifies.add(identify);
datasource.setIdentifiers(identifies);
semanticSchema.getDatasource().put("user_department", HeadlessSchemaManager.getDatasource(datasource));
semanticSchema.getDatasource().put("user_department", SemanticSchemaManager.getDatasource(datasource));
DimensionYamlTpl dimension1 = new DimensionYamlTpl();
dimension1.setExpr("department");
@@ -250,6 +250,6 @@ class HeadlessParserServiceTest {
dimensionYamlTpls.add(dimension1);
semanticSchema.getDimension()
.put("user_department", HeadlessSchemaManager.getDimensions(dimensionYamlTpls));
.put("user_department", SemanticSchemaManager.getDimensions(dimensionYamlTpls));
}
}

21
headless/server/src/test/java/com/tencent/supersonic/headless/server/service/DownloadServiceImplTest.java
View File

@@ -1,5 +1,8 @@
package com.tencent.supersonic.headless.server.service;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import com.alibaba.excel.util.FileUtils;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -11,20 +14,16 @@ import com.tencent.supersonic.headless.api.request.BatchDownloadReq;
import com.tencent.supersonic.headless.api.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.headless.api.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.impl.DownloadServiceImpl;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.io.File;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
class DownloadServiceImplTest {
@@ -98,16 +97,16 @@ class DownloadServiceImplTest {
return dateConf;
}
private QueryResultWithSchemaResp mockQueryResult() {
QueryResultWithSchemaResp queryResultWithSchemaResp = new QueryResultWithSchemaResp();
private SemanticQueryResp mockQueryResult() {
SemanticQueryResp semanticQueryResp = new SemanticQueryResp();
List<Map<String, Object>> resultList = Lists.newArrayList();
resultList.add(createMap("2023-10-11", "tom", "hr", "1"));
resultList.add(createMap("2023-10-12", "alice", "sales", "2"));
resultList.add(createMap("2023-10-13", "jack", "sales", "3"));
resultList.add(createMap("2023-10-14", "luck", "market", "4"));
resultList.add(createMap("2023-10-15", "tom", "hr", "5"));
queryResultWithSchemaResp.setResultList(resultList);
return queryResultWithSchemaResp;
semanticQueryResp.setResultList(resultList);
return semanticQueryResp;
}
private static Map<String, Object> createMap(String sysImpDate, String d1, String d2, String m1) {