topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 05:43:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][Headless] Fix the caching issue in struct queries and add headless test cases. (#689)

Browse Source
This commit is contained in:
lexluo09
2024-01-25 11:31:15 +08:00
committed by GitHub
parent 922201c181
commit 841db25198
23 changed files with 214 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java
View File

@@ -42,32 +42,28 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter {
@Override @Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QuerySqlReq querySQLReq = new QuerySqlReq(); QuerySqlReq querySqlReq = new QuerySqlReq();
querySQLReq.setSql(queryStructReq.getCorrectS2SQL()); querySqlReq.setSql(queryStructReq.getCorrectS2SQL());
querySQLReq.setModelIds(queryStructReq.getModelIdSet()); querySqlReq.setModelIds(queryStructReq.getModelIdSet());
querySQLReq.setParams(new ArrayList<>()); querySqlReq.setParams(new ArrayList<>());
return queryByS2SQL(querySQLReq, user); return queryByS2SQL(querySqlReq, user);
} }
queryService = ContextUtils.getBean(QueryService.class); queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryByReq(queryStructReq, user); return queryService.queryByReq(queryStructReq, user);
} }
@Override @Override
@SneakyThrows
public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) {
try {
queryService = ContextUtils.getBean(QueryService.class); queryService = ContextUtils.getBean(QueryService.class);
return queryService.queryByReq(queryMultiStructReq, user); return queryService.queryByReq(queryMultiStructReq, user);
} catch (Exception e) {
log.info("queryByMultiStruct has an exception:{}", e);
}
return null;
} }
@Override @Override
@SneakyThrows @SneakyThrows
public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) { public SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user) {
queryService = ContextUtils.getBean(QueryService.class); queryService = ContextUtils.getBean(QueryService.class);
SemanticQueryResp object = queryService.queryByReq(querySQLReq, user); SemanticQueryResp object = queryService.queryByReq(querySqlReq, user);
return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class); return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class);
} }

14
chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java
View File

@@ -72,11 +72,11 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
@Override @Override
public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) {
if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) {
QuerySqlReq querySQLReq = new QuerySqlReq(); QuerySqlReq querySqlReq = new QuerySqlReq();
querySQLReq.setSql(queryStructReq.getCorrectS2SQL()); querySqlReq.setSql(queryStructReq.getCorrectS2SQL());
querySQLReq.setModelIds(queryStructReq.getModelIdSet()); querySqlReq.setModelIds(queryStructReq.getModelIdSet());
querySQLReq.setParams(new ArrayList<>()); querySqlReq.setParams(new ArrayList<>());
return queryByS2SQL(querySQLReq, user); return queryByS2SQL(querySqlReq, user);
} }
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
@@ -94,10 +94,10 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter {
} }
@Override @Override
public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) { public SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user) {
DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class);
return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(), return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(),
new Gson().toJson(querySQLReq)); new Gson().toJson(querySqlReq));
} }
public SemanticQueryResp searchByRestTemplate(String url, String jsonReq) { public SemanticQueryResp searchByRestTemplate(String url, String jsonReq) {

2
chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java
View File

@@ -38,7 +38,7 @@ public interface SemanticInterpreter {
SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user);
SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user); SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user);
SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);

6
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java
View File

@@ -121,9 +121,9 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable {
} }
QueryStructReq queryStructReq = convertQueryStruct(); QueryStructReq queryStructReq = convertQueryStruct();
convertBizNameToName(semanticSchema, queryStructReq); convertBizNameToName(semanticSchema, queryStructReq);
QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq);
parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql()); parseInfo.getSqlInfo().setS2SQL(querySqlReq.getSql());
parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql()); parseInfo.getSqlInfo().setCorrectS2SQL(querySqlReq.getSql());
} }
} }

4
chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java
View File

@@ -42,8 +42,8 @@ public class LLMSqlQuery extends LLMSemanticQuery {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); String querySql = parseInfo.getSqlInfo().getCorrectS2SQL();
QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds()); QuerySqlReq querySqlReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds());
SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySQLReq, user); SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySqlReq, user);
log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql);

2
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java
View File

@@ -159,7 +159,7 @@ public class DictQueryHelper {
private QueryStructReq generateQueryStructCmd(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict) { private QueryStructReq generateQueryStructCmd(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict) {
QueryStructReq queryStructCmd = new QueryStructReq(); QueryStructReq queryStructCmd = new QueryStructReq();
queryStructCmd.setModelId(modelId); queryStructCmd.addModelId(modelId);
queryStructCmd.setGroups(Arrays.asList(dim4Dict.getBizName())); queryStructCmd.setGroups(Arrays.asList(dim4Dict.getBizName()));
List<Filter> filters = generateFilters(dim4Dict, queryStructCmd); List<Filter> filters = generateFilters(dim4Dict, queryStructCmd);

8
chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java
View File

@@ -134,12 +134,12 @@ public class QueryReqBuilder {
* @return * @return
*/ */
public static QuerySqlReq buildS2SQLReq(String querySql, Set<Long> modelIds) { public static QuerySqlReq buildS2SQLReq(String querySql, Set<Long> modelIds) {
QuerySqlReq querySQLReq = new QuerySqlReq(); QuerySqlReq querySqlReq = new QuerySqlReq();
if (Objects.nonNull(querySql)) { if (Objects.nonNull(querySql)) {
querySQLReq.setSql(querySql); querySqlReq.setSql(querySql);
} }
querySQLReq.setModelIds(modelIds); querySqlReq.setModelIds(modelIds);
return querySQLReq; return querySqlReq;
} }
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java
View File

@@ -692,7 +692,7 @@ public class QueryServiceImpl implements QueryService {
dateConf.setPeriod("DAY"); dateConf.setPeriod("DAY");
queryStructReq.setDateInfo(dateConf); queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L); queryStructReq.setLimit(20L);
queryStructReq.setModelId(dimensionValueReq.getModelId()); queryStructReq.addModelId(dimensionValueReq.getModelId());
queryStructReq.setQueryType(QueryType.ID); queryStructReq.setQueryType(QueryType.ID);
List<String> groups = new ArrayList<>(); List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName()); groups.add(dimensionValueReq.getBizName());

10
chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java
View File

@@ -29,7 +29,7 @@ class QueryReqBuilderTest {
void buildS2SQLReq() { void buildS2SQLReq() {
init(); init();
QueryStructReq queryStructReq = new QueryStructReq(); QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setModelId(1L); queryStructReq.addModelId(1L);
queryStructReq.setQueryType(QueryType.METRIC); queryStructReq.setQueryType(QueryType.METRIC);
queryStructReq.setModelName("内容库"); queryStructReq.setModelName("内容库");
@@ -51,18 +51,18 @@ class QueryReqBuilderTest {
orders.add(order); orders.add(order);
queryStructReq.setOrders(orders); queryStructReq.setOrders(orders);
QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals( Assert.assertEquals(
"SELECT department, SUM(pv) AS pv FROM 内容库 " "SELECT department, SUM(pv) AS pv FROM 内容库 "
+ "WHERE (sys_imp_date IN ('2023-08-01')) GROUP " + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP "
+ "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql()); + "BY department ORDER BY uv LIMIT 2000", querySqlReq.getSql());
queryStructReq.setQueryType(QueryType.TAG); queryStructReq.setQueryType(QueryType.TAG);
querySQLReq = queryStructReq.convert(queryStructReq); querySqlReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals( Assert.assertEquals(
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "ORDER BY uv LIMIT 2000", + "ORDER BY uv LIMIT 2000",
querySQLReq.getSql()); querySqlReq.getSql());
} }

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -272,7 +272,8 @@ public class QueryStructReq extends SemanticQueryReq {
} }
public String getModelName() { public String getModelName() {
return Objects.nonNull(modelName) ? modelName : "m_" + String.valueOf(StringUtils.join(modelIds, "_")); return Objects.nonNull(modelName) ? modelName :
Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_");
} }
} }

5
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java
View File

@@ -19,13 +19,12 @@ public abstract class SemanticQueryReq {
protected boolean needAuth = true; protected boolean needAuth = true;
protected Set<Long> modelIds; protected Set<Long> modelIds = new HashSet<>();
protected List<Param> params = new ArrayList<>(); protected List<Param> params = new ArrayList<>();
protected Cache cacheInfo = new Cache(); protected Cache cacheInfo = new Cache();
public void setModelId(Long modelId) { public void addModelId(Long modelId) {
modelIds = new HashSet<>();
modelIds.add(modelId); modelIds.add(modelId);
} }

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java vendored
View File

@@ -20,8 +20,7 @@ public class DefaultQueryCache implements QueryCache {
@Autowired @Autowired
private CacheManager cacheManager; private CacheManager cacheManager;
public Object query(SemanticQueryReq semanticQueryReq) { public Object query(SemanticQueryReq semanticQueryReq, String cacheKey) {
String cacheKey = getCacheKey(semanticQueryReq);
if (isCache(semanticQueryReq)) { if (isCache(semanticQueryReq)) {
Object result = cacheManager.get(cacheKey); Object result = cacheManager.get(cacheKey);
log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq); log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq);
@@ -30,15 +29,14 @@ public class DefaultQueryCache implements QueryCache {
return null; return null;
} }
public Boolean put(SemanticQueryReq semanticQueryReq, Object value) { public Boolean put(String cacheKey, Object value) {
if (cacheEnable && Objects.nonNull(value)) { if (cacheEnable && Objects.nonNull(value)) {
String key = getCacheKey(semanticQueryReq); CompletableFuture.supplyAsync(() -> cacheManager.put(cacheKey, value))
CompletableFuture.supplyAsync(() -> cacheManager.put(key, value))
.exceptionally(exception -> { .exceptionally(exception -> {
log.warn("exception:", exception); log.warn("exception:", exception);
return null; return null;
}); });
log.info("add record to cache, key:{}", key); log.info("add record to cache, key:{}", cacheKey);
return true; return true;
} }
return false; return false;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java vendored
View File

@@ -5,9 +5,9 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
public interface QueryCache { public interface QueryCache {
Object query(SemanticQueryReq semanticQueryReq); Object query(SemanticQueryReq semanticQueryReq, String cacheKey);
Boolean put(SemanticQueryReq semanticQueryReq, Object value); Boolean put(String cacheKey, Object value);
String getCacheKey(SemanticQueryReq semanticQueryReq); String getCacheKey(SemanticQueryReq semanticQueryReq);

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
View File

@@ -84,9 +84,9 @@ public class DimValueAspect {
public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable { public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
Object[] args = joinPoint.getArgs(); Object[] args = joinPoint.getArgs();
QuerySqlReq querySQLReq = (QuerySqlReq) args[0]; QuerySqlReq querySqlReq = (QuerySqlReq) args[0];
MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySQLReq.getModelIds())); MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySqlReq.getModelIds()));
String sql = querySQLReq.getSql(); String sql = querySqlReq.getSql();
log.info("correctorSql before replacing:{}", sql); log.info("correctorSql before replacing:{}", sql);
List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql); List<FieldExpression> fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql);
List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter); List<DimensionResp> dimensions = dimensionService.getDimensions(metaFilter);
@@ -117,7 +117,7 @@ public class DimValueAspect {
} }
sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap); sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap);
log.info("correctorSql after replacing:{}", sql); log.info("correctorSql after replacing:{}", sql);
querySQLReq.setSql(sql); querySqlReq.setSql(sql);
Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions); Map<String, Map<String, String>> techNameToBizName = getTechNameToBizName(dimensions);
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();

26
headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java
View File

@@ -99,13 +99,13 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
} }
private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySQLReq) private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySqlReq)
throws Throwable { throws Throwable {
Object[] objects = joinPoint.getArgs(); Object[] objects = joinPoint.getArgs();
User user = (User) objects[1]; User user = (User) objects[1];
List<Long> modelIds = querySQLReq.getModelIds(); List<Long> modelIds = querySqlReq.getModelIds();
// fetch data permission meta information // fetch data permission meta information
Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySQLReq, user); Set<String> res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, user);
log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege); log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege);
Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds); Set<String> sensitiveResByModel = getHighSensitiveColsByModelId(modelIds);
@@ -119,10 +119,10 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds); Set<String> resAuthSet = getAuthResNameSet(authorizedResource, modelIds);
// if sensitive fields without permission are involved in filter, thrown an exception // if sensitive fields without permission are involved in filter, thrown an exception
doFilterCheckLogic(querySQLReq, resAuthSet, sensitiveResReq); doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq);
// row permission pre-filter // row permission pre-filter
doRowPermission(querySQLReq, authorizedResource); doRowPermission(querySqlReq, authorizedResource);
// proceed // proceed
SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed();
@@ -144,14 +144,14 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
return queryResultAfterDesensitization; return queryResultAfterDesensitization;
} }
private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set<String> resAuthName, private void doFilterCheckLogic(QuerySqlReq querySqlReq, Set<String> resAuthName,
Set<String> sensitiveResReq) { Set<String> sensitiveResReq) {
Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq); Set<String> resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySqlReq);
Set<String> need2Apply = resFilterSet.stream() Set<String> need2Apply = resFilterSet.stream()
.filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet());
Set<String> nameCnSet = new HashSet<>(); Set<String> nameCnSet = new HashSet<>();
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds()); List<Long> modelIds = Lists.newArrayList(querySqlReq.getModelIds());
ModelFilter modelFilter = new ModelFilter(); ModelFilter modelFilter = new ModelFilter();
modelFilter.setModelIds(modelIds); modelFilter.setModelIds(modelIds);
List<ModelResp> modelInfos = modelService.getModelList(modelFilter); List<ModelResp> modelInfos = modelService.getModelList(modelFilter);
@@ -249,7 +249,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
return false; return false;
} }
private void doRowPermission(QuerySqlReq querySQLReq, AuthorizedResourceResp authorizedResource) { private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) {
log.debug("start doRowPermission logic"); log.debug("start doRowPermission logic");
StringJoiner joiner = new StringJoiner(" OR "); StringJoiner joiner = new StringJoiner(" OR ");
List<String> dimensionFilters = new ArrayList<>(); List<String> dimensionFilters = new ArrayList<>();
@@ -271,10 +271,10 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect {
try { try {
Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) ");
if (StringUtils.isNotEmpty(joiner.toString())) { if (StringUtils.isNotEmpty(joiner.toString())) {
String sql = SqlParserAddHelper.addWhere(querySQLReq.getSql(), expression); String sql = SqlParserAddHelper.addWhere(querySqlReq.getSql(), expression);
log.info("before doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql()); log.info("before doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());
querySQLReq.setSql(sql); querySqlReq.setSql(sql);
log.info("after doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql()); log.info("after doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql());
} }
} catch (JSQLParserException jsqlParserException) { } catch (JSQLParserException jsqlParserException) {
log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); log.info("jsqlParser has an exception:{}", jsqlParserException.toString());

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java
View File

@@ -42,11 +42,11 @@ public class QueryController {
private DownloadService downloadService; private DownloadService downloadService;
@PostMapping("/sql") @PostMapping("/sql")
public Object queryBySql(@RequestBody QuerySqlReq querySQLReq, public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response) throws Exception { HttpServletResponse response) throws Exception {
User user = UserHolder.findUser(request, response); User user = UserHolder.findUser(request, response);
return queryService.queryByReq(querySQLReq, user); return queryService.queryByReq(querySqlReq, user);
} }
@PostMapping("/struct") @PostMapping("/struct")

29
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
View File

@@ -117,7 +117,8 @@ public class QueryServiceImpl implements QueryService {
//1.initStatInfo //1.initStatInfo
statUtils.initStatInfo(queryReq, user); statUtils.initStatInfo(queryReq, user);
//2.query from cache //2.query from cache
Object query = queryCache.query(queryReq); String cacheKey = queryCache.getCacheKey(queryReq);
Object query = queryCache.query(queryReq, cacheKey);
if (Objects.nonNull(query)) { if (Objects.nonNull(query)) {
return (SemanticQueryResp) query; return (SemanticQueryResp) query;
} }
@@ -126,10 +127,10 @@ public class QueryServiceImpl implements QueryService {
QueryStatement queryStatement = buildQueryStatement(queryReq, user); QueryStatement queryStatement = buildQueryStatement(queryReq, user);
SemanticQueryResp result = query(queryStatement); SemanticQueryResp result = query(queryStatement);
//4 reset cache and set stateInfo //4 reset cache and set stateInfo
Boolean setCacheSuccess = queryCache.put(queryReq, result); Boolean setCacheSuccess = queryCache.put(cacheKey, result);
if (setCacheSuccess) { if (setCacheSuccess) {
// if result is not null, update cache data // if result is not null, update cache data
statUtils.updateResultCacheKey(queryCache.getCacheKey(queryReq)); statUtils.updateResultCacheKey(cacheKey);
} }
if (Objects.isNull(result)) { if (Objects.isNull(result)) {
state = TaskStatusEnum.ERROR; state = TaskStatusEnum.ERROR;
@@ -144,15 +145,15 @@ public class QueryServiceImpl implements QueryService {
} }
} }
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySQLReq, User user) throws Exception { private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception {
ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
filter.setModelIds(querySQLReq.getModelIds()); filter.setModelIds(querySqlReq.getModelIds());
SchemaService schemaService = ContextUtils.getBean(SchemaService.class); SchemaService schemaService = ContextUtils.getBean(SchemaService.class);
List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user); List<ModelSchemaResp> modelSchemaResps = schemaService.fetchModelSchema(filter, user);
QueryStatement queryStatement = queryReqConverter.convert(querySQLReq, modelSchemaResps); QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, modelSchemaResps);
queryStatement.setModelIds(querySQLReq.getModelIds()); queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setEnableOptimize(queryUtils.enableOptimize());
SemanticModel semanticModel = semanticSchemaManager.get(querySQLReq.getModelIdStr()); SemanticModel semanticModel = semanticSchemaManager.get(querySqlReq.getModelIdStr());
queryStatement.setSemanticModel(semanticModel); queryStatement.setSemanticModel(semanticModel);
return queryStatement; return queryStatement;
} }
@@ -200,8 +201,8 @@ public class QueryServiceImpl implements QueryService {
@Override @Override
@SneakyThrows @SneakyThrows
public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) {
QuerySqlReq querySQLReq = buildQuerySqlReq(queryDimValueReq); QuerySqlReq querySqlReq = buildQuerySqlReq(queryDimValueReq);
return queryByReq(querySQLReq, user); return queryByReq(querySqlReq, user);
} }
@Override @Override
@@ -312,7 +313,7 @@ public class QueryServiceImpl implements QueryService {
} }
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) { private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
QuerySqlReq querySQLReq = new QuerySqlReq(); QuerySqlReq querySqlReq = new QuerySqlReq();
List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); List<ModelResp> modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId()));
DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(), DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(),
queryDimValueReq.getModelId()); queryDimValueReq.getModelId());
@@ -324,9 +325,9 @@ public class QueryServiceImpl implements QueryService {
queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(),
queryDimValueReq.getDateInfo().getEndDate()); queryDimValueReq.getDateInfo().getEndDate());
} }
querySQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId()));
querySQLReq.setSql(sql); querySqlReq.setSql(sql);
return querySQLReq; return querySqlReq;
} }
private QueryStatement plan(QueryStatement queryStatement) throws Exception { private QueryStatement plan(QueryStatement queryStatement) throws Exception {

30
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -62,7 +62,7 @@ public class QueryReqConverter {
@Autowired @Autowired
private Catalog catalog; private Catalog catalog;
public QueryStatement convert(QuerySqlReq querySQLReq, public QueryStatement convert(QuerySqlReq querySqlReq,
List<ModelSchemaResp> modelSchemaResps) throws Exception { List<ModelSchemaResp> modelSchemaResps) throws Exception {
if (CollectionUtils.isEmpty(modelSchemaResps)) { if (CollectionUtils.isEmpty(modelSchemaResps)) {
@@ -71,18 +71,18 @@ public class QueryReqConverter {
Map<Long, ModelSchemaResp> modelSchemaRespMap = modelSchemaResps.stream() Map<Long, ModelSchemaResp> modelSchemaRespMap = modelSchemaResps.stream()
.collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp)); .collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp));
//1.convert name to bizName //1.convert name to bizName
convertNameToBizName(querySQLReq, modelSchemaResps); convertNameToBizName(querySqlReq, modelSchemaResps);
//2.functionName corrector //2.functionName corrector
functionNameCorrector(querySQLReq); functionNameCorrector(querySqlReq);
//3.correct tableName //3.correct tableName
correctTableName(querySQLReq); correctTableName(querySqlReq);
String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql()); String tableName = SqlParserSelectHelper.getTableName(querySqlReq.getSql());
if (StringUtils.isEmpty(tableName)) { if (StringUtils.isEmpty(tableName)) {
return new QueryStatement(); return new QueryStatement();
} }
//4.build MetricTables //4.build MetricTables
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); List<String> allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql());
List<String> metrics = getMetrics(modelSchemaResps, allFields); List<String> metrics = getMetrics(modelSchemaResps, allFields);
QueryStructReq queryStructReq = new QueryStructReq(); QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable(); MetricTable metricTable = new MetricTable();
@@ -96,7 +96,7 @@ public class QueryReqConverter {
// if metric empty , fill model default // if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) { if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>()); metricTable.setMetrics(new ArrayList<>());
for (Long modelId : querySQLReq.getModelIds()) { for (Long modelId : querySqlReq.getModelIds()) {
ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId); ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId);
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName())); metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName()));
} }
@@ -105,27 +105,27 @@ public class QueryReqConverter {
metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect(
Collectors.toList())); Collectors.toList()));
} }
AggOption aggOption = getAggOption(querySQLReq); AggOption aggOption = getAggOption(querySqlReq);
metricTable.setAggOption(aggOption); metricTable.setAggOption(aggOption);
List<MetricTable> tables = new ArrayList<>(); List<MetricTable> tables = new ArrayList<>();
tables.add(metricTable); tables.add(metricTable);
//4.build ParseSqlReq //4.build ParseSqlReq
ParseSqlReq result = new ParseSqlReq(); ParseSqlReq result = new ParseSqlReq();
BeanUtils.copyProperties(querySQLReq, result); BeanUtils.copyProperties(querySqlReq, result);
result.setRootPath(querySQLReq.getModelIdStr()); result.setRootPath(querySqlReq.getModelIdStr());
result.setTables(tables); result.setTables(tables);
DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0)); DatabaseResp database = catalog.getDatabaseByModelId(querySqlReq.getModelIds().get(0));
if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()),
database.getVersion())) { database.getVersion())) {
result.setSupportWith(false); result.setSupportWith(false);
result.setWithAlias(false); result.setWithAlias(false);
} }
//5. do deriveMetric //5. do deriveMetric
generateDerivedMetric(querySQLReq.getModelIds(), modelSchemaResps, aggOption, result); generateDerivedMetric(querySqlReq.getModelIds(), modelSchemaResps, aggOption, result);
//6.physicalSql by ParseSqlReq //6.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql()));
queryStructReq.setModelIds(new HashSet<>(querySQLReq.getModelIds())); queryStructReq.setModelIds(new HashSet<>(querySqlReq.getModelIds()));
queryStructReq.setQueryType(getQueryType(aggOption)); queryStructReq.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructReq[{}]", queryStructReq); log.info("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
@@ -133,7 +133,7 @@ public class QueryReqConverter {
queryStatement.setParseSqlReq(result); queryStatement.setParseSqlReq(result);
queryStatement.setIsS2SQL(true); queryStatement.setIsS2SQL(true);
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setModelIds(querySQLReq.getModelIds()); queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setEnableLimitWrapper(limitWrapper); queryStatement.setEnableLimitWrapper(limitWrapper);
return queryStatement; return queryStatement;

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java
View File

@@ -129,8 +129,8 @@ public class QueryStructUtils {
return resNameEnSet; return resNameEnSet;
} }
public Set<String> getResName(QuerySqlReq querySQLReq) { public Set<String> getResName(QuerySqlReq querySqlReq) {
Set<String> resNameSet = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()) Set<String> resNameSet = SqlParserSelectHelper.getAllFields(querySqlReq.getSql())
.stream().collect(Collectors.toSet()); .stream().collect(Collectors.toSet());
return resNameSet; return resNameSet;
} }
@@ -140,11 +140,11 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }
public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySQLReq, User user) { public Set<String> getResNameEnExceptInternalCol(QuerySqlReq querySqlReq, User user) {
Set<String> resNameSet = getResName(querySQLReq); Set<String> resNameSet = getResName(querySqlReq);
Set<String> resNameEnSet = new HashSet<>(); Set<String> resNameEnSet = new HashSet<>();
ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq();
List<Long> modelIds = Lists.newArrayList(querySQLReq.getModelIds()); List<Long> modelIds = Lists.newArrayList(querySqlReq.getModelIds());
filter.setModelIds(modelIds); filter.setModelIds(modelIds);
List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user); List<ModelSchemaResp> modelSchemaRespList = schemaService.fetchModelSchema(filter, user);
if (!CollectionUtils.isEmpty(modelSchemaRespList)) { if (!CollectionUtils.isEmpty(modelSchemaRespList)) {
@@ -175,8 +175,8 @@ public class QueryStructUtils {
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }
public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySQLReq) { public Set<String> getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) {
String sql = querySQLReq.getSql(); String sql = querySqlReq.getSql();
Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); Set<String> resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet());
return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());
} }

19
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java
View File

@@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.QueryTypeBack;
import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.pojo.QueryStat;
import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
@@ -93,13 +94,17 @@ public class StatUtils {
if (semanticQueryReq instanceof QueryStructReq) { if (semanticQueryReq instanceof QueryStructReq) {
initStructStatInfo((QueryStructReq) semanticQueryReq, facadeUser); initStructStatInfo((QueryStructReq) semanticQueryReq, facadeUser);
} }
if (semanticQueryReq instanceof QueryMultiStructReq) {
QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0);
initStructStatInfo(queryStructCmd, facadeUser);
}
} }
public void initSqlStatInfo(QuerySqlReq querySQLReq, User facadeUser) { public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) {
QueryStat queryStatInfo = new QueryStat(); QueryStat queryStatInfo = new QueryStat();
List<String> allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); List<String> allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql());
queryStatInfo.setModelId(querySQLReq.getModelIds().get(0)); queryStatInfo.setModelId(querySqlReq.getModelIds().get(0));
ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySQLReq.getModelIds().get(0)); ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySqlReq.getModelIds().get(0));
List<String> dimensions = new ArrayList<>(); List<String> dimensions = new ArrayList<>();
List<String> metrics = new ArrayList<>(); List<String> metrics = new ArrayList<>();
@@ -111,12 +116,12 @@ public class StatUtils {
String userName = getUserName(facadeUser); String userName = getUserName(facadeUser);
try { try {
queryStatInfo.setTraceId("") queryStatInfo.setTraceId("")
.setModelId(querySQLReq.getModelIds().get(0)) .setModelId(querySqlReq.getModelIds().get(0))
.setUser(userName) .setUser(userName)
.setQueryType(QueryType.SQL.getValue()) .setQueryType(QueryType.SQL.getValue())
.setQueryTypeBack(QueryTypeBack.NORMAL.getState()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState())
.setQuerySqlCmd(querySQLReq.toString()) .setQuerySqlCmd(querySqlReq.toString())
.setQuerySqlCmdMd5(DigestUtils.md5Hex(querySQLReq.toString())) .setQuerySqlCmdMd5(DigestUtils.md5Hex(querySqlReq.toString()))
.setStartTime(System.currentTimeMillis()) .setStartTime(System.currentTimeMillis())
.setUseResultCache(true) .setUseResultCache(true)
.setUseSqlCache(true) .setUseSqlCache(true)

7
launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/BaseTest.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.integration;
import com.tencent.supersonic.StandaloneLauncher; import com.tencent.supersonic.StandaloneLauncher;
import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.server.service.QueryService; import com.tencent.supersonic.headless.server.service.QueryService;
import java.util.HashSet; import java.util.HashSet;
@@ -29,7 +30,11 @@ public class BaseTest {
return queryService.queryByReq(buildQuerySqlReq(sql), user); return queryService.queryByReq(buildQuerySqlReq(sql), user);
} }
protected QuerySqlReq buildQuerySqlReq(String sql) { protected SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception {
return queryService.queryByReq(queryReq, user);
}
protected SemanticQueryReq buildQuerySqlReq(String sql) {
QuerySqlReq querySqlCmd = new QuerySqlReq(); QuerySqlReq querySqlCmd = new QuerySqlReq();
querySqlCmd.setSql(sql); querySqlCmd.setSql(sql);
Set<Long> modelIds = new HashSet<>(); Set<Long> modelIds = new HashSet<>();

10
launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/ExplainTest.java Normal file
View File

@@ -0,0 +1,10 @@
package com.tencent.supersonic.headless.integration;
import org.junit.Test;
public class ExplainTest extends BaseTest {
@Test
public void testSumExplain() {
}
}

86
launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/QueryByStructTest.java Normal file
View File

@@ -0,0 +1,86 @@
package com.tencent.supersonic.headless.integration;
import static java.time.LocalDate.now;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.junit.Test;
public class QueryByStructTest extends BaseTest {
@Test
public void testSumQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(null);
SemanticQueryResp semanticQueryResp = queryByReq(queryStructReq, User.getFakeUser());
assertEquals(1, semanticQueryResp.getColumns().size());
QueryColumn queryColumn = semanticQueryResp.getColumns().get(0);
assertEquals("访问次数", queryColumn.getName());
assertEquals(1, semanticQueryResp.getResultList().size());
}
@Test
public void testGroupByQuery() throws Exception {
QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department"));
SemanticQueryResp result = queryByReq(queryStructReq, User.getFakeUser());
assertEquals(2, result.getColumns().size());
QueryColumn firstColumn = result.getColumns().get(0);
QueryColumn secondColumn = result.getColumns().get(1);
assertEquals("部门", firstColumn.getName());
assertEquals("访问次数", secondColumn.getName());
assertNotNull(result.getResultList().size());
}
@Test
public void testCacheQuery() throws Exception {
QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department"));
QueryStructReq queryStructReq2 = buildQueryStructReq(Arrays.asList("department"));
SemanticQueryResp result1 = queryByReq(queryStructReq1, User.getFakeUser());
SemanticQueryResp result2 = queryByReq(queryStructReq2, User.getFakeUser());
assertEquals(result1, result2);
}
private QueryStructReq buildQueryStructReq(List<String> groups) {
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.addModelId(1L);
queryStructReq.addModelId(2L);
queryStructReq.addModelId(3L);
queryStructReq.setQueryType(QueryType.METRIC);
Aggregator aggregator = new Aggregator();
aggregator.setFunc(AggOperatorEnum.SUM);
aggregator.setColumn("pv");
queryStructReq.setAggregators(Arrays.asList(aggregator));
if (CollectionUtils.isNotEmpty(groups)) {
queryStructReq.setGroups(groups);
queryStructReq.setGroups(Arrays.asList("department"));
}
DateConf dateConf = new DateConf();
dateConf.setDateMode(DateMode.BETWEEN);
dateConf.setStartDate(now().plusDays(-1).toString());
dateConf.setEndDate(now().plusDays(-10).toString());
queryStructReq.setDateInfo(dateConf);
List<Order> orders = new ArrayList<>();
Order order = new Order();
order.setColumn("pv");
orders.add(order);
queryStructReq.setOrders(orders);
return queryStructReq;
}
}