diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java index fbb7e54e9..b81669f8f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/LocalSemanticInterpreter.java @@ -42,32 +42,28 @@ public class LocalSemanticInterpreter extends BaseSemanticInterpreter { @Override public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { - QuerySqlReq querySQLReq = new QuerySqlReq(); - querySQLReq.setSql(queryStructReq.getCorrectS2SQL()); - querySQLReq.setModelIds(queryStructReq.getModelIdSet()); - querySQLReq.setParams(new ArrayList<>()); - return queryByS2SQL(querySQLReq, user); + QuerySqlReq querySqlReq = new QuerySqlReq(); + querySqlReq.setSql(queryStructReq.getCorrectS2SQL()); + querySqlReq.setModelIds(queryStructReq.getModelIdSet()); + querySqlReq.setParams(new ArrayList<>()); + return queryByS2SQL(querySqlReq, user); } queryService = ContextUtils.getBean(QueryService.class); return queryService.queryByReq(queryStructReq, user); } @Override + @SneakyThrows public SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user) { - try { - queryService = ContextUtils.getBean(QueryService.class); - return queryService.queryByReq(queryMultiStructReq, user); - } catch (Exception e) { - log.info("queryByMultiStruct has an exception:{}", e); - } - return null; + queryService = ContextUtils.getBean(QueryService.class); + return queryService.queryByReq(queryMultiStructReq, user); } @Override @SneakyThrows - public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) { + public SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user) { queryService = ContextUtils.getBean(QueryService.class); - SemanticQueryResp object = queryService.queryByReq(querySQLReq, user); + SemanticQueryResp object = queryService.queryByReq(querySqlReq, user); return JsonUtil.toObject(JsonUtil.toString(object), SemanticQueryResp.class); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java index ba9e38eb8..916a6561f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/RemoteSemanticInterpreter.java @@ -72,11 +72,11 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { @Override public SemanticQueryResp queryByStruct(QueryStructReq queryStructReq, User user) { if (StringUtils.isNotBlank(queryStructReq.getCorrectS2SQL())) { - QuerySqlReq querySQLReq = new QuerySqlReq(); - querySQLReq.setSql(queryStructReq.getCorrectS2SQL()); - querySQLReq.setModelIds(queryStructReq.getModelIdSet()); - querySQLReq.setParams(new ArrayList<>()); - return queryByS2SQL(querySQLReq, user); + QuerySqlReq querySqlReq = new QuerySqlReq(); + querySqlReq.setSql(queryStructReq.getCorrectS2SQL()); + querySqlReq.setModelIds(queryStructReq.getModelIdSet()); + querySqlReq.setParams(new ArrayList<>()); + return queryByS2SQL(querySqlReq, user); } DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); @@ -94,10 +94,10 @@ public class RemoteSemanticInterpreter extends BaseSemanticInterpreter { } @Override - public SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user) { + public SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user) { DefaultSemanticConfig defaultSemanticConfig = ContextUtils.getBean(DefaultSemanticConfig.class); return searchByRestTemplate(defaultSemanticConfig.getSemanticUrl() + defaultSemanticConfig.getSearchBySqlPath(), - new Gson().toJson(querySQLReq)); + new Gson().toJson(querySqlReq)); } public SemanticQueryResp searchByRestTemplate(String url, String jsonReq) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java index 1bf90f7f1..4d4c0b407 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/semantic/SemanticInterpreter.java @@ -38,7 +38,7 @@ public interface SemanticInterpreter { SemanticQueryResp queryByMultiStruct(QueryMultiStructReq queryMultiStructReq, User user); - SemanticQueryResp queryByS2SQL(QuerySqlReq querySQLReq, User user); + SemanticQueryResp queryByS2SQL(QuerySqlReq querySqlReq, User user); SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java index 0b3558fbf..7bd899a84 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java @@ -121,9 +121,9 @@ public abstract class BaseSemanticQuery implements SemanticQuery, Serializable { } QueryStructReq queryStructReq = convertQueryStruct(); convertBizNameToName(semanticSchema, queryStructReq); - QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); - parseInfo.getSqlInfo().setS2SQL(querySQLReq.getSql()); - parseInfo.getSqlInfo().setCorrectS2SQL(querySQLReq.getSql()); + QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq); + parseInfo.getSqlInfo().setS2SQL(querySqlReq.getSql()); + parseInfo.getSqlInfo().setCorrectS2SQL(querySqlReq.getSql()); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java index 749291e2a..da22871c4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java @@ -42,8 +42,8 @@ public class LLMSqlQuery extends LLMSemanticQuery { long startTime = System.currentTimeMillis(); String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); - QuerySqlReq querySQLReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds()); - SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySQLReq, user); + QuerySqlReq querySqlReq = QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getModel().getModelIds()); + SemanticQueryResp queryResp = semanticInterpreter.queryByS2SQL(querySqlReq, user); log.info("queryByS2SQL cost:{},querySql:{}", System.currentTimeMillis() - startTime, querySql); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java index 56b2ecc15..ebc29fe91 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java @@ -159,7 +159,7 @@ public class DictQueryHelper { private QueryStructReq generateQueryStructCmd(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict) { QueryStructReq queryStructCmd = new QueryStructReq(); - queryStructCmd.setModelId(modelId); + queryStructCmd.addModelId(modelId); queryStructCmd.setGroups(Arrays.asList(dim4Dict.getBizName())); List filters = generateFilters(dim4Dict, queryStructCmd); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java index ed72fa06c..09ada33d4 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java @@ -134,12 +134,12 @@ public class QueryReqBuilder { * @return */ public static QuerySqlReq buildS2SQLReq(String querySql, Set modelIds) { - QuerySqlReq querySQLReq = new QuerySqlReq(); + QuerySqlReq querySqlReq = new QuerySqlReq(); if (Objects.nonNull(querySql)) { - querySQLReq.setSql(querySql); + querySqlReq.setSql(querySql); } - querySQLReq.setModelIds(modelIds); - return querySQLReq; + querySqlReq.setModelIds(modelIds); + return querySqlReq; } private static List getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java index f437de221..d28da9c09 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java @@ -692,7 +692,7 @@ public class QueryServiceImpl implements QueryService { dateConf.setPeriod("DAY"); queryStructReq.setDateInfo(dateConf); queryStructReq.setLimit(20L); - queryStructReq.setModelId(dimensionValueReq.getModelId()); + queryStructReq.addModelId(dimensionValueReq.getModelId()); queryStructReq.setQueryType(QueryType.ID); List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); diff --git a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java index ddf32bead..916ed7ec7 100644 --- a/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java +++ b/chat/server/src/test/java/com/tencent/supersonic/chat/server/utils/QueryReqBuilderTest.java @@ -29,7 +29,7 @@ class QueryReqBuilderTest { void buildS2SQLReq() { init(); QueryStructReq queryStructReq = new QueryStructReq(); - queryStructReq.setModelId(1L); + queryStructReq.addModelId(1L); queryStructReq.setQueryType(QueryType.METRIC); queryStructReq.setModelName("内容库"); @@ -51,18 +51,18 @@ class QueryReqBuilderTest { orders.add(order); queryStructReq.setOrders(orders); - QuerySqlReq querySQLReq = queryStructReq.convert(queryStructReq); + QuerySqlReq querySqlReq = queryStructReq.convert(queryStructReq); Assert.assertEquals( "SELECT department, SUM(pv) AS pv FROM 内容库 " + "WHERE (sys_imp_date IN ('2023-08-01')) GROUP " - + "BY department ORDER BY uv LIMIT 2000", querySQLReq.getSql()); + + "BY department ORDER BY uv LIMIT 2000", querySqlReq.getSql()); queryStructReq.setQueryType(QueryType.TAG); - querySQLReq = queryStructReq.convert(queryStructReq); + querySqlReq = queryStructReq.convert(queryStructReq); Assert.assertEquals( "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " + "ORDER BY uv LIMIT 2000", - querySQLReq.getSql()); + querySqlReq.getSql()); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 911e50907..274caaa55 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -272,7 +272,8 @@ public class QueryStructReq extends SemanticQueryReq { } public String getModelName() { - return Objects.nonNull(modelName) ? modelName : "m_" + String.valueOf(StringUtils.join(modelIds, "_")); + return Objects.nonNull(modelName) ? modelName : + Constants.TABLE_PREFIX + StringUtils.join(modelIds, "_"); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java index 5b310ec39..d424d4928 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java @@ -19,13 +19,12 @@ public abstract class SemanticQueryReq { protected boolean needAuth = true; - protected Set modelIds; + protected Set modelIds = new HashSet<>(); protected List params = new ArrayList<>(); protected Cache cacheInfo = new Cache(); - public void setModelId(Long modelId) { - modelIds = new HashSet<>(); + public void addModelId(Long modelId) { modelIds.add(modelId); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java index 8d6884e47..908e91f36 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/DefaultQueryCache.java @@ -20,8 +20,7 @@ public class DefaultQueryCache implements QueryCache { @Autowired private CacheManager cacheManager; - public Object query(SemanticQueryReq semanticQueryReq) { - String cacheKey = getCacheKey(semanticQueryReq); + public Object query(SemanticQueryReq semanticQueryReq, String cacheKey) { if (isCache(semanticQueryReq)) { Object result = cacheManager.get(cacheKey); log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq); @@ -30,15 +29,14 @@ public class DefaultQueryCache implements QueryCache { return null; } - public Boolean put(SemanticQueryReq semanticQueryReq, Object value) { + public Boolean put(String cacheKey, Object value) { if (cacheEnable && Objects.nonNull(value)) { - String key = getCacheKey(semanticQueryReq); - CompletableFuture.supplyAsync(() -> cacheManager.put(key, value)) + CompletableFuture.supplyAsync(() -> cacheManager.put(cacheKey, value)) .exceptionally(exception -> { log.warn("exception:", exception); return null; }); - log.info("add record to cache, key:{}", key); + log.info("add record to cache, key:{}", cacheKey); return true; } return false; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java index 9eb2f3b05..8fe71782e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java @@ -5,9 +5,9 @@ import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; public interface QueryCache { - Object query(SemanticQueryReq semanticQueryReq); + Object query(SemanticQueryReq semanticQueryReq, String cacheKey); - Boolean put(SemanticQueryReq semanticQueryReq, Object value); + Boolean put(String cacheKey, Object value); String getCacheKey(SemanticQueryReq semanticQueryReq); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index c84e77433..571e882a9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -84,9 +84,9 @@ public class DimValueAspect { public Object handleSqlDimValue(ProceedingJoinPoint joinPoint) throws Throwable { Object[] args = joinPoint.getArgs(); - QuerySqlReq querySQLReq = (QuerySqlReq) args[0]; - MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySQLReq.getModelIds())); - String sql = querySQLReq.getSql(); + QuerySqlReq querySqlReq = (QuerySqlReq) args[0]; + MetaFilter metaFilter = new MetaFilter(Lists.newArrayList(querySqlReq.getModelIds())); + String sql = querySqlReq.getSql(); log.info("correctorSql before replacing:{}", sql); List fieldExpressionList = SqlParserSelectHelper.getWhereExpressions(sql); List dimensions = dimensionService.getDimensions(metaFilter); @@ -117,7 +117,7 @@ public class DimValueAspect { } sql = SqlParserReplaceHelper.replaceValue(sql, filedNameToValueMap); log.info("correctorSql after replacing:{}", sql); - querySQLReq.setSql(sql); + querySqlReq.setSql(sql); Map> techNameToBizName = getTechNameToBizName(dimensions); SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java index 3b4d8d5ce..84d14254d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/S2DataPermissionAspect.java @@ -99,13 +99,13 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); } - private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySQLReq) + private Object checkSqlPermission(ProceedingJoinPoint joinPoint, QuerySqlReq querySqlReq) throws Throwable { Object[] objects = joinPoint.getArgs(); User user = (User) objects[1]; - List modelIds = querySQLReq.getModelIds(); + List modelIds = querySqlReq.getModelIds(); // fetch data permission meta information - Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySQLReq, user); + Set res4Privilege = queryStructUtils.getResNameEnExceptInternalCol(querySqlReq, user); log.info("modelId:{}, res4Privilege:{}", modelIds, res4Privilege); Set sensitiveResByModel = getHighSensitiveColsByModelId(modelIds); @@ -119,10 +119,10 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { Set resAuthSet = getAuthResNameSet(authorizedResource, modelIds); // if sensitive fields without permission are involved in filter, thrown an exception - doFilterCheckLogic(querySQLReq, resAuthSet, sensitiveResReq); + doFilterCheckLogic(querySqlReq, resAuthSet, sensitiveResReq); // row permission pre-filter - doRowPermission(querySQLReq, authorizedResource); + doRowPermission(querySqlReq, authorizedResource); // proceed SemanticQueryResp queryResultWithColumns = (SemanticQueryResp) joinPoint.proceed(); @@ -144,14 +144,14 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { return queryResultAfterDesensitization; } - private void doFilterCheckLogic(QuerySqlReq querySQLReq, Set resAuthName, + private void doFilterCheckLogic(QuerySqlReq querySqlReq, Set resAuthName, Set sensitiveResReq) { - Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySQLReq); + Set resFilterSet = queryStructUtils.getFilterResNameEnExceptInternalCol(querySqlReq); Set need2Apply = resFilterSet.stream() .filter(res -> !resAuthName.contains(res) && sensitiveResReq.contains(res)).collect(Collectors.toSet()); Set nameCnSet = new HashSet<>(); - List modelIds = Lists.newArrayList(querySQLReq.getModelIds()); + List modelIds = Lists.newArrayList(querySqlReq.getModelIds()); ModelFilter modelFilter = new ModelFilter(); modelFilter.setModelIds(modelIds); List modelInfos = modelService.getModelList(modelFilter); @@ -249,7 +249,7 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { return false; } - private void doRowPermission(QuerySqlReq querySQLReq, AuthorizedResourceResp authorizedResource) { + private void doRowPermission(QuerySqlReq querySqlReq, AuthorizedResourceResp authorizedResource) { log.debug("start doRowPermission logic"); StringJoiner joiner = new StringJoiner(" OR "); List dimensionFilters = new ArrayList<>(); @@ -271,10 +271,10 @@ public class S2DataPermissionAspect extends AuthCheckBaseAspect { try { Expression expression = CCJSqlParserUtil.parseCondExpression(" ( " + joiner + " ) "); if (StringUtils.isNotEmpty(joiner.toString())) { - String sql = SqlParserAddHelper.addWhere(querySQLReq.getSql(), expression); - log.info("before doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql()); - querySQLReq.setSql(sql); - log.info("after doRowPermission, queryS2SQLReq:{}", querySQLReq.getSql()); + String sql = SqlParserAddHelper.addWhere(querySqlReq.getSql(), expression); + log.info("before doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql()); + querySqlReq.setSql(sql); + log.info("after doRowPermission, queryS2SQLReq:{}", querySqlReq.getSql()); } } catch (JSQLParserException jsqlParserException) { log.info("jsqlParser has an exception:{}", jsqlParserException.toString()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java index f3eb6f727..c08001777 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/QueryController.java @@ -42,11 +42,11 @@ public class QueryController { private DownloadService downloadService; @PostMapping("/sql") - public Object queryBySql(@RequestBody QuerySqlReq querySQLReq, + public Object queryBySql(@RequestBody QuerySqlReq querySqlReq, HttpServletRequest request, HttpServletResponse response) throws Exception { User user = UserHolder.findUser(request, response); - return queryService.queryByReq(querySQLReq, user); + return queryService.queryByReq(querySqlReq, user); } @PostMapping("/struct") diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index e395828bf..e550257ca 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -117,7 +117,8 @@ public class QueryServiceImpl implements QueryService { //1.initStatInfo statUtils.initStatInfo(queryReq, user); //2.query from cache - Object query = queryCache.query(queryReq); + String cacheKey = queryCache.getCacheKey(queryReq); + Object query = queryCache.query(queryReq, cacheKey); if (Objects.nonNull(query)) { return (SemanticQueryResp) query; } @@ -126,10 +127,10 @@ public class QueryServiceImpl implements QueryService { QueryStatement queryStatement = buildQueryStatement(queryReq, user); SemanticQueryResp result = query(queryStatement); //4 reset cache and set stateInfo - Boolean setCacheSuccess = queryCache.put(queryReq, result); + Boolean setCacheSuccess = queryCache.put(cacheKey, result); if (setCacheSuccess) { // if result is not null, update cache data - statUtils.updateResultCacheKey(queryCache.getCacheKey(queryReq)); + statUtils.updateResultCacheKey(cacheKey); } if (Objects.isNull(result)) { state = TaskStatusEnum.ERROR; @@ -144,15 +145,15 @@ public class QueryServiceImpl implements QueryService { } } - private QueryStatement buildSqlQueryStatement(QuerySqlReq querySQLReq, User user) throws Exception { + private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) throws Exception { ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - filter.setModelIds(querySQLReq.getModelIds()); + filter.setModelIds(querySqlReq.getModelIds()); SchemaService schemaService = ContextUtils.getBean(SchemaService.class); List modelSchemaResps = schemaService.fetchModelSchema(filter, user); - QueryStatement queryStatement = queryReqConverter.convert(querySQLReq, modelSchemaResps); - queryStatement.setModelIds(querySQLReq.getModelIds()); + QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, modelSchemaResps); + queryStatement.setModelIds(querySqlReq.getModelIds()); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); - SemanticModel semanticModel = semanticSchemaManager.get(querySQLReq.getModelIdStr()); + SemanticModel semanticModel = semanticSchemaManager.get(querySqlReq.getModelIdStr()); queryStatement.setSemanticModel(semanticModel); return queryStatement; } @@ -200,8 +201,8 @@ public class QueryServiceImpl implements QueryService { @Override @SneakyThrows public SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { - QuerySqlReq querySQLReq = buildQuerySqlReq(queryDimValueReq); - return queryByReq(querySQLReq, user); + QuerySqlReq querySqlReq = buildQuerySqlReq(queryDimValueReq); + return queryByReq(querySqlReq, user); } @Override @@ -312,7 +313,7 @@ public class QueryServiceImpl implements QueryService { } private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) { - QuerySqlReq querySQLReq = new QuerySqlReq(); + QuerySqlReq querySqlReq = new QuerySqlReq(); List modelResps = catalog.getModelList(Lists.newArrayList(queryDimValueReq.getModelId())); DimensionResp dimensionResp = catalog.getDimension(queryDimValueReq.getDimensionBizName(), queryDimValueReq.getModelId()); @@ -324,9 +325,9 @@ public class QueryServiceImpl implements QueryService { queryDimValueReq.getDateInfo().getStartDate(), TimeDimensionEnum.DAY.getName(), queryDimValueReq.getDateInfo().getEndDate()); } - querySQLReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); - querySQLReq.setSql(sql); - return querySQLReq; + querySqlReq.setModelIds(Sets.newHashSet(queryDimValueReq.getModelId())); + querySqlReq.setSql(sql); + return querySqlReq; } private QueryStatement plan(QueryStatement queryStatement) throws Exception { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 531701e0f..6b0522689 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -62,7 +62,7 @@ public class QueryReqConverter { @Autowired private Catalog catalog; - public QueryStatement convert(QuerySqlReq querySQLReq, + public QueryStatement convert(QuerySqlReq querySqlReq, List modelSchemaResps) throws Exception { if (CollectionUtils.isEmpty(modelSchemaResps)) { @@ -71,18 +71,18 @@ public class QueryReqConverter { Map modelSchemaRespMap = modelSchemaResps.stream() .collect(Collectors.toMap(ModelSchemaResp::getId, modelSchemaResp -> modelSchemaResp)); //1.convert name to bizName - convertNameToBizName(querySQLReq, modelSchemaResps); + convertNameToBizName(querySqlReq, modelSchemaResps); //2.functionName corrector - functionNameCorrector(querySQLReq); + functionNameCorrector(querySqlReq); //3.correct tableName - correctTableName(querySQLReq); + correctTableName(querySqlReq); - String tableName = SqlParserSelectHelper.getTableName(querySQLReq.getSql()); + String tableName = SqlParserSelectHelper.getTableName(querySqlReq.getSql()); if (StringUtils.isEmpty(tableName)) { return new QueryStatement(); } //4.build MetricTables - List allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); + List allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql()); List metrics = getMetrics(modelSchemaResps, allFields); QueryStructReq queryStructReq = new QueryStructReq(); MetricTable metricTable = new MetricTable(); @@ -96,7 +96,7 @@ public class QueryReqConverter { // if metric empty , fill model default if (CollectionUtils.isEmpty(metricTable.getMetrics())) { metricTable.setMetrics(new ArrayList<>()); - for (Long modelId : querySQLReq.getModelIds()) { + for (Long modelId : querySqlReq.getModelIds()) { ModelSchemaResp modelSchemaResp = modelSchemaRespMap.get(modelId); metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(modelSchemaResp.getBizName())); } @@ -105,27 +105,27 @@ public class QueryReqConverter { metricTable.getMetrics().stream().map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)).collect( Collectors.toList())); } - AggOption aggOption = getAggOption(querySQLReq); + AggOption aggOption = getAggOption(querySqlReq); metricTable.setAggOption(aggOption); List tables = new ArrayList<>(); tables.add(metricTable); //4.build ParseSqlReq ParseSqlReq result = new ParseSqlReq(); - BeanUtils.copyProperties(querySQLReq, result); + BeanUtils.copyProperties(querySqlReq, result); - result.setRootPath(querySQLReq.getModelIdStr()); + result.setRootPath(querySqlReq.getModelIdStr()); result.setTables(tables); - DatabaseResp database = catalog.getDatabaseByModelId(querySQLReq.getModelIds().get(0)); + DatabaseResp database = catalog.getDatabaseByModelId(querySqlReq.getModelIds().get(0)); if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), database.getVersion())) { result.setSupportWith(false); result.setWithAlias(false); } //5. do deriveMetric - generateDerivedMetric(querySQLReq.getModelIds(), modelSchemaResps, aggOption, result); + generateDerivedMetric(querySqlReq.getModelIds(), modelSchemaResps, aggOption, result); //6.physicalSql by ParseSqlReq - queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); - queryStructReq.setModelIds(new HashSet<>(querySQLReq.getModelIds())); + queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql())); + queryStructReq.setModelIds(new HashSet<>(querySqlReq.getModelIds())); queryStructReq.setQueryType(getQueryType(aggOption)); log.info("QueryReqConverter queryStructReq[{}]", queryStructReq); QueryStatement queryStatement = new QueryStatement(); @@ -133,7 +133,7 @@ public class QueryReqConverter { queryStatement.setParseSqlReq(result); queryStatement.setIsS2SQL(true); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - queryStatement.setModelIds(querySQLReq.getModelIds()); + queryStatement.setModelIds(querySqlReq.getModelIds()); queryStatement.setEnableLimitWrapper(limitWrapper); return queryStatement; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java index afbc8434e..aee4b627e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryStructUtils.java @@ -129,8 +129,8 @@ public class QueryStructUtils { return resNameEnSet; } - public Set getResName(QuerySqlReq querySQLReq) { - Set resNameSet = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()) + public Set getResName(QuerySqlReq querySqlReq) { + Set resNameSet = SqlParserSelectHelper.getAllFields(querySqlReq.getSql()) .stream().collect(Collectors.toSet()); return resNameSet; } @@ -140,11 +140,11 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public Set getResNameEnExceptInternalCol(QuerySqlReq querySQLReq, User user) { - Set resNameSet = getResName(querySQLReq); + public Set getResNameEnExceptInternalCol(QuerySqlReq querySqlReq, User user) { + Set resNameSet = getResName(querySqlReq); Set resNameEnSet = new HashSet<>(); ModelSchemaFilterReq filter = new ModelSchemaFilterReq(); - List modelIds = Lists.newArrayList(querySQLReq.getModelIds()); + List modelIds = Lists.newArrayList(querySqlReq.getModelIds()); filter.setModelIds(modelIds); List modelSchemaRespList = schemaService.fetchModelSchema(filter, user); if (!CollectionUtils.isEmpty(modelSchemaRespList)) { @@ -175,8 +175,8 @@ public class QueryStructUtils { return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } - public Set getFilterResNameEnExceptInternalCol(QuerySqlReq querySQLReq) { - String sql = querySQLReq.getSql(); + public Set getFilterResNameEnExceptInternalCol(QuerySqlReq querySqlReq) { + String sql = querySqlReq.getSql(); Set resNameEnSet = SqlParserSelectHelper.getWhereFields(sql).stream().collect(Collectors.toSet()); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index d36138a32..e7dcae4ca 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.QueryTypeBack; import com.tencent.supersonic.headless.api.pojo.QueryStat; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.ItemUseReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; @@ -93,13 +94,17 @@ public class StatUtils { if (semanticQueryReq instanceof QueryStructReq) { initStructStatInfo((QueryStructReq) semanticQueryReq, facadeUser); } + if (semanticQueryReq instanceof QueryMultiStructReq) { + QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0); + initStructStatInfo(queryStructCmd, facadeUser); + } } - public void initSqlStatInfo(QuerySqlReq querySQLReq, User facadeUser) { + public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { QueryStat queryStatInfo = new QueryStat(); - List allFields = SqlParserSelectHelper.getAllFields(querySQLReq.getSql()); - queryStatInfo.setModelId(querySQLReq.getModelIds().get(0)); - ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySQLReq.getModelIds().get(0)); + List allFields = SqlParserSelectHelper.getAllFields(querySqlReq.getSql()); + queryStatInfo.setModelId(querySqlReq.getModelIds().get(0)); + ModelSchemaResp modelSchemaResp = modelService.fetchSingleModelSchema(querySqlReq.getModelIds().get(0)); List dimensions = new ArrayList<>(); List metrics = new ArrayList<>(); @@ -111,12 +116,12 @@ public class StatUtils { String userName = getUserName(facadeUser); try { queryStatInfo.setTraceId("") - .setModelId(querySQLReq.getModelIds().get(0)) + .setModelId(querySqlReq.getModelIds().get(0)) .setUser(userName) .setQueryType(QueryType.SQL.getValue()) .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) - .setQuerySqlCmd(querySQLReq.toString()) - .setQuerySqlCmdMd5(DigestUtils.md5Hex(querySQLReq.toString())) + .setQuerySqlCmd(querySqlReq.toString()) + .setQuerySqlCmdMd5(DigestUtils.md5Hex(querySqlReq.toString())) .setStartTime(System.currentTimeMillis()) .setUseResultCache(true) .setUseSqlCache(true) diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/BaseTest.java index 90bd2fb44..55f491909 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/BaseTest.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.integration; import com.tencent.supersonic.StandaloneLauncher; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; +import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.service.QueryService; import java.util.HashSet; @@ -29,7 +30,11 @@ public class BaseTest { return queryService.queryByReq(buildQuerySqlReq(sql), user); } - protected QuerySqlReq buildQuerySqlReq(String sql) { + protected SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception { + return queryService.queryByReq(queryReq, user); + } + + protected SemanticQueryReq buildQuerySqlReq(String sql) { QuerySqlReq querySqlCmd = new QuerySqlReq(); querySqlCmd.setSql(sql); Set modelIds = new HashSet<>(); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/ExplainTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/ExplainTest.java new file mode 100644 index 000000000..c450eb196 --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/ExplainTest.java @@ -0,0 +1,10 @@ +package com.tencent.supersonic.headless.integration; + +import org.junit.Test; + +public class ExplainTest extends BaseTest { + + @Test + public void testSumExplain() { + } +} diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/QueryByStructTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/QueryByStructTest.java new file mode 100644 index 000000000..7b4ff730a --- /dev/null +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/integration/QueryByStructTest.java @@ -0,0 +1,86 @@ +package com.tencent.supersonic.headless.integration; + +import static java.time.LocalDate.now; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.DateConf.DateMode; +import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; +import org.junit.Test; + +public class QueryByStructTest extends BaseTest { + + @Test + public void testSumQuery() throws Exception { + QueryStructReq queryStructReq = buildQueryStructReq(null); + SemanticQueryResp semanticQueryResp = queryByReq(queryStructReq, User.getFakeUser()); + assertEquals(1, semanticQueryResp.getColumns().size()); + QueryColumn queryColumn = semanticQueryResp.getColumns().get(0); + assertEquals("访问次数", queryColumn.getName()); + assertEquals(1, semanticQueryResp.getResultList().size()); + } + + @Test + public void testGroupByQuery() throws Exception { + QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department")); + SemanticQueryResp result = queryByReq(queryStructReq, User.getFakeUser()); + assertEquals(2, result.getColumns().size()); + QueryColumn firstColumn = result.getColumns().get(0); + QueryColumn secondColumn = result.getColumns().get(1); + assertEquals("部门", firstColumn.getName()); + assertEquals("访问次数", secondColumn.getName()); + assertNotNull(result.getResultList().size()); + } + + @Test + public void testCacheQuery() throws Exception { + QueryStructReq queryStructReq1 = buildQueryStructReq(Arrays.asList("department")); + QueryStructReq queryStructReq2 = buildQueryStructReq(Arrays.asList("department")); + SemanticQueryResp result1 = queryByReq(queryStructReq1, User.getFakeUser()); + SemanticQueryResp result2 = queryByReq(queryStructReq2, User.getFakeUser()); + assertEquals(result1, result2); + } + + private QueryStructReq buildQueryStructReq(List groups) { + QueryStructReq queryStructReq = new QueryStructReq(); + queryStructReq.addModelId(1L); + queryStructReq.addModelId(2L); + queryStructReq.addModelId(3L); + + queryStructReq.setQueryType(QueryType.METRIC); + Aggregator aggregator = new Aggregator(); + aggregator.setFunc(AggOperatorEnum.SUM); + aggregator.setColumn("pv"); + queryStructReq.setAggregators(Arrays.asList(aggregator)); + + if (CollectionUtils.isNotEmpty(groups)) { + queryStructReq.setGroups(groups); + queryStructReq.setGroups(Arrays.asList("department")); + } + + DateConf dateConf = new DateConf(); + dateConf.setDateMode(DateMode.BETWEEN); + dateConf.setStartDate(now().plusDays(-1).toString()); + dateConf.setEndDate(now().plusDays(-10).toString()); + queryStructReq.setDateInfo(dateConf); + + List orders = new ArrayList<>(); + Order order = new Order(); + order.setColumn("pv"); + orders.add(order); + queryStructReq.setOrders(orders); + return queryStructReq; + } +}