(improvement)(Headless) Remove QueryType from queryBySql and add a new interface to query multiple SQL statements at once. (#895)

2025-12-10 19:51:00 +00:00 · 2024-04-08 14:36:56 +08:00
parent a83c63829b
commit 7dcc8b438b
8 changed files with 94 additions and 39 deletions
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlReq.java
@@ -1,13 +1,11 @@
 package com.tencent.supersonic.headless.api.pojo.request;

-import com.tencent.supersonic.common.pojo.enums.QueryType;
 import lombok.Data;
 import lombok.ToString;

@Data
@ToString
 public class QuerySqlReq extends SemanticQueryReq {
-    private QueryType queryType;
    private String sql;

    @Override
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlsReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QuerySqlsReq.java
@@ -0,0 +1,30 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.List;
+
+@Data
+@ToString
+public class QuerySqlsReq extends SemanticQueryReq {
+    private List<String> sqls;
+
+    @Override
+    public String toCustomizedString() {
+        StringBuilder stringBuilder = new StringBuilder("{");
+        stringBuilder.append("\"dataSetId\":")
+                .append(dataSetId);
+        stringBuilder.append("\"modelIds\":")
+                .append(modelIds);
+        stringBuilder.append(",\"params\":")
+                .append(params);
+        stringBuilder.append(",\"cacheInfo\":")
+                .append(cacheInfo);
+        stringBuilder.append(",\"sqls\":")
+                .append(sqls);
+        stringBuilder.append('}');
+        return stringBuilder.toString();
+    }
+
+}
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java
@@ -19,6 +19,8 @@ public abstract class SemanticQueryReq {

    protected boolean needAuth = true;

+    protected boolean innerLayerNative = false;
+
    protected Long dataSetId;

    protected String dataSetName;
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/SqlQueryApiController.java
@@ -3,8 +3,11 @@ package com.tencent.supersonic.headless.server.rest.api;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
+import com.tencent.supersonic.headless.api.pojo.request.QuerySqlsReq;
+import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.server.service.QueryService;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -13,6 +16,8 @@ import org.springframework.web.bind.annotation.RestController;

 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import java.util.List;
+import java.util.stream.Collectors;

@RestController
@RequestMapping("/api/semantic/query")
@@ -24,9 +29,24 @@ public class SqlQueryApiController {

    @PostMapping("/sql")
    public Object queryBySql(@RequestBody QuerySqlReq querySqlReq,
-            HttpServletRequest request,
-            HttpServletResponse response) throws Exception {
+                             HttpServletRequest request,
+                             HttpServletResponse response) throws Exception {
        User user = UserHolder.findUser(request, response);
        return queryService.queryByReq(querySqlReq, user);
    }
+
+    @PostMapping("/sqls")
+    public Object queryBySqls(@RequestBody QuerySqlsReq querySqlsReq,
+                              HttpServletRequest request,
+                              HttpServletResponse response) throws Exception {
+        User user = UserHolder.findUser(request, response);
+        List<SemanticQueryReq> semanticQueryReqs = querySqlsReq.getSqls()
+                .stream().map(sql -> {
+                    QuerySqlReq querySqlReq = new QuerySqlReq();
+                    BeanUtils.copyProperties(querySqlsReq, querySqlReq);
+                    querySqlReq.setSql(sql);
+                    return querySqlReq;
+                }).collect(Collectors.toList());
+        return queryService.queryByReqs(semanticQueryReqs, user);
+    }
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/QueryService.java
@@ -12,9 +12,10 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import java.util.List;

 public interface QueryService {
-
    SemanticQueryResp queryByReq(SemanticQueryReq queryReq, User user) throws Exception;

+    List<SemanticQueryResp> queryByReqs(List<SemanticQueryReq> queryReqs, User user) throws Exception;
+
    SemanticQueryResp queryDimValue(QueryDimValueReq queryDimValueReq, User user);

    List<ItemUseResp> getStatInfo(ItemUseReq itemUseCommend);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java
@@ -45,6 +45,8 @@ import org.springframework.stereotype.Service;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;


@Service
@@ -118,6 +120,14 @@ public class QueryServiceImpl implements QueryService {
        }
    }

+    @Override
+    public List<SemanticQueryResp> queryByReqs(List<SemanticQueryReq> queryReqs, User user) throws Exception {
+        List<CompletableFuture<SemanticQueryResp>> futures = queryReqs.stream()
+                .map(querySqlReq -> CompletableFuture.supplyAsync(() -> queryByReq(querySqlReq, user)))
+                .collect(Collectors.toList());
+        return futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
+    }
+
    private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq) throws Exception {
        SchemaFilterReq filter = buildSchemaFilterReq(querySqlReq);
        SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
@@ -1,11 +1,5 @@
 package com.tencent.supersonic.headless.server.utils;

-import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
-import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
-import static com.tencent.supersonic.common.pojo.Constants.COMMA;
-import static com.tencent.supersonic.common.pojo.Constants.POUND;
-import static com.tencent.supersonic.common.pojo.Constants.SPACE;
-
 import com.google.common.base.Strings;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Aggregator;
@@ -15,7 +9,6 @@ import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.common.pojo.Order;
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
@@ -41,6 +34,13 @@ import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.QueryService;
 import com.tencent.supersonic.headless.server.service.TagMetaService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -53,12 +53,12 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
+
+import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
+import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
+import static com.tencent.supersonic.common.pojo.Constants.COMMA;
+import static com.tencent.supersonic.common.pojo.Constants.POUND;
+import static com.tencent.supersonic.common.pojo.Constants.SPACE;

@Slf4j
@Component
@@ -242,14 +242,7 @@ public class DictUtils {

    private SemanticQueryReq constructQueryReq(DictItemResp dictItemResp) {
        if (TypeEnums.DIMENSION.equals(dictItemResp.getType())) {
-            QuerySqlReq querySqlReq = constructDimQueryReq(dictItemResp);
-            querySqlReq.setQueryType(QueryType.METRIC);
-            return querySqlReq;
-        }
-        if (TypeEnums.TAG.equals(dictItemResp.getType())) {
-            QuerySqlReq querySqlReq = constructTagQueryReq(dictItemResp);
-            querySqlReq.setQueryType(QueryType.TAG);
-            return querySqlReq;
+            return constructDimQueryReq(dictItemResp);
        }
        log.warn("constructQueryReq failed");
        return null;
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
@@ -28,6 +28,15 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
 import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -37,14 +46,6 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;

@Component
@Slf4j
@@ -60,7 +61,7 @@ public class QueryReqConverter {
    private SqlGenerateUtils sqlGenerateUtils;

    public QueryStatement convert(QuerySqlReq querySQLReq,
-            SemanticSchemaResp semanticSchemaResp) throws Exception {
+                                  SemanticSchemaResp semanticSchemaResp) throws Exception {

        if (semanticSchemaResp == null) {
            return new QueryStatement();
@@ -150,7 +151,7 @@ public class QueryReqConverter {
                || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
            return AggOption.NATIVE;
        }
-        if (Objects.nonNull(databaseReq.getQueryType()) && QueryType.TAG.equals(databaseReq.getQueryType())) {
+        if (databaseReq.isInnerLayerNative()) {
            return AggOption.NATIVE;
        }
        return AggOption.DEFAULT;
@@ -242,7 +243,7 @@ public class QueryReqConverter {
    }

    private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
-            DataSetQueryParam viewQueryParam) {
+                                       DataSetQueryParam viewQueryParam) {
        String sql = viewQueryParam.getSql();
        for (MetricTable metricTable : viewQueryParam.getTables()) {
            List<String> measures = new ArrayList<>();
@@ -264,8 +265,8 @@ public class QueryReqConverter {
    }

    private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption,
-            List<String> metrics, List<String> dimensions,
-            List<String> measures, Map<String, String> replaces) {
+                                       List<String> metrics, List<String> dimensions,
+                                       List<String> measures, Map<String, String> replaces) {
        List<MetricSchemaResp> metricResps = semanticSchemaResp.getMetrics();
        List<DimSchemaResp> dimensionResps = semanticSchemaResp.getDimensions();
        // check metrics has derived