(improvement)(headless) Parse sql variable (#763)

Co-authored-by: jolunoluo

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java

@@ -9,18 +9,19 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
 import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
-import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.pojo.Database;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
 
 /**
  * supplement the QueryStatement when query with custom aggregation method
@@ -110,7 +111,6 @@ public class CalculateAggConverter implements HeadlessConverter {
                 EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
         sqlCommend.setSql(viewQueryParam.getSql());
         sqlCommend.setTables(viewQueryParam.getTables());
-        sqlCommend.setVariables(viewQueryParam.getVariables());
         sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java

@@ -1,20 +1,20 @@
 package com.tencent.supersonic.headless.core.parser.converter;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import com.tencent.supersonic.headless.api.pojo.Param;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
-import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 /**
  * HeadlessConverter default implement
  */
@@ -59,8 +59,6 @@ public class ParserDefaultConverter implements HeadlessConverter {
         metricQueryParam.setWhere(where);
         metricQueryParam.setOrder(queryParam.getOrders().stream()
                 .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
-        metricQueryParam.setVariables(queryParam.getParams().stream()
-                .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
         metricQueryParam.setLimit(queryParam.getLimit());
 
         // support detail query

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java

@@ -0,0 +1,49 @@
+package com.tencent.supersonic.headless.core.parser.converter;
+
+import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
+import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+@Slf4j
+@Component("SqlVariableParseConverter")
+public class SqlVariableParseConverter implements HeadlessConverter {
+
+    @Override
+    public boolean accept(QueryStatement queryStatement) {
+        if (Objects.isNull(queryStatement.getQueryParam())) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void convert(QueryStatement queryStatement) {
+        SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
+        List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
+        if (CollectionUtils.isEmpty(modelResps)) {
+            return;
+        }
+        for (ModelResp modelResp : modelResps) {
+            if (ModelDefineType.SQL_QUERY.getName()
+                    .equalsIgnoreCase(modelResp.getModelDetail().getQueryType())) {
+                String sqlParsed = SqlVariableParseUtils.parse(
+                        modelResp.getModelDetail().getSqlQuery(),
+                        modelResp.getModelDetail().getSqlVariables(),
+                        queryStatement.getQueryParam().getParams()
+                );
+                DataSource dataSource = queryStatement.getSemanticModel()
+                        .getDatasourceMap().get(modelResp.getBizName());
+                dataSource.setSqlQuery(sqlParsed);
+            }
+        }
+    }
+}

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java

@@ -1,16 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
+import java.util.List;
 
 @Data
 public class MetricQueryParam {
 
     private List<String> metrics;
     private List<String> dimensions;
-    private Map<String, String> variables;
     private String where;
     private Long limit;
     private List<ColumnOrder> order;

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java

@@ -1,24 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.headless.api.pojo.MetricTable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
 
+import java.util.List;
+
 @Data
 public class ViewQueryParam {
-
-    private Map<String, String> variables;
     private String sql = "";
     private List<MetricTable> tables;
     private boolean supportWith = true;
     private boolean withAlias = true;
-
-    public Map<String, String> getVariables() {
-        if (variables == null) {
-            variables = new HashMap<>();
-        }
-        return variables;
-    }
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.core.utils;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
 import com.tencent.supersonic.headless.core.executor.QueryExecutor;
+import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter;
 import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer;
 import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
 import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
@@ -83,6 +84,7 @@ public class ComponentFactory {
 
     private static void initSemanticConverter() {
         headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
+        headlessConverters.add(getBean("SqlVariableParseConverter", SqlVariableParseConverter.class));
         headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
         headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
     }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java

@@ -30,13 +30,13 @@ public class SqlVariableParseUtils {
     private static final char delimiter = '$';
 
     public static String parse(String sql, List<SqlVariable> sqlVariables, List<Param> params) {
+        Map<String, Object> variables = new HashMap<>();
         if (CollectionUtils.isEmpty(sqlVariables)) {
             return sql;
         }
-        Map<String, Object> queryParams = new HashMap<>();
         //1. handle default variable value
         sqlVariables.forEach(variable -> {
-            queryParams.put(variable.getName().trim(),
+            variables.put(variable.getName().trim(),
                     getValues(variable.getValueType(), variable.getDefaultValues()));
         });
 
@@ -49,21 +49,25 @@ public class SqlVariableParseUtils {
                     List<SqlVariable> list = map.get(p.getName());
                     if (!CollectionUtils.isEmpty(list)) {
                         SqlVariable v = list.get(list.size() - 1);
-                        queryParams.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
+                        variables.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
                     }
                 }
             });
         }
 
-        queryParams.forEach((k, v) -> {
+        variables.forEach((k, v) -> {
             if (v instanceof List && ((List) v).size() > 0) {
                 v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString();
             }
-            queryParams.put(k, v);
+            variables.put(k, v);
         });
+        return parse(sql, variables);
+    }
+
+    public static String parse(String sql, Map<String, Object> variables) {
         ST st = new ST(sql, delimiter, delimiter);
-        if (!CollectionUtils.isEmpty(queryParams)) {
-            queryParams.forEach(st::add);
+        if (!CollectionUtils.isEmpty(variables)) {
+            variables.forEach(st::add);
         }
         return st.render();
     }