(improvement)(headless) Parse sql variable (#763)

Co-authored-by: jolunoluo

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java

@@ -6,12 +6,11 @@ import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.Filter;
 import com.tencent.supersonic.common.pojo.Order;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
+import lombok.Data;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import lombok.Data;
 
 @Data
 public class QueryParam {
@@ -34,7 +33,6 @@ public class QueryParam {
     // metric
     private List<String> metrics = new ArrayList();
     private List<String> dimensions;
-    private Map<String, String> variables;
     private String where;
     private List<ColumnOrder> order;
     private boolean nativeQuery = false;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/ModelDefineType.java

@@ -5,7 +5,7 @@ package com.tencent.supersonic.headless.api.pojo.enums;
  * sql_query : view sql begin as select
  * table_query: dbName.tableName
  */
-public enum DatasourceQuery {
+public enum ModelDefineType {
 
     SQL_QUERY("sql_query"),
     TABLE_QUERY("table_query");
@@ -13,7 +13,7 @@ public enum DatasourceQuery {
     private String name;
 
 
-    DatasourceQuery(String name) {
+    ModelDefineType(String name) {
         this.name = name;
     }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SqlExecuteReq.java

@@ -1,10 +1,12 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
 
-import javax.validation.constraints.NotBlank;
-import javax.validation.constraints.NotNull;
+import com.tencent.supersonic.headless.api.pojo.SqlVariable;
 import lombok.Data;
 import org.apache.commons.lang3.StringUtils;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+import java.util.List;
 
 @Data
 public class SqlExecuteReq {
@@ -16,6 +18,8 @@ public class SqlExecuteReq {
     @NotBlank(message = "sql can not be blank")
     private String sql;
 
+    private List<SqlVariable> sqlVariables;
+
     public String getSql() {
         if (StringUtils.isNotBlank(sql) && sql.endsWith(";")) {
             sql = sql.substring(0, sql.length() - 1);

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java

@@ -9,18 +9,19 @@ import com.tencent.supersonic.headless.api.pojo.MetricTable;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
 import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.enums.EngineType;
-import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.pojo.Database;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.pojo.ViewQueryParam;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
 
 /**
  * supplement the QueryStatement when query with custom aggregation method
@@ -110,7 +111,6 @@ public class CalculateAggConverter implements HeadlessConverter {
                 EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
         sqlCommend.setSql(viewQueryParam.getSql());
         sqlCommend.setTables(viewQueryParam.getTables());
-        sqlCommend.setVariables(viewQueryParam.getVariables());
         sqlCommend.setSupportWith(viewQueryParam.isSupportWith());
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java

@@ -1,20 +1,20 @@
 package com.tencent.supersonic.headless.core.parser.converter;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import com.tencent.supersonic.headless.api.pojo.Param;
 import com.tencent.supersonic.headless.api.pojo.QueryParam;
-import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 /**
  * HeadlessConverter default implement
  */
@@ -59,8 +59,6 @@ public class ParserDefaultConverter implements HeadlessConverter {
         metricQueryParam.setWhere(where);
         metricQueryParam.setOrder(queryParam.getOrders().stream()
                 .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())).collect(Collectors.toList()));
-        metricQueryParam.setVariables(queryParam.getParams().stream()
-                .collect(Collectors.toMap(Param::getName, Param::getValue, (k1, k2) -> k1)));
         metricQueryParam.setLimit(queryParam.getLimit());
 
         // support detail query

headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/SqlVariableParseConverter.java

@@ -0,0 +1,49 @@
+package com.tencent.supersonic.headless.core.parser.converter;
+
+import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
+import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
+import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+@Slf4j
+@Component("SqlVariableParseConverter")
+public class SqlVariableParseConverter implements HeadlessConverter {
+
+    @Override
+    public boolean accept(QueryStatement queryStatement) {
+        if (Objects.isNull(queryStatement.getQueryParam())) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void convert(QueryStatement queryStatement) {
+        SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp();
+        List<ModelResp> modelResps = semanticSchemaResp.getModelResps();
+        if (CollectionUtils.isEmpty(modelResps)) {
+            return;
+        }
+        for (ModelResp modelResp : modelResps) {
+            if (ModelDefineType.SQL_QUERY.getName()
+                    .equalsIgnoreCase(modelResp.getModelDetail().getQueryType())) {
+                String sqlParsed = SqlVariableParseUtils.parse(
+                        modelResp.getModelDetail().getSqlQuery(),
+                        modelResp.getModelDetail().getSqlVariables(),
+                        queryStatement.getQueryParam().getParams()
+                );
+                DataSource dataSource = queryStatement.getSemanticModel()
+                        .getDatasourceMap().get(modelResp.getBizName());
+                dataSource.setSqlQuery(sqlParsed);
+            }
+        }
+    }
+}

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java

@@ -1,16 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.common.pojo.ColumnOrder;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
+import java.util.List;
 
 @Data
 public class MetricQueryParam {
 
     private List<String> metrics;
     private List<String> dimensions;
-    private Map<String, String> variables;
     private String where;
     private Long limit;
     private List<ColumnOrder> order;

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/ViewQueryParam.java

@@ -1,24 +1,14 @@
 package com.tencent.supersonic.headless.core.pojo;
 
 import com.tencent.supersonic.headless.api.pojo.MetricTable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import lombok.Data;
 
+import java.util.List;
+
 @Data
 public class ViewQueryParam {
-
-    private Map<String, String> variables;
     private String sql = "";
     private List<MetricTable> tables;
     private boolean supportWith = true;
     private boolean withAlias = true;
-
-    public Map<String, String> getVariables() {
-        if (variables == null) {
-            variables = new HashMap<>();
-        }
-        return variables;
-    }
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.core.utils;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
 import com.tencent.supersonic.headless.core.executor.QueryExecutor;
+import com.tencent.supersonic.headless.core.parser.converter.SqlVariableParseConverter;
 import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer;
 import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
 import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
@@ -83,6 +84,7 @@ public class ComponentFactory {
 
     private static void initSemanticConverter() {
         headlessConverters.add(getBean("DefaultDimValueConverter", DefaultDimValueConverter.class));
+        headlessConverters.add(getBean("SqlVariableParseConverter", SqlVariableParseConverter.class));
         headlessConverters.add(getBean("CalculateAggConverter", CalculateAggConverter.class));
         headlessConverters.add(getBean("ParserDefaultConverter", ParserDefaultConverter.class));
     }

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlVariableParseUtils.java

@@ -30,13 +30,13 @@ public class SqlVariableParseUtils {
     private static final char delimiter = '$';
 
     public static String parse(String sql, List<SqlVariable> sqlVariables, List<Param> params) {
+        Map<String, Object> variables = new HashMap<>();
         if (CollectionUtils.isEmpty(sqlVariables)) {
             return sql;
         }
-        Map<String, Object> queryParams = new HashMap<>();
         //1. handle default variable value
         sqlVariables.forEach(variable -> {
-            queryParams.put(variable.getName().trim(),
+            variables.put(variable.getName().trim(),
                     getValues(variable.getValueType(), variable.getDefaultValues()));
         });
 
@@ -49,21 +49,25 @@ public class SqlVariableParseUtils {
                     List<SqlVariable> list = map.get(p.getName());
                     if (!CollectionUtils.isEmpty(list)) {
                         SqlVariable v = list.get(list.size() - 1);
-                        queryParams.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
+                        variables.put(p.getName().trim(), getValue(v.getValueType(), p.getValue()));
                     }
                 }
             });
         }
 
-        queryParams.forEach((k, v) -> {
+        variables.forEach((k, v) -> {
             if (v instanceof List && ((List) v).size() > 0) {
                 v = ((List) v).stream().collect(Collectors.joining(COMMA)).toString();
             }
-            queryParams.put(k, v);
+            variables.put(k, v);
         });
+        return parse(sql, variables);
+    }
+
+    public static String parse(String sql, Map<String, Object> variables) {
         ST st = new ST(sql, delimiter, delimiter);
-        if (!CollectionUtils.isEmpty(queryParams)) {
-            queryParams.forEach(st::add);
+        if (!CollectionUtils.isEmpty(variables)) {
+            variables.forEach(st::add);
         }
         return st.render();
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -4,7 +4,7 @@ import com.tencent.supersonic.headless.api.pojo.Dim;
 import com.tencent.supersonic.headless.api.pojo.Identify;
 import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.ModelDetail;
-import com.tencent.supersonic.headless.api.pojo.enums.DatasourceQuery;
+import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
@@ -46,7 +46,7 @@ public class ModelYamlManager {
                 .collect(Collectors.toList()));
         dataModelYamlTpl.setName(modelResp.getBizName());
         dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
-        if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) {
+        if (modelDetail.getQueryType().equalsIgnoreCase(ModelDefineType.SQL_QUERY.getName())) {
             dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery());
         } else {
             dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DatabaseController.java

@@ -73,7 +73,7 @@ public class DatabaseController {
             HttpServletRequest request,
             HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return databaseService.executeSql(sqlExecuteReq.getSql(), sqlExecuteReq.getId(), user);
+        return databaseService.executeSql(sqlExecuteReq, sqlExecuteReq.getId(), user);
     }
 
     @RequestMapping("/getDbNames/{id}")

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DatabaseService.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
+import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
@@ -13,7 +14,7 @@ public interface DatabaseService {
 
     SemanticQueryResp executeSql(String sql, DatabaseResp databaseResp);
 
-    SemanticQueryResp executeSql(String sql, Long id, User user);
+    SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user);
 
     DatabaseResp getDatabase(Long id, User user);
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -1,8 +1,9 @@
 package com.tencent.supersonic.headless.server.service.impl;
 
+import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.exception.InvalidPermissionException;
 import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
+import com.tencent.supersonic.headless.api.pojo.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
@@ -11,6 +12,7 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
 import com.tencent.supersonic.headless.core.pojo.Database;
 import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils;
 import com.tencent.supersonic.headless.core.utils.SqlUtils;
+import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils;
 import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DatabaseRepository;
 import com.tencent.supersonic.headless.server.pojo.DatabaseParameter;
@@ -116,32 +118,19 @@ public class DatabaseServiceImpl implements DatabaseService {
     @Override
     public DatabaseResp getDatabase(Long id, User user) {
         DatabaseResp databaseResp = getDatabase(id);
-        if (!databaseResp.getAdmins().contains(user.getName())
-                && !databaseResp.getViewers().contains(user.getName())
-                && !databaseResp.getCreatedBy().equals(user.getName())) {
-            throw new InvalidPermissionException("您暂无查看该数据库详情的权限, 请联系创建人: "
-                    + databaseResp.getCreatedBy());
-        }
+        checkPermission(databaseResp, user);
         return databaseResp;
     }
 
     @Override
-    public SemanticQueryResp executeSql(String sql, Long id, User user) {
+    public SemanticQueryResp executeSql(SqlExecuteReq sqlExecuteReq, Long id, User user) {
         DatabaseResp databaseResp = getDatabase(id);
         if (databaseResp == null) {
             return new SemanticQueryResp();
         }
-        List<String> admins = databaseResp.getAdmins();
-        List<String> viewers = databaseResp.getViewers();
-        if (!admins.contains(user.getName())
-                && !viewers.contains(user.getName())
-                && !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())
-                && !user.isSuperAdmin()) {
-            String message = String.format("您暂无当前数据库%s权限, 请联系数据库管理员%s开通",
-                    databaseResp.getName(),
-                    String.join(",", admins));
-            throw new RuntimeException(message);
-        }
+        checkPermission(databaseResp, user);
+        String sql = sqlExecuteReq.getSql();
+        sql = SqlVariableParseUtils.parse(sql, sqlExecuteReq.getSqlVariables(), Lists.newArrayList());
         return executeSql(sql, databaseResp);
     }
 
@@ -195,4 +184,18 @@ public class DatabaseServiceImpl implements DatabaseService {
         return queryWithColumns(metaQuerySql, DatabaseConverter.convert(databaseResp));
     }
 
+    private void checkPermission(DatabaseResp databaseResp, User user) {
+        List<String> admins = databaseResp.getAdmins();
+        List<String> viewers = databaseResp.getViewers();
+        if (!admins.contains(user.getName())
+                && !viewers.contains(user.getName())
+                && !databaseResp.getCreatedBy().equalsIgnoreCase(user.getName())
+                && !user.isSuperAdmin()) {
+            String message = String.format("您暂无当前数据库%s权限, 请联系数据库创建人:%s开通",
+                    databaseResp.getName(),
+                    databaseResp.getCreatedBy());
+            throw new RuntimeException(message);
+        }
+    }
+
 }