Merge 738093bc88 into f764236657

common/src/main/java/com/tencent/supersonic/common/calcite/SqlMergeWithUtils.java

@@ -2,11 +2,19 @@ package com.tencent.supersonic.common.calcite;
 
 import com.tencent.supersonic.common.pojo.enums.EngineType;
 import lombok.extern.slf4j.Slf4j;
-import net.sf.jsqlparser.expression.Alias;
-import net.sf.jsqlparser.parser.CCJSqlParserUtil;
-import net.sf.jsqlparser.statement.select.ParenthesedSelect;
-import net.sf.jsqlparser.statement.select.Select;
-import net.sf.jsqlparser.statement.select.WithItem;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlOrderBy;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlWith;
+import org.apache.calcite.sql.SqlWithItem;
+import org.apache.calcite.sql.SqlWriterConfig;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.pretty.SqlPrettyWriter;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -14,37 +22,85 @@ import java.util.List;
 @Slf4j
 public class SqlMergeWithUtils {
     public static String mergeWith(EngineType engineType, String sql, List<String> parentSqlList,
-            List<String> parentWithNameList) throws Exception {
+            List<String> parentWithNameList) throws SqlParseException {
+        SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
 
-        Select selectStatement = (Select) CCJSqlParserUtil.parse(sql);
-        List<WithItem> withItemList = new ArrayList<>();
+        // Parse the main SQL statement
+        SqlParser parser = SqlParser.create(sql, parserConfig);
+        SqlNode sqlNode1 = parser.parseQuery();
 
+        // List to hold all WITH items
+        List<SqlNode> withItemList = new ArrayList<>();
+
+        // Iterate over each parentSql and parentWithName pair
         for (int i = 0; i < parentSqlList.size(); i++) {
             String parentSql = parentSqlList.get(i);
             String parentWithName = parentWithNameList.get(i);
 
-            Select parentSelect = (Select) CCJSqlParserUtil.parse(parentSql);
-            ParenthesedSelect select = new ParenthesedSelect();
-            select.setSelect(parentSelect);
+            // Parse the parent SQL statement
+            parser = SqlParser.create(parentSql, parserConfig);
+            SqlNode sqlNode2 = parser.parseQuery();
 
             // Create a new WITH item for parentWithName without quotes
-            WithItem withItem = new WithItem();
-            withItem.setAlias(new Alias(parentWithName));
-            withItem.setSelect(select);
+            SqlWithItem withItem = new SqlWithItem(SqlParserPos.ZERO,
+                    new SqlIdentifier(parentWithName, SqlParserPos.ZERO), null, sqlNode2,
+                    SqlLiteral.createBoolean(false, SqlParserPos.ZERO));
 
             // Add the new WITH item to the list
             withItemList.add(withItem);
         }
 
-        // Extract existing WITH items from mainSelectBody if it has any
-        if (selectStatement.getWithItemsList() != null) {
-            withItemList.addAll(selectStatement.getWithItemsList());
+        // Check if the main SQL node contains an ORDER BY or LIMIT clause
+        SqlNode limitNode = null;
+        SqlNodeList orderByList = null;
+        if (sqlNode1 instanceof SqlOrderBy) {
+            SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode1;
+            limitNode = sqlOrderBy.fetch;
+            orderByList = sqlOrderBy.orderList;
+            sqlNode1 = sqlOrderBy.query;
+        } else if (sqlNode1 instanceof SqlSelect) {
+            SqlSelect sqlSelect = (SqlSelect) sqlNode1;
+            limitNode = sqlSelect.getFetch();
+            sqlSelect.setFetch(null);
+            sqlNode1 = sqlSelect;
         }
 
-        // Set the new WITH items list to the main select body
-        selectStatement.setWithItemsList(withItemList);
+        // Extract existing WITH items from sqlNode1 if it is a SqlWith
+        if (sqlNode1 instanceof SqlWith) {
+            SqlWith sqlWith = (SqlWith) sqlNode1;
+            withItemList.addAll(sqlWith.withList.getList());
+            sqlNode1 = sqlWith.body;
+        }
 
+        // Create a new SqlWith node
+        SqlWith finalSqlNode = new SqlWith(SqlParserPos.ZERO,
+                new SqlNodeList(withItemList, SqlParserPos.ZERO), sqlNode1);
+
+        // If there was an ORDER BY or LIMIT clause, wrap the finalSqlNode in a SqlOrderBy
+        SqlNode resultNode = finalSqlNode;
+        if (orderByList != null || limitNode != null) {
+            resultNode = new SqlOrderBy(SqlParserPos.ZERO, finalSqlNode,
+                    orderByList != null ? orderByList : SqlNodeList.EMPTY, null, limitNode);
+        }
+
+        // Custom SqlPrettyWriter configuration to avoid quoting identifiers
+        SqlWriterConfig config = Configuration.getSqlWriterConfig(engineType);
         // Pretty print the final SQL
-        return selectStatement.toString();
+        SqlPrettyWriter writer = new SqlPrettyWriter(config);
+        return writer.format(resultNode);
+    }
+
+    public static boolean hasWith(EngineType engineType, String sql) throws SqlParseException {
+        SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
+        SqlParser parser = SqlParser.create(sql, parserConfig);
+        SqlNode sqlNode = parser.parseQuery();
+        SqlNode sqlSelect = sqlNode;
+        if (sqlNode instanceof SqlOrderBy) {
+            SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode;
+            sqlSelect = sqlOrderBy.query;
+        } else if (sqlNode instanceof SqlSelect) {
+            sqlSelect = (SqlSelect) sqlNode;
+        }
+        return sqlSelect instanceof SqlWith;
     }
 }

common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java

@@ -91,8 +91,7 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
         }
 
         ExpressionList<?> leftFunctionParams = leftFunction.getParameters();
-        if (CollectionUtils.isEmpty(leftFunctionParams)
-                || !(leftFunctionParams.get(0) instanceof Column)) {
+        if (CollectionUtils.isEmpty(leftFunctionParams)) {
             return result;
         }
 

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java

@@ -989,15 +989,6 @@ public class SqlSelectHelper {
         for (SelectItem selectItem : selectItems) {
             selectItem.accept(visitor);
         }
-        if (plainSelect.getHaving() != null) {
-            plainSelect.getHaving().accept(visitor);
-        }
-        if (!CollectionUtils.isEmpty(plainSelect.getOrderByElements())) {
-            for (OrderByElement orderByElement : plainSelect.getOrderByElements()) {
-                orderByElement.getExpression().accept(visitor);
-            }
-        }
-
         return !visitor.getFunctionNames().isEmpty();
     }
 

common/src/test/java/com/tencent/supersonic/common/calcite/SqlWithMergerTest.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.common.calcite;
 
 import com.tencent.supersonic.common.pojo.enums.EngineType;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.calcite.sql.parser.SqlParseException;
 import org.junit.Assert;
 import org.junit.jupiter.api.Test;
 
@@ -11,7 +12,7 @@ import java.util.Collections;
 class SqlWithMergerTest {
 
     @Test
-    void test1() throws Exception {
+    void test1() throws SqlParseException {
         String sql1 = "WITH DepartmentVisits AS (\n" + "    SELECT department, SUM(pv) AS 总访问次数\n"
                 + "    FROM t_1\n"
                 + "    WHERE sys_imp_date >= '2024-09-01' AND sys_imp_date <= '2024-09-29'\n"
@@ -37,7 +38,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test2() throws Exception {
+    void test2() throws SqlParseException {
 
         String sql1 =
                 "WITH DepartmentVisits AS (SELECT department, SUM(pv) AS 总访问次数 FROM t_1 WHERE sys_imp_date >= '2024-08-28' "
@@ -64,7 +65,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test3() throws Exception {
+    void test3() throws SqlParseException {
 
         String sql1 = "SELECT COUNT(*) FROM DepartmentVisits WHERE 总访问次数 > 100 LIMIT 1000";
 
@@ -88,7 +89,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test4() throws Exception {
+    void test4() throws SqlParseException {
         String sql1 = "SELECT COUNT(*) FROM DepartmentVisits WHERE 总访问次数 > 100";
 
         String sql2 =
@@ -111,7 +112,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test5() throws Exception {
+    void test5() throws SqlParseException {
 
         String sql1 = "SELECT COUNT(*) FROM Department join Visits WHERE 总访问次数 > 100";
 
@@ -131,13 +132,13 @@ class SqlWithMergerTest {
                 "WITH t_1 AS (SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv` "
                         + "FROM (SELECT `user_name`, `department` FROM `s2_user_department`) AS `t2` LEFT JOIN "
                         + "(SELECT 1 AS `s2_pv_uv_statis_pv`, `imp_date` AS `sys_imp_date`, `user_name` FROM `s2_pv_uv_statis`) "
-                        + "AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) SELECT COUNT(*) FROM Department JOIN Visits "
+                        + "AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) SELECT COUNT(*) FROM Department INNER JOIN Visits "
                         + "WHERE 总访问次数 > 100");
     }
 
 
     @Test
-    void test6() throws Exception {
+    void test6() throws SqlParseException {
 
         String sql1 =
                 "SELECT COUNT(*) FROM Department join Visits WHERE 总访问次数 > 100 ORDER BY 总访问次数 LIMIT 10";
@@ -158,36 +159,7 @@ class SqlWithMergerTest {
                 "WITH t_1 AS (SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv` FROM "
                         + "(SELECT `user_name`, `department` FROM `s2_user_department`) AS `t2` LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`,"
                         + " `imp_date` AS `sys_imp_date`, `user_name` FROM `s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) "
-                        + "SELECT COUNT(*) FROM Department JOIN Visits WHERE 总访问次数 > 100 ORDER BY 总访问次数 LIMIT 10");
-    }
-
-    @Test
-    void test7() throws Exception {
-
-        String sql1 =
-                "SELECT COUNT(*) FROM Department join Visits WHERE 总访问次数 > 100 AND imp_date >= CURRENT_DATE - "
-                        + "INTERVAL '1 year' AND  sys_imp_date < CURRENT_DATE ORDER"
-                        + " BY 总访问次数 LIMIT 10";
-
-        String sql2 =
-                "SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv`\n"
-                        + "FROM\n" + "(SELECT `user_name`, `department`\n" + "FROM\n"
-                        + "`s2_user_department`) AS `t2`\n"
-                        + "LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`, `imp_date` AS `sys_imp_date`, `user_name`\n"
-                        + "FROM\n"
-                        + "`s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`";
-
-        String mergeSql = SqlMergeWithUtils.mergeWith(EngineType.MYSQL, sql1,
-                Collections.singletonList(sql2), Collections.singletonList("t_1"));
-
-
-        Assert.assertEquals(format(mergeSql),
-                "WITH t_1 AS (SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv` FROM "
-                        + "(SELECT `user_name`, `department` FROM `s2_user_department`) AS `t2` LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`,"
-                        + " `imp_date` AS `sys_imp_date`, `user_name` FROM `s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) "
-                        + "SELECT COUNT(*) FROM Department JOIN Visits WHERE 总访问次数 > 100 AND imp_date >= "
-                        + "CURRENT_DATE - INTERVAL '1 year' AND sys_imp_date < CURRENT_DATE ORDER BY 总访问次数 "
-                        + "LIMIT 10");
+                        + "SELECT COUNT(*) FROM Department INNER JOIN Visits WHERE 总访问次数 > 100 ORDER BY 总访问次数 LIMIT 10");
     }
 
     private static String format(String mergeSql) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java

@@ -24,8 +24,6 @@ public class ModelDetail {
 
     private String tableQuery;
 
-    private String filterSql;
-
     private List<Identify> identifiers = Lists.newArrayList();
 
     private List<Dimension> dimensions = Lists.newArrayList();

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java

@@ -19,8 +19,6 @@ public class ModelBuildReq {
 
     private String sql;
 
-    private String filterSql;
-
     private String catalog;
 
     private String db;

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java

@@ -50,19 +50,19 @@ public class SqlQueryParser implements QueryParser {
         queryFields.removeAll(queryAliases);
         Ontology ontology = queryStatement.getOntology();
         OntologyQuery ontologyQuery = buildOntologyQuery(ontology, queryFields);
-        // // check if there are fields not matched with any metric or dimension
-        // if (queryFields.size() > ontologyQuery.getMetrics().size()
-        //         + ontologyQuery.getDimensions().size()) {
-        //     List<String> semanticFields = Lists.newArrayList();
-        //     ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName()));
-        //     ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName()));
-        //     String errMsg =
-        //             String.format("Querying columns[%s] not matched with semantic fields[%s].",
-        //                     queryFields, semanticFields);
-        //     queryStatement.setErrMsg(errMsg);
-        //     queryStatement.setStatus(QueryState.INVALID);
-        //     return;
-        // }
+        // check if there are fields not matched with any metric or dimension
+        if (queryFields.size() > ontologyQuery.getMetrics().size()
+                + ontologyQuery.getDimensions().size()) {
+            List<String> semanticFields = Lists.newArrayList();
+            ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName()));
+            ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName()));
+            String errMsg =
+                    String.format("Querying columns[%s] not matched with semantic fields[%s].",
+                            queryFields, semanticFields);
+            queryStatement.setErrMsg(errMsg);
+            queryStatement.setStatus(QueryState.INVALID);
+            return;
+        }
         queryStatement.setOntologyQuery(ontologyQuery);
 
         AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), ontologyQuery.getMetrics());

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java

@@ -40,23 +40,11 @@ public class DataModelNode extends SemanticNode {
                     .equalsIgnoreCase(EngineType.POSTGRESQL.getName())) {
                 String fullTableName = String.join(".public.",
                         dataModel.getModelDetail().getTableQuery().split("\\."));
-                sqlTable = "SELECT * FROM " + fullTableName;
+                sqlTable = "select * from " + fullTableName;
             } else {
-                sqlTable = "SELECT * FROM " + dataModel.getModelDetail().getTableQuery();
+                sqlTable = "select * from " + dataModel.getModelDetail().getTableQuery();
             }
         }
-
-        // String filterSql = dataModel.getFilterSql();
-        String filterSql = dataModel.getModelDetail().getFilterSql();
-        if (filterSql != null && !filterSql.isEmpty()) {
-            boolean sqlContainWhere = sqlTable.toUpperCase().matches("(?s).*\\bWHERE\\b.*");
-            if (sqlContainWhere) {
-                sqlTable = String.format("%s AND %s", sqlTable, filterSql);
-            } else {
-                sqlTable = String.format("%s WHERE %s", sqlTable, filterSql);
-            }
-        }
-
         if (sqlTable.isEmpty()) {
             throw new Exception("DataModelNode build error [tableSqlNode not found]");
         }
@@ -81,7 +69,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static void addSchemaTable(SqlValidatorScope scope, ModelResp dataModel, String db,
-                                       String tb, Set<String> fields) throws Exception {
+            String tb, Set<String> fields) throws Exception {
         Set<String> dateInfo = new HashSet<>();
         Set<String> dimensions = new HashSet<>();
         Set<String> metrics = new HashSet<>();
@@ -118,7 +106,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     public static List<SqlNode> getExtendField(Map<String, String> exprList,
-                                               SqlValidatorScope scope, EngineType engineType) throws Exception {
+            SqlValidatorScope scope, EngineType engineType) throws Exception {
         List<SqlNode> sqlNodeList = new ArrayList<>();
         for (String expr : exprList.keySet()) {
             sqlNodeList.add(parse(expr, scope, engineType));
@@ -139,7 +127,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     public static List<ModelResp> getQueryDataModels(Ontology ontology,
-                                                     OntologyQuery ontologyQuery) {
+            OntologyQuery ontologyQuery) {
         // get query measures and dimensions
         Set<String> queryMeasures = new HashSet<>();
         Set<String> queryDimensions = new HashSet<>();
@@ -174,7 +162,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery ontologyQuery,
-                                                Set<String> queryDimensions, Set<String> queryMeasures) {
+            Set<String> queryDimensions, Set<String> queryMeasures) {
         ontologyQuery.getMetrics().forEach(m -> {
             if (Objects.nonNull(m.getMetricDefineByMeasureParams())) {
                 m.getMetricDefineByMeasureParams().getMeasures()
@@ -227,7 +215,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static boolean checkMatch(ModelResp baseDataModel, Set<String> queryMeasures,
-                                      Set<String> queryDimension) {
+            Set<String> queryDimension) {
         boolean isAllMatch = true;
         Set<String> baseMeasures = baseDataModel.getMeasures().stream().map(Measure::getName)
                 .collect(Collectors.toSet());
@@ -260,8 +248,8 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static List<ModelResp> findRelatedModelsByRelation(Ontology ontology,
-                                                               OntologyQuery ontologyQuery, ModelResp baseDataModel, Set<String> queryDimensions,
-                                                               Set<String> queryMeasures) {
+            OntologyQuery ontologyQuery, ModelResp baseDataModel, Set<String> queryDimensions,
+            Set<String> queryMeasures) {
         Set<String> joinDataModelNames = new HashSet<>();
         List<ModelResp> joinDataModels = new ArrayList<>();
         Set<String> before = new HashSet<>();
@@ -345,7 +333,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static void sortJoinRelation(List<JoinRelation> joinRelations, String next,
-                                         Set<Long> visited, List<JoinRelation> sortedJoins) {
+            Set<Long> visited, List<JoinRelation> sortedJoins) {
         for (JoinRelation link : joinRelations) {
             if (!visited.contains(link.getId())) {
                 if (link.getLeft().equals(next) || link.getRight().equals(next)) {
@@ -360,7 +348,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static List<ModelResp> findRelatedModelsByIdentifier(Ontology ontology,
-                                                                 ModelResp baseDataModel, Set<String> queryDimension, Set<String> measures) {
+            ModelResp baseDataModel, Set<String> queryDimension, Set<String> measures) {
         Set<String> baseIdentifiers = baseDataModel.getModelDetail().getIdentifiers().stream()
                 .map(Identify::getName).collect(Collectors.toSet());
         if (baseIdentifiers.isEmpty()) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -36,7 +36,6 @@ public class ModelYamlManager {
         } else {
             dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
         }
-        dataModelYamlTpl.setFilterSql(modelDetail.getFilterSql());
         dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
         dataModelYamlTpl.setId(modelResp.getId());
         return dataModelYamlTpl;

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -97,7 +97,6 @@ public class SemanticSchemaManager {
         modelDetail.setDbType(d.getType());
         modelDetail.setSqlQuery(d.getSqlQuery());
         modelDetail.setTableQuery(d.getTableQuery());
-        modelDetail.setFilterSql(d.getFilterSql());
         modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers()));
         modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
         modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java

@@ -21,8 +21,6 @@ public class DataModelYamlTpl {
 
     private String tableQuery;
 
-    private String filterSql;
-
     private List<IdentifyYamlTpl> identifiers;
 
     private List<DimensionYamlTpl> dimensions;

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -157,7 +157,6 @@ public class ModelConverter {
             modelDetail.setQueryType(ModelDefineType.TABLE_QUERY.getName());
             modelDetail.setTableQuery(String.format("%s.%s", modelBuildReq.getDb(), tableName));
         }
-        modelDetail.setFilterSql(modelBuildReq.getFilterSql());
         for (ColumnSchema columnSchema : modelSchema.getColumnSchemas()) {
             FieldType fieldType = columnSchema.getFiledType();
             if (getIdentifyType(fieldType) != null) {