Merge 668f872743 into a3293e6788

(improvement)(headless)Optimize compatibility and robustness in ontology query translation.

.github/workflows/centos-ci.yml

@@ -47,7 +47,7 @@ jobs:
           mvn -version
 
       - name: Cache Maven packages
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

.github/workflows/mac-ci.yml

@@ -26,7 +26,7 @@ jobs:
           distribution: 'adopt'
 
       - name: Cache Maven packages
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/Library/Caches/Maven  # macOS Maven cache path
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

.github/workflows/ubuntu-ci.yml

@@ -26,7 +26,7 @@ jobs:
           distribution: 'adopt'
 
       - name: Cache Maven packages
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

.github/workflows/windows-ci.yml

@@ -26,7 +26,7 @@ jobs:
           distribution: 'adopt'  # You might need to change this if 'adopt' doesn't support JDK 21
 
       - name: Cache Maven packages
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~\.m2  # Windows uses a backslash for paths
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

LICENSE

@@ -14,7 +14,7 @@ code and logo.
 b. a commercial license must be obtained from the author if you want to develop and distribute a derivative work based
 on SuperSonic.
 
-Please contact zhangjun2915@163.com by email to inquire about licensing matters.
+Please contact supersonicbi@qq.com by email to inquire about licensing matters.
 
 
 2. As a contributor, you should agree that:

README.md

@@ -75,7 +75,7 @@ SuperSonic comes with sample semantic models as well as chat conversations that
 
 ## Build and Development
 
-Please refer to project [Docs](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E7%BC%96%E8%AF%91%E6%9E%84%E5%BB%BA/). 
+Please refer to project [Docs](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E6%BA%90%E7%A0%81%E7%BC%96%E8%AF%91%E9%83%A8%E7%BD%B2/). 
 
 ## WeChat Contact
 

README_CN.md

@@ -75,7 +75,7 @@ SuperSonic自带样例的语义模型和问答对话，只需以下三步即可
 
 ## 如何构建和部署
 
-请参考项目[文档](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E7%BC%96%E8%AF%91%E6%9E%84%E5%BB%BA/)。
+请参考项目[文档](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E6%BA%90%E7%A0%81%E7%BC%96%E8%AF%91%E9%83%A8%E7%BD%B2/)。
 
 ## 微信联系方式
 

README_JP.md

@@ -71,7 +71,7 @@ SuperSonicには、サンプルのセマンティックモデルとチャット
 
 ## ビルドと開発
 
-プロジェクト[ドキュメント](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E7%BC%96%E8%AF%91%E6%9E%84%E5%BB%BA/)を参照してください。
+プロジェクト[ドキュメント](https://supersonicbi.github.io/docs/%E7%B3%BB%E7%BB%9F%E9%83%A8%E7%BD%B2/%E6%BA%90%E7%A0%81%E7%BC%96%E8%AF%91%E9%83%A8%E7%BD%B2/)を参照してください。
 
 ## WeChat連絡先
 

auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/AuthenticationInterceptor.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.auth.authentication.interceptor;
 
 import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
-import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
+import com.tencent.supersonic.auth.api.authentication.service.UserService;
 import com.tencent.supersonic.auth.authentication.utils.TokenService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -16,7 +16,7 @@ public abstract class AuthenticationInterceptor implements HandlerInterceptor {
 
     protected AuthenticationConfig authenticationConfig;
 
-    protected UserServiceImpl userServiceImpl;
+    protected UserService userService;
 
     protected TokenService tokenService;
 

auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/interceptor/DefaultAuthenticationInterceptor.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.auth.authentication.interceptor;
 import com.tencent.supersonic.auth.api.authentication.annotation.AuthenticationIgnore;
 import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
 import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
-import com.tencent.supersonic.auth.authentication.service.UserServiceImpl;
+import com.tencent.supersonic.auth.api.authentication.service.UserService;
 import com.tencent.supersonic.auth.authentication.utils.TokenService;
 import com.tencent.supersonic.common.pojo.exception.AccessException;
 import com.tencent.supersonic.common.util.ContextUtils;
@@ -16,12 +16,7 @@ import org.springframework.web.method.HandlerMethod;
 import java.lang.reflect.Method;
 import java.util.Optional;
 
-import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_IS_ADMIN;
-import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_DISPLAY_NAME;
-import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_EMAIL;
-import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_ID;
-import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_NAME;
-import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.TOKEN_USER_PASSWORD;
+import static com.tencent.supersonic.auth.api.authentication.constant.UserConstants.*;
 
 @Slf4j
 public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor {
@@ -30,7 +25,7 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
     public boolean preHandle(HttpServletRequest request, HttpServletResponse response,
             Object handler) throws AccessException {
         authenticationConfig = ContextUtils.getBean(AuthenticationConfig.class);
-        userServiceImpl = ContextUtils.getBean(UserServiceImpl.class);
+        userService = ContextUtils.getBean(UserService.class);
         tokenService = ContextUtils.getBean(TokenService.class);
         if (!authenticationConfig.isEnabled()) {
             return true;

common/src/main/java/com/tencent/supersonic/common/calcite/SqlDialectFactory.java

@@ -26,6 +26,16 @@ public class SqlDialectFactory {
                     .withLiteralQuoteString("'").withIdentifierQuoteString("\"")
                     .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
                     .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
+    public static final Context PRESTO_CONTEXT =
+            SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.PRESTO)
+                    .withLiteralQuoteString("'").withIdentifierQuoteString("\"")
+                    .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
+                    .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
+    public static final Context KYUUBI_CONTEXT =
+            SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
+                    .withLiteralQuoteString("'").withIdentifierQuoteString("`")
+                    .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
+                    .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(false);
     private static Map<EngineType, SemanticSqlDialect> sqlDialectMap;
 
     static {
@@ -35,6 +45,10 @@ public class SqlDialectFactory {
         sqlDialectMap.put(EngineType.H2, new SemanticSqlDialect(DEFAULT_CONTEXT));
         sqlDialectMap.put(EngineType.POSTGRESQL, new SemanticSqlDialect(POSTGRESQL_CONTEXT));
         sqlDialectMap.put(EngineType.HANADB, new SemanticSqlDialect(HANADB_CONTEXT));
+        sqlDialectMap.put(EngineType.STARROCKS, new SemanticSqlDialect(DEFAULT_CONTEXT));
+        sqlDialectMap.put(EngineType.KYUUBI, new SemanticSqlDialect(KYUUBI_CONTEXT));
+        sqlDialectMap.put(EngineType.PRESTO, new SemanticSqlDialect(PRESTO_CONTEXT));
+        sqlDialectMap.put(EngineType.TRINO, new SemanticSqlDialect(PRESTO_CONTEXT));
     }
 
     public static SemanticSqlDialect getSqlDialect(EngineType engineType) {

common/src/main/java/com/tencent/supersonic/common/calcite/SqlMergeWithUtils.java

@@ -2,19 +2,11 @@ package com.tencent.supersonic.common.calcite;
 
 import com.tencent.supersonic.common.pojo.enums.EngineType;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.calcite.sql.SqlLiteral;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.SqlOrderBy;
-import org.apache.calcite.sql.SqlSelect;
-import org.apache.calcite.sql.SqlWith;
-import org.apache.calcite.sql.SqlWithItem;
-import org.apache.calcite.sql.SqlWriterConfig;
-import org.apache.calcite.sql.parser.SqlParseException;
-import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.calcite.sql.parser.SqlParserPos;
-import org.apache.calcite.sql.pretty.SqlPrettyWriter;
+import net.sf.jsqlparser.expression.Alias;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.statement.select.ParenthesedSelect;
+import net.sf.jsqlparser.statement.select.Select;
+import net.sf.jsqlparser.statement.select.WithItem;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -22,85 +14,37 @@ import java.util.List;
 @Slf4j
 public class SqlMergeWithUtils {
     public static String mergeWith(EngineType engineType, String sql, List<String> parentSqlList,
-            List<String> parentWithNameList) throws SqlParseException {
-        SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
+            List<String> parentWithNameList) throws Exception {
 
-        // Parse the main SQL statement
-        SqlParser parser = SqlParser.create(sql, parserConfig);
-        SqlNode sqlNode1 = parser.parseQuery();
+        Select selectStatement = (Select) CCJSqlParserUtil.parse(sql);
+        List<WithItem> withItemList = new ArrayList<>();
 
-        // List to hold all WITH items
-        List<SqlNode> withItemList = new ArrayList<>();
-
-        // Iterate over each parentSql and parentWithName pair
         for (int i = 0; i < parentSqlList.size(); i++) {
             String parentSql = parentSqlList.get(i);
             String parentWithName = parentWithNameList.get(i);
 
-            // Parse the parent SQL statement
-            parser = SqlParser.create(parentSql, parserConfig);
-            SqlNode sqlNode2 = parser.parseQuery();
+            Select parentSelect = (Select) CCJSqlParserUtil.parse(parentSql);
+            ParenthesedSelect select = new ParenthesedSelect();
+            select.setSelect(parentSelect);
 
             // Create a new WITH item for parentWithName without quotes
-            SqlWithItem withItem = new SqlWithItem(SqlParserPos.ZERO,
-                    new SqlIdentifier(parentWithName, SqlParserPos.ZERO), null, sqlNode2,
-                    SqlLiteral.createBoolean(false, SqlParserPos.ZERO));
+            WithItem withItem = new WithItem();
+            withItem.setAlias(new Alias(parentWithName));
+            withItem.setSelect(select);
 
             // Add the new WITH item to the list
             withItemList.add(withItem);
         }
 
-        // Check if the main SQL node contains an ORDER BY or LIMIT clause
-        SqlNode limitNode = null;
-        SqlNodeList orderByList = null;
-        if (sqlNode1 instanceof SqlOrderBy) {
-            SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode1;
-            limitNode = sqlOrderBy.fetch;
-            orderByList = sqlOrderBy.orderList;
-            sqlNode1 = sqlOrderBy.query;
-        } else if (sqlNode1 instanceof SqlSelect) {
-            SqlSelect sqlSelect = (SqlSelect) sqlNode1;
-            limitNode = sqlSelect.getFetch();
-            sqlSelect.setFetch(null);
-            sqlNode1 = sqlSelect;
+        // Extract existing WITH items from mainSelectBody if it has any
+        if (selectStatement.getWithItemsList() != null) {
+            withItemList.addAll(selectStatement.getWithItemsList());
         }
 
-        // Extract existing WITH items from sqlNode1 if it is a SqlWith
-        if (sqlNode1 instanceof SqlWith) {
-            SqlWith sqlWith = (SqlWith) sqlNode1;
-            withItemList.addAll(sqlWith.withList.getList());
-            sqlNode1 = sqlWith.body;
-        }
+        // Set the new WITH items list to the main select body
+        selectStatement.setWithItemsList(withItemList);
 
-        // Create a new SqlWith node
-        SqlWith finalSqlNode = new SqlWith(SqlParserPos.ZERO,
-                new SqlNodeList(withItemList, SqlParserPos.ZERO), sqlNode1);
-
-        // If there was an ORDER BY or LIMIT clause, wrap the finalSqlNode in a SqlOrderBy
-        SqlNode resultNode = finalSqlNode;
-        if (orderByList != null || limitNode != null) {
-            resultNode = new SqlOrderBy(SqlParserPos.ZERO, finalSqlNode,
-                    orderByList != null ? orderByList : SqlNodeList.EMPTY, null, limitNode);
-        }
-
-        // Custom SqlPrettyWriter configuration to avoid quoting identifiers
-        SqlWriterConfig config = Configuration.getSqlWriterConfig(engineType);
         // Pretty print the final SQL
-        SqlPrettyWriter writer = new SqlPrettyWriter(config);
-        return writer.format(resultNode);
-    }
-
-    public static boolean hasWith(EngineType engineType, String sql) throws SqlParseException {
-        SqlParser.Config parserConfig = Configuration.getParserConfig(engineType);
-        SqlParser parser = SqlParser.create(sql, parserConfig);
-        SqlNode sqlNode = parser.parseQuery();
-        SqlNode sqlSelect = sqlNode;
-        if (sqlNode instanceof SqlOrderBy) {
-            SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode;
-            sqlSelect = sqlOrderBy.query;
-        } else if (sqlNode instanceof SqlSelect) {
-            sqlSelect = (SqlSelect) sqlNode;
-        }
-        return sqlSelect instanceof SqlWith;
+        return selectStatement.toString();
     }
 }

common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java

@@ -91,7 +91,8 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
         }
 
         ExpressionList<?> leftFunctionParams = leftFunction.getParameters();
-        if (CollectionUtils.isEmpty(leftFunctionParams)) {
+        if (CollectionUtils.isEmpty(leftFunctionParams)
+                || !(leftFunctionParams.get(0) instanceof Column)) {
             return result;
         }
 

common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java

@@ -225,7 +225,7 @@ public class SqlSelectHelper {
     public static Select getSelect(String sql) {
         Statement statement = null;
         try {
-            statement = CCJSqlParserUtil.parse(sql);
+            statement = CCJSqlParserUtil.parse(sql, parser -> parser.withTimeOut(20000));
         } catch (JSQLParserException e) {
             log.error("parse error, sql:{}", sql, e);
             throw new RuntimeException(e);
@@ -989,6 +989,15 @@ public class SqlSelectHelper {
         for (SelectItem selectItem : selectItems) {
             selectItem.accept(visitor);
         }
+        if (plainSelect.getHaving() != null) {
+            plainSelect.getHaving().accept(visitor);
+        }
+        if (!CollectionUtils.isEmpty(plainSelect.getOrderByElements())) {
+            for (OrderByElement orderByElement : plainSelect.getOrderByElements()) {
+                orderByElement.getExpression().accept(visitor);
+            }
+        }
+
         return !visitor.getFunctionNames().isEmpty();
     }
 

common/src/main/java/com/tencent/supersonic/common/pojo/Order.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.common.pojo;
 import com.google.common.base.Objects;
 import jakarta.validation.constraints.NotBlank;
 import lombok.Data;
+
 import java.io.Serializable;
 
 import static com.tencent.supersonic.common.pojo.Constants.ASC_UPPER;

common/src/test/java/com/tencent/supersonic/common/calcite/SqlWithMergerTest.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.common.calcite;
 
 import com.tencent.supersonic.common.pojo.enums.EngineType;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.calcite.sql.parser.SqlParseException;
 import org.junit.Assert;
 import org.junit.jupiter.api.Test;
 
@@ -12,7 +11,7 @@ import java.util.Collections;
 class SqlWithMergerTest {
 
     @Test
-    void test1() throws SqlParseException {
+    void test1() throws Exception {
         String sql1 = "WITH DepartmentVisits AS (\n" + "    SELECT department, SUM(pv) AS 总访问次数\n"
                 + "    FROM t_1\n"
                 + "    WHERE sys_imp_date >= '2024-09-01' AND sys_imp_date <= '2024-09-29'\n"
@@ -38,7 +37,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test2() throws SqlParseException {
+    void test2() throws Exception {
 
         String sql1 =
                 "WITH DepartmentVisits AS (SELECT department, SUM(pv) AS 总访问次数 FROM t_1 WHERE sys_imp_date >= '2024-08-28' "
@@ -65,7 +64,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test3() throws SqlParseException {
+    void test3() throws Exception {
 
         String sql1 = "SELECT COUNT(*) FROM DepartmentVisits WHERE 总访问次数 > 100 LIMIT 1000";
 
@@ -89,7 +88,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test4() throws SqlParseException {
+    void test4() throws Exception {
         String sql1 = "SELECT COUNT(*) FROM DepartmentVisits WHERE 总访问次数 > 100";
 
         String sql2 =
@@ -112,7 +111,7 @@ class SqlWithMergerTest {
     }
 
     @Test
-    void test5() throws SqlParseException {
+    void test5() throws Exception {
 
         String sql1 = "SELECT COUNT(*) FROM Department join Visits WHERE 总访问次数 > 100";
 
@@ -132,13 +131,13 @@ class SqlWithMergerTest {
                 "WITH t_1 AS (SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv` "
                         + "FROM (SELECT `user_name`, `department` FROM `s2_user_department`) AS `t2` LEFT JOIN "
                         + "(SELECT 1 AS `s2_pv_uv_statis_pv`, `imp_date` AS `sys_imp_date`, `user_name` FROM `s2_pv_uv_statis`) "
-                        + "AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) SELECT COUNT(*) FROM Department INNER JOIN Visits "
+                        + "AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) SELECT COUNT(*) FROM Department JOIN Visits "
                         + "WHERE 总访问次数 > 100");
     }
 
 
     @Test
-    void test6() throws SqlParseException {
+    void test6() throws Exception {
 
         String sql1 =
                 "SELECT COUNT(*) FROM Department join Visits WHERE 总访问次数 > 100 ORDER BY 总访问次数 LIMIT 10";
@@ -159,7 +158,36 @@ class SqlWithMergerTest {
                 "WITH t_1 AS (SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv` FROM "
                         + "(SELECT `user_name`, `department` FROM `s2_user_department`) AS `t2` LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`,"
                         + " `imp_date` AS `sys_imp_date`, `user_name` FROM `s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) "
-                        + "SELECT COUNT(*) FROM Department INNER JOIN Visits WHERE 总访问次数 > 100 ORDER BY 总访问次数 LIMIT 10");
+                        + "SELECT COUNT(*) FROM Department JOIN Visits WHERE 总访问次数 > 100 ORDER BY 总访问次数 LIMIT 10");
+    }
+
+    @Test
+    void test7() throws Exception {
+
+        String sql1 =
+                "SELECT COUNT(*) FROM Department join Visits WHERE 总访问次数 > 100 AND imp_date >= CURRENT_DATE - "
+                        + "INTERVAL '1 year' AND  sys_imp_date < CURRENT_DATE ORDER"
+                        + " BY 总访问次数 LIMIT 10";
+
+        String sql2 =
+                "SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv`\n"
+                        + "FROM\n" + "(SELECT `user_name`, `department`\n" + "FROM\n"
+                        + "`s2_user_department`) AS `t2`\n"
+                        + "LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`, `imp_date` AS `sys_imp_date`, `user_name`\n"
+                        + "FROM\n"
+                        + "`s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`";
+
+        String mergeSql = SqlMergeWithUtils.mergeWith(EngineType.MYSQL, sql1,
+                Collections.singletonList(sql2), Collections.singletonList("t_1"));
+
+
+        Assert.assertEquals(format(mergeSql),
+                "WITH t_1 AS (SELECT `t3`.`sys_imp_date`, `t2`.`department`, `t3`.`s2_pv_uv_statis_pv` AS `pv` FROM "
+                        + "(SELECT `user_name`, `department` FROM `s2_user_department`) AS `t2` LEFT JOIN (SELECT 1 AS `s2_pv_uv_statis_pv`,"
+                        + " `imp_date` AS `sys_imp_date`, `user_name` FROM `s2_pv_uv_statis`) AS `t3` ON `t2`.`user_name` = `t3`.`user_name`) "
+                        + "SELECT COUNT(*) FROM Department JOIN Visits WHERE 总访问次数 > 100 AND imp_date >= "
+                        + "CURRENT_DATE - INTERVAL '1 year' AND sys_imp_date < CURRENT_DATE ORDER BY 总访问次数 "
+                        + "LIMIT 10");
     }
 
     private static String format(String mergeSql) {

docker/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     ports:
       - "15432:5432"
 #    volumes:
-#      - postgres_data:/var/lib/postgresql
+#      - postgres_data:/var/lib/postgresql/data
     networks:
       - supersonic_network
     dns:
@@ -62,4 +62,4 @@ services:
 #  supersonic_data:
 
 networks:
-  supersonic_network:
+  supersonic_network:

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java

@@ -23,12 +23,16 @@ public class Measure {
 
     private String alias;
 
-    public Measure(String name, String bizName, String expr, String agg, Integer isCreateMetric) {
+    private String unit;
+
+    public Measure(String name, String bizName, String expr, String agg, String unit,
+            Integer isCreateMetric) {
         this.name = name;
         this.agg = agg;
         this.isCreateMetric = isCreateMetric;
         this.bizName = bizName;
         this.expr = expr;
+        this.unit = unit;
     }
 
     public Measure(String name, String bizName, String agg, Integer isCreateMetric) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java

@@ -49,27 +49,4 @@ public class ModelDetail {
                 .collect(Collectors.toList());
     }
 
-    public List<Field> getFields() {
-        if (!CollectionUtils.isEmpty(fields)) {
-            return fields;
-        }
-        List<Field> fieldList = Lists.newArrayList();
-        // Compatible with older versions
-        if (!CollectionUtils.isEmpty(identifiers)) {
-            fieldList.addAll(identifiers.stream()
-                    .map(identify -> Field.builder().fieldName(identify.getFieldName()).build())
-                    .collect(Collectors.toSet()));
-        }
-        if (!CollectionUtils.isEmpty(dimensions)) {
-            fieldList.addAll(dimensions.stream()
-                    .map(dim -> Field.builder().fieldName(dim.getFieldName()).build())
-                    .collect(Collectors.toSet()));
-        }
-        if (!CollectionUtils.isEmpty(measures)) {
-            fieldList.addAll(measures.stream()
-                    .map(measure -> Field.builder().fieldName(measure.getFieldName()).build())
-                    .collect(Collectors.toSet()));
-        }
-        return fieldList;
-    }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelSchema.java

@@ -14,11 +14,11 @@ public class ModelSchema {
 
     private String description;
 
-    private List<ColumnSchema> columnSchemas;
+    private List<SemanticColumn> semanticColumns;
 
     @JsonIgnore
-    public ColumnSchema getColumnByName(String columnName) {
-        for (ColumnSchema fieldSchema : columnSchemas) {
+    public SemanticColumn getColumnByName(String columnName) {
+        for (SemanticColumn fieldSchema : semanticColumns) {
             if (fieldSchema.getColumnName().equalsIgnoreCase(columnName)) {
                 return fieldSchema;
             }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticColumn.java

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.FieldType;
 import lombok.Data;
 
 @Data
-public class ColumnSchema {
+public class SemanticColumn {
 
     private String columnName;
 
@@ -19,4 +19,8 @@ public class ColumnSchema {
 
     private String name;
 
+    private String expr;
+
+    private String unit;
+
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/adaptor/db/PrestoAdaptor.java

@@ -83,6 +83,7 @@ public class PrestoAdaptor extends BaseDbAdaptor {
 
     @Override
     public String rewriteSql(String sql) {
+        sql = sql.replaceAll("`", "\"");
         return sql;
     }
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.headless.core.translator;
 
 import com.tencent.supersonic.common.calcite.SqlMergeWithUtils;
+import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
 import com.tencent.supersonic.common.pojo.enums.EngineType;
 import com.tencent.supersonic.headless.api.pojo.response.QueryState;
 import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -73,7 +74,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
         String finalSql = null;
         if (sqlQuery.isSupportWith()) {
             EngineType engineType = queryStatement.getOntology().getDatabaseType();
-            if (!SqlMergeWithUtils.hasWith(engineType, ontologyOuterSql)) {
+            if (!SqlSelectHelper.hasWith(ontologyOuterSql)) {
                 finalSql = "with " + tables.stream()
                         .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight()))
                         .collect(Collectors.joining(",")) + "\n" + ontologyOuterSql;

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SemanticNode.java

@@ -78,7 +78,8 @@ public abstract class SemanticNode {
                 scope.getValidator().getCatalogReader().getRootSchema(), engineType);
         if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType).getReservedAndKeyWords()
                 .contains(expression.toUpperCase())) {
-            if (engineType == EngineType.HANADB) {
+            if (engineType == EngineType.HANADB || engineType == EngineType.PRESTO
+                    || engineType == EngineType.TRINO) {
                 expression = String.format("\"%s\"", expression);
             } else {
                 expression = String.format("`%s`", expression);
@@ -166,9 +167,9 @@ public abstract class SemanticNode {
         if (sqlNode instanceof SqlBasicCall) {
             SqlBasicCall sqlBasicCall = (SqlBasicCall) sqlNode;
             if (sqlBasicCall.getOperator().getKind().equals(SqlKind.AS)) {
-                if (sqlBasicCall.getOperandList().get(0) instanceof SqlSelect) {
-                    SqlSelect table = (SqlSelect) sqlBasicCall.getOperandList().get(0);
-                    return table;
+                SqlNode innerQuery = sqlBasicCall.getOperandList().get(0);
+                if (innerQuery instanceof SqlCall) {
+                    return innerQuery;
                 }
             }
         }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java

@@ -69,7 +69,13 @@ public class SqlBuilder {
         SqlNode parserNode = tableView.build();
         DatabaseResp database = queryStatement.getOntology().getDatabase();
         EngineType engineType = EngineType.fromString(database.getType());
-        parserNode = optimizeParseNode(parserNode, engineType);
+        try {
+            parserNode = optimizeParseNode(parserNode, engineType);
+        } catch (Exception e) {
+            // failure in optimization phase doesn't affect the query result,
+            // just ignore it
+            log.error("optimizeParseNode error", e);
+        }
         return SemanticNode.getSql(parserNode, engineType);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -100,6 +100,7 @@ public class SemanticSchemaManager {
         modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers()));
         modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
         modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));
+        modelDetail.getFields().addAll(d.getFields());
 
         return dataModel;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/LLMSemanticModeller.java

@@ -75,6 +75,7 @@ public class LLMSemanticModeller implements SemanticModeller {
         if (!chatApp.isPresent() || !chatApp.get().isEnable()) {
             return;
         }
+
         List<DbSchema> otherDbSchema = getOtherDbSchema(dbSchema, dbSchemas);
         ModelSchemaExtractor extractor =
                 AiServices.create(ModelSchemaExtractor.class, getChatModel(modelBuildReq));

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/RuleSemanticModeller.java

@@ -1,9 +1,9 @@
 package com.tencent.supersonic.headless.server.modeller;
 
-import com.tencent.supersonic.headless.api.pojo.ColumnSchema;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
 import com.tencent.supersonic.headless.api.pojo.DbSchema;
 import com.tencent.supersonic.headless.api.pojo.ModelSchema;
+import com.tencent.supersonic.headless.api.pojo.SemanticColumn;
 import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
 import lombok.extern.slf4j.Slf4j;
 
@@ -16,19 +16,20 @@ public class RuleSemanticModeller implements SemanticModeller {
     @Override
     public void build(DbSchema dbSchema, List<DbSchema> dbSchemas, ModelSchema modelSchema,
             ModelBuildReq modelBuildReq) {
-        List<ColumnSchema> columnSchemas =
+        List<SemanticColumn> semanticColumns =
                 dbSchema.getDbColumns().stream().map(this::convert).collect(Collectors.toList());
-        modelSchema.setColumnSchemas(columnSchemas);
+        modelSchema.setSemanticColumns(semanticColumns);
     }
 
-    private ColumnSchema convert(DBColumn dbColumn) {
-        ColumnSchema columnSchema = new ColumnSchema();
-        columnSchema.setName(dbColumn.getColumnName());
-        columnSchema.setColumnName(dbColumn.getColumnName());
-        columnSchema.setComment(dbColumn.getComment());
-        columnSchema.setDataType(dbColumn.getDataType());
-        columnSchema.setFiledType(dbColumn.getFieldType());
-        return columnSchema;
+    private SemanticColumn convert(DBColumn dbColumn) {
+        SemanticColumn semanticColumn = new SemanticColumn();
+        semanticColumn.setName(dbColumn.getColumnName());
+        semanticColumn.setColumnName(dbColumn.getColumnName());
+        semanticColumn.setExpr(dbColumn.getColumnName());
+        semanticColumn.setComment(dbColumn.getComment());
+        semanticColumn.setDataType(dbColumn.getDataType());
+        semanticColumn.setFiledType(dbColumn.getFieldType());
+        return semanticColumn;
     }
 
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/modeller/SemanticModeller.java

@@ -7,6 +7,9 @@ import com.tencent.supersonic.headless.api.pojo.request.ModelBuildReq;
 
 import java.util.List;
 
+/**
+ * A semantic modeler builds semantic-layer schemas from database-layer schemas.
+ */
 public interface SemanticModeller {
 
     void build(DbSchema dbSchema, List<DbSchema> otherDbSchema, ModelSchema modelSchema,

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/DataSetController.java

@@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.server.rest;
 
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
 import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.headless.api.pojo.MetaFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DataSetReq;
 import com.tencent.supersonic.headless.api.pojo.response.DataSetResp;
@@ -51,6 +52,7 @@ public class DataSetController {
     public List<DataSetResp> getDataSetList(@RequestParam("domainId") Long domainId) {
         MetaFilter metaFilter = new MetaFilter();
         metaFilter.setDomainId(domainId);
+        metaFilter.setStatus(StatusEnum.ONLINE.getCode());
         return dataSetService.getDataSetList(metaFilter);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DataSetServiceImpl.java

@@ -56,9 +56,9 @@ public class DataSetServiceImpl extends ServiceImpl<DataSetDOMapper, DataSetDO>
     public DataSetResp save(DataSetReq dataSetReq, User user) {
         dataSetReq.createdBy(user.getName());
         DataSetDO dataSetDO = convert(dataSetReq);
-        dataSetDO.setStatus(StatusEnum.ONLINE.getCode());
+        dataSetDO.setStatus(dataSetReq.getStatus() != null ? dataSetReq.getStatus()
+                : StatusEnum.ONLINE.getCode());
         DataSetResp dataSetResp = convert(dataSetDO);
-        // conflictCheck(dataSetResp);
         save(dataSetDO);
         dataSetResp.setId(dataSetDO.getId());
         return dataSetResp;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.pojo.enums.EngineType;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
 import com.tencent.supersonic.headless.api.pojo.enums.DataType;
@@ -79,8 +80,9 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
 
     @Override
     public List<DatabaseResp> getDatabaseList(User user) {
-        List<DatabaseResp> databaseResps =
-                list().stream().map(DatabaseConverter::convert).collect(Collectors.toList());
+        List<DatabaseResp> databaseResps = list().stream().map(DatabaseConverter::convert)
+                .filter(database -> filterByAuth(database, user, AuthType.VIEWER))
+                .collect(Collectors.toList());
         fillPermission(databaseResps, user);
         return databaseResps;
     }
@@ -100,6 +102,43 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
         });
     }
 
+    private boolean filterByAuth(DatabaseResp database, User user, AuthType authType) {
+        if (user.isSuperAdmin() || user.getName().equals(database.getCreatedBy())) {
+            return true;
+        }
+        authType = authType == null ? AuthType.VIEWER : authType;
+        switch (authType) {
+            case ADMIN:
+                return checkAdminPermission(user, database);
+            case VIEWER:
+            default:
+                return checkViewPermission(user, database);
+        }
+    }
+
+    private boolean checkAdminPermission(User user, DatabaseResp database) {
+        List<String> admins = database.getAdmins();
+        if (user.isSuperAdmin()) {
+            return true;
+        }
+        if (admins.contains(user.getName()) || database.getCreatedBy().equals(user.getName())) {
+            return true;
+        }
+        return false;
+    }
+
+    private boolean checkViewPermission(User user, DatabaseResp database) {
+        if (checkAdminPermission(user, database)) {
+            return true;
+        }
+        List<String> viewers = database.getViewers();
+
+        if (viewers.contains(user.getName())) {
+            return true;
+        }
+        return false;
+    }
+
     @Override
     public void deleteDatabase(Long databaseId) {
         ModelFilter modelFilter = new ModelFilter();

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java

@@ -277,8 +277,8 @@ public class ModelServiceImpl implements ModelService {
         dimensionService.createDimensionBatch(dimensionReqs, user);
     }
 
-    private void batchCreateMetric(ModelDO datasourceDO, User user) throws Exception {
-        List<MetricReq> metricReqs = ModelConverter.convertMetricList(datasourceDO);
+    private void batchCreateMetric(ModelDO modelDO, User user) throws Exception {
+        List<MetricReq> metricReqs = ModelConverter.convertMetricList(modelDO);
         metricService.createMetricBatch(metricReqs, user);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -425,8 +425,12 @@ public class DictUtils {
                 .format(DateTimeFormatter.ofPattern(format));
         String end = LocalDate.now().minusDays(itemValueDateEnd)
                 .format(DateTimeFormatter.ofPattern(format));
-        return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start,
-                dateConf.getDateField(), end);
+        if (Objects.nonNull(dateConf)) {
+            return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start,
+                    dateConf.getDateField(), end);
+        } else {
+            return String.format("( %s >= '%s' and %s <= '%s' )", "dt", start, "dt", end);
+        }
     }
 
     private String generateDictDateFilter(DictItemResp dictItemResp) {
@@ -440,7 +444,7 @@ public class DictUtils {
         }
         // 未进行设置
         if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) {
-            return defaultDateFilter(config.getDateConf());
+            return defaultDateFilter(null);
         }
         // 全表扫描
         if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -157,23 +157,28 @@ public class ModelConverter {
             modelDetail.setQueryType(ModelDefineType.TABLE_QUERY.getName());
             modelDetail.setTableQuery(String.format("%s.%s", modelBuildReq.getDb(), tableName));
         }
-        for (ColumnSchema columnSchema : modelSchema.getColumnSchemas()) {
-            FieldType fieldType = columnSchema.getFiledType();
+        List<Field> fields = new ArrayList<>();
+        for (SemanticColumn semanticColumn : modelSchema.getSemanticColumns()) {
+            FieldType fieldType = semanticColumn.getFiledType();
+            fields.add(new Field(semanticColumn.getName(), semanticColumn.getDataType()));
+
             if (getIdentifyType(fieldType) != null) {
-                Identify identify = new Identify(columnSchema.getName(),
-                        getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1);
+                Identify identify = new Identify(semanticColumn.getName(),
+                        getIdentifyType(fieldType).name(), semanticColumn.getColumnName(), 1);
                 modelDetail.getIdentifiers().add(identify);
             } else if (FieldType.measure.equals(fieldType)) {
-                Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(),
-                        columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1);
+                Measure measure = new Measure(semanticColumn.getName(),
+                        semanticColumn.getColumnName(), semanticColumn.getExpr(),
+                        semanticColumn.getAgg().getOperator(), semanticColumn.getUnit(), 1);
                 modelDetail.getMeasures().add(measure);
             } else {
-                Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
-                        columnSchema.getColumnName(),
-                        DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
+                Dimension dim = new Dimension(semanticColumn.getName(),
+                        semanticColumn.getColumnName(), semanticColumn.getExpr(),
+                        DimensionType.valueOf(semanticColumn.getFiledType().name()), 1);
                 modelDetail.getDimensions().add(dim);
             }
         }
+        modelDetail.setFields(fields);
         modelReq.setModelDetail(modelDetail);
         return modelReq;
     }

launchers/standalone/src/main/resources/config.update/sql-update-mysql.sql

@@ -410,4 +410,7 @@ ALTER TABLE s2_query_stat_info RENAME COLUMN `sql` TO `query_sql`;
 --20250224
 ALTER TABLE s2_agent add column `admin_org` varchar(3000) DEFAULT NULL COMMENT '管理员组织';
 ALTER TABLE s2_agent add column `view_org` varchar(3000) DEFAULT NULL COMMENT '可用组织';
-ALTER TABLE s2_agent add column `is_open` tinyint DEFAULT NULL COMMENT '是否公开';
+ALTER TABLE s2_agent add column `is_open` tinyint DEFAULT NULL COMMENT '是否公开';
+
+--20250309
+ALTER TABLE s2_model_rela alter column join_condition type text;

launchers/standalone/src/main/resources/db/schema-h2.sql

@@ -268,7 +268,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
     from_model_id    BIGINT,
     to_model_id      BIGINT,
     join_type       VARCHAR(255),
-    join_condition  VARCHAR(255),
+    join_condition  TEXT,
     PRIMARY KEY (`id`)
 );
 

launchers/standalone/src/main/resources/db/schema-mysql.sql

@@ -404,7 +404,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
     from_model_id    bigint,
     to_model_id      bigint,
     join_type       VARCHAR(255),
-    join_condition  VARCHAR(255)
+    join_condition  text
 )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 CREATE TABLE IF NOT EXISTS `s2_collect` (

launchers/standalone/src/main/resources/db/schema-postgres.sql

@@ -465,7 +465,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela (
     from_model_id bigint,
     to_model_id bigint,
     join_type VARCHAR(255),
-    join_condition VARCHAR(255)
+    join_condition text
 );
 
 CREATE TABLE IF NOT EXISTS s2_collect (

launchers/standalone/src/test/java/com/tencent/supersonic/headless/SemanticModellerTest.java

@@ -39,14 +39,14 @@ public class SemanticModellerTest extends BaseTest {
         Map<String, ModelSchema> modelSchemaMap = modelService.buildModelSchema(modelSchemaReq);
 
         ModelSchema userModelSchema = modelSchemaMap.get("s2_user_department");
-        Assertions.assertEquals(2, userModelSchema.getColumnSchemas().size());
+        Assertions.assertEquals(2, userModelSchema.getSemanticColumns().size());
         Assertions.assertEquals(FieldType.primary_key,
                 userModelSchema.getColumnByName("user_name").getFiledType());
         Assertions.assertEquals(FieldType.categorical,
                 userModelSchema.getColumnByName("department").getFiledType());
 
         ModelSchema stayTimeModelSchema = modelSchemaMap.get("s2_stay_time_statis");
-        Assertions.assertEquals(4, stayTimeModelSchema.getColumnSchemas().size());
+        Assertions.assertEquals(4, stayTimeModelSchema.getSemanticColumns().size());
         Assertions.assertEquals(FieldType.foreign_key,
                 stayTimeModelSchema.getColumnByName("user_name").getFiledType());
         Assertions.assertEquals(FieldType.partition_time,
@@ -72,7 +72,7 @@ public class SemanticModellerTest extends BaseTest {
         Map<String, ModelSchema> modelSchemaMap = modelService.buildModelSchema(modelSchemaReq);
 
         ModelSchema pvModelSchema = modelSchemaMap.values().iterator().next();
-        Assertions.assertEquals(5, pvModelSchema.getColumnSchemas().size());
+        Assertions.assertEquals(5, pvModelSchema.getSemanticColumns().size());
         Assertions.assertEquals(FieldType.partition_time,
                 pvModelSchema.getColumnByName("imp_date").getFiledType());
         Assertions.assertEquals(FieldType.categorical,

pom.xml

@@ -26,13 +26,13 @@
     </parent>
 
     <properties>
-        <revision>0.9.10</revision>
+        <revision>1.0.0-SNAPSHOT</revision>
         <java.source.version>21</java.source.version>
         <java.target.version>21</java.target.version>
         <maven.compiler.source>21</maven.compiler.source>
         <maven.compiler.target>21</maven.compiler.target>
         <file.encoding>UTF-8</file.encoding>
-        <jsqlparser.version>4.7</jsqlparser.version>
+        <jsqlparser.version>4.9</jsqlparser.version>
         <pagehelper.version>6.1.0</pagehelper.version>
         <pagehelper.spring.version>2.1.0</pagehelper.spring.version>
         <mybatis.version>3.5.3</mybatis.version>