[improvement][headless] Current mainstream large models are not as intelligent, and setting this restriction leads to excessive query failures.

(improvement)(headless)Optimize compatibility and robustness in ontology query translation.

common/src/main/java/com/tencent/supersonic/common/calcite/SqlDialectFactory.java

@@ -26,6 +26,16 @@ public class SqlDialectFactory {
                     .withLiteralQuoteString("'").withIdentifierQuoteString("\"")
                     .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
                     .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
+    public static final Context PRESTO_CONTEXT =
+            SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.PRESTO)
+                    .withLiteralQuoteString("'").withIdentifierQuoteString("\"")
+                    .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
+                    .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true);
+    public static final Context KYUUBI_CONTEXT =
+            SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY)
+                    .withLiteralQuoteString("'").withIdentifierQuoteString("`")
+                    .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED)
+                    .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(false);
     private static Map<EngineType, SemanticSqlDialect> sqlDialectMap;
 
     static {
@@ -35,6 +45,10 @@ public class SqlDialectFactory {
         sqlDialectMap.put(EngineType.H2, new SemanticSqlDialect(DEFAULT_CONTEXT));
         sqlDialectMap.put(EngineType.POSTGRESQL, new SemanticSqlDialect(POSTGRESQL_CONTEXT));
         sqlDialectMap.put(EngineType.HANADB, new SemanticSqlDialect(HANADB_CONTEXT));
+        sqlDialectMap.put(EngineType.STARROCKS, new SemanticSqlDialect(DEFAULT_CONTEXT));
+        sqlDialectMap.put(EngineType.KYUUBI, new SemanticSqlDialect(KYUUBI_CONTEXT));
+        sqlDialectMap.put(EngineType.PRESTO, new SemanticSqlDialect(PRESTO_CONTEXT));
+        sqlDialectMap.put(EngineType.TRINO, new SemanticSqlDialect(PRESTO_CONTEXT));
     }
 
     public static SemanticSqlDialect getSqlDialect(EngineType engineType) {

common/src/main/java/com/tencent/supersonic/common/jsqlparser/FiledFilterReplaceVisitor.java

@@ -91,7 +91,8 @@ public class FiledFilterReplaceVisitor extends ExpressionVisitorAdapter {
         }
 
         ExpressionList<?> leftFunctionParams = leftFunction.getParameters();
-        if (CollectionUtils.isEmpty(leftFunctionParams)) {
+        if (CollectionUtils.isEmpty(leftFunctionParams)
+                || !(leftFunctionParams.get(0) instanceof Column)) {
             return result;
         }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java

@@ -24,6 +24,8 @@ public class ModelDetail {
 
     private String tableQuery;
 
+    private String filterSql;
+
     private List<Identify> identifiers = Lists.newArrayList();
 
     private List<Dimension> dimensions = Lists.newArrayList();

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java

@@ -19,6 +19,8 @@ public class ModelBuildReq {
 
     private String sql;
 
+    private String filterSql;
+
     private String catalog;
 
     private String db;

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java

@@ -50,19 +50,19 @@ public class SqlQueryParser implements QueryParser {
         queryFields.removeAll(queryAliases);
         Ontology ontology = queryStatement.getOntology();
         OntologyQuery ontologyQuery = buildOntologyQuery(ontology, queryFields);
-        // check if there are fields not matched with any metric or dimension
+        // // check if there are fields not matched with any metric or dimension
-        if (queryFields.size() > ontologyQuery.getMetrics().size()
+        // if (queryFields.size() > ontologyQuery.getMetrics().size()
-                + ontologyQuery.getDimensions().size()) {
+        //         + ontologyQuery.getDimensions().size()) {
-            List<String> semanticFields = Lists.newArrayList();
+        //     List<String> semanticFields = Lists.newArrayList();
-            ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName()));
+        //     ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName()));
-            ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName()));
+        //     ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName()));
-            String errMsg =
+        //     String errMsg =
-                    String.format("Querying columns[%s] not matched with semantic fields[%s].",
+        //             String.format("Querying columns[%s] not matched with semantic fields[%s].",
-                            queryFields, semanticFields);
+        //                     queryFields, semanticFields);
-            queryStatement.setErrMsg(errMsg);
+        //     queryStatement.setErrMsg(errMsg);
-            queryStatement.setStatus(QueryState.INVALID);
+        //     queryStatement.setStatus(QueryState.INVALID);
-            return;
+        //     return;
-        }
+        // }
         queryStatement.setOntologyQuery(ontologyQuery);
 
         AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), ontologyQuery.getMetrics());

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java

@@ -40,11 +40,23 @@ public class DataModelNode extends SemanticNode {
                     .equalsIgnoreCase(EngineType.POSTGRESQL.getName())) {
                 String fullTableName = String.join(".public.",
                         dataModel.getModelDetail().getTableQuery().split("\\."));
-                sqlTable = "select * from " + fullTableName;
+                sqlTable = "SELECT * FROM " + fullTableName;
             } else {
-                sqlTable = "select * from " + dataModel.getModelDetail().getTableQuery();
+                sqlTable = "SELECT * FROM " + dataModel.getModelDetail().getTableQuery();
             }
         }
+
+        // String filterSql = dataModel.getFilterSql();
+        String filterSql = dataModel.getModelDetail().getFilterSql();
+        if (filterSql != null && !filterSql.isEmpty()) {
+            boolean sqlContainWhere = sqlTable.toUpperCase().matches("(?s).*\\bWHERE\\b.*");
+            if (sqlContainWhere) {
+                sqlTable = String.format("%s AND %s", sqlTable, filterSql);
+            } else {
+                sqlTable = String.format("%s WHERE %s", sqlTable, filterSql);
+            }
+        }
+
         if (sqlTable.isEmpty()) {
             throw new Exception("DataModelNode build error [tableSqlNode not found]");
         }
@@ -69,7 +81,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static void addSchemaTable(SqlValidatorScope scope, ModelResp dataModel, String db,
-            String tb, Set<String> fields) throws Exception {
+                                       String tb, Set<String> fields) throws Exception {
         Set<String> dateInfo = new HashSet<>();
         Set<String> dimensions = new HashSet<>();
         Set<String> metrics = new HashSet<>();
@@ -106,7 +118,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     public static List<SqlNode> getExtendField(Map<String, String> exprList,
-            SqlValidatorScope scope, EngineType engineType) throws Exception {
+                                               SqlValidatorScope scope, EngineType engineType) throws Exception {
         List<SqlNode> sqlNodeList = new ArrayList<>();
         for (String expr : exprList.keySet()) {
             sqlNodeList.add(parse(expr, scope, engineType));
@@ -127,7 +139,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     public static List<ModelResp> getQueryDataModels(Ontology ontology,
-            OntologyQuery ontologyQuery) {
+                                                     OntologyQuery ontologyQuery) {
         // get query measures and dimensions
         Set<String> queryMeasures = new HashSet<>();
         Set<String> queryDimensions = new HashSet<>();
@@ -162,7 +174,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery ontologyQuery,
-            Set<String> queryDimensions, Set<String> queryMeasures) {
+                                                Set<String> queryDimensions, Set<String> queryMeasures) {
         ontologyQuery.getMetrics().forEach(m -> {
             if (Objects.nonNull(m.getMetricDefineByMeasureParams())) {
                 m.getMetricDefineByMeasureParams().getMeasures()
@@ -215,7 +227,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static boolean checkMatch(ModelResp baseDataModel, Set<String> queryMeasures,
-            Set<String> queryDimension) {
+                                      Set<String> queryDimension) {
         boolean isAllMatch = true;
         Set<String> baseMeasures = baseDataModel.getMeasures().stream().map(Measure::getName)
                 .collect(Collectors.toSet());
@@ -248,8 +260,8 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static List<ModelResp> findRelatedModelsByRelation(Ontology ontology,
-            OntologyQuery ontologyQuery, ModelResp baseDataModel, Set<String> queryDimensions,
+                                                               OntologyQuery ontologyQuery, ModelResp baseDataModel, Set<String> queryDimensions,
-            Set<String> queryMeasures) {
+                                                               Set<String> queryMeasures) {
         Set<String> joinDataModelNames = new HashSet<>();
         List<ModelResp> joinDataModels = new ArrayList<>();
         Set<String> before = new HashSet<>();
@@ -333,7 +345,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static void sortJoinRelation(List<JoinRelation> joinRelations, String next,
-            Set<Long> visited, List<JoinRelation> sortedJoins) {
+                                         Set<Long> visited, List<JoinRelation> sortedJoins) {
         for (JoinRelation link : joinRelations) {
             if (!visited.contains(link.getId())) {
                 if (link.getLeft().equals(next) || link.getRight().equals(next)) {
@@ -348,7 +360,7 @@ public class DataModelNode extends SemanticNode {
     }
 
     private static List<ModelResp> findRelatedModelsByIdentifier(Ontology ontology,
-            ModelResp baseDataModel, Set<String> queryDimension, Set<String> measures) {
+                                                                 ModelResp baseDataModel, Set<String> queryDimension, Set<String> measures) {
         Set<String> baseIdentifiers = baseDataModel.getModelDetail().getIdentifiers().stream()
                 .map(Identify::getName).collect(Collectors.toSet());
         if (baseIdentifiers.isEmpty()) {

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SemanticNode.java

@@ -78,7 +78,8 @@ public abstract class SemanticNode {
                 scope.getValidator().getCatalogReader().getRootSchema(), engineType);
         if (Configuration.getSqlAdvisor(sqlValidatorWithHints, engineType).getReservedAndKeyWords()
                 .contains(expression.toUpperCase())) {
-            if (engineType == EngineType.HANADB) {
+            if (engineType == EngineType.HANADB || engineType == EngineType.PRESTO
+                    || engineType == EngineType.TRINO) {
                 expression = String.format("\"%s\"", expression);
             } else {
                 expression = String.format("`%s`", expression);
@@ -166,9 +167,9 @@ public abstract class SemanticNode {
         if (sqlNode instanceof SqlBasicCall) {
             SqlBasicCall sqlBasicCall = (SqlBasicCall) sqlNode;
             if (sqlBasicCall.getOperator().getKind().equals(SqlKind.AS)) {
-                if (sqlBasicCall.getOperandList().get(0) instanceof SqlSelect) {
+                SqlNode innerQuery = sqlBasicCall.getOperandList().get(0);
-                    SqlSelect table = (SqlSelect) sqlBasicCall.getOperandList().get(0);
+                if (innerQuery instanceof SqlCall) {
-                    return table;
+                    return innerQuery;
                 }
             }
         }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java

@@ -69,7 +69,13 @@ public class SqlBuilder {
         SqlNode parserNode = tableView.build();
         DatabaseResp database = queryStatement.getOntology().getDatabase();
         EngineType engineType = EngineType.fromString(database.getType());
-        parserNode = optimizeParseNode(parserNode, engineType);
+        try {
+            parserNode = optimizeParseNode(parserNode, engineType);
+        } catch (Exception e) {
+            // failure in optimization phase doesn't affect the query result,
+            // just ignore it
+            log.error("optimizeParseNode error", e);
+        }
         return SemanticNode.getSql(parserNode, engineType);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java

@@ -36,6 +36,7 @@ public class ModelYamlManager {
         } else {
             dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
         }
+        dataModelYamlTpl.setFilterSql(modelDetail.getFilterSql());
         dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
         dataModelYamlTpl.setId(modelResp.getId());
         return dataModelYamlTpl;

headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java

@@ -97,6 +97,7 @@ public class SemanticSchemaManager {
         modelDetail.setDbType(d.getType());
         modelDetail.setSqlQuery(d.getSqlQuery());
         modelDetail.setTableQuery(d.getTableQuery());
+        modelDetail.setFilterSql(d.getFilterSql());
         modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers()));
         modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures()));
         modelDetail.getDimensions().addAll(getDimensions(d.getDimensions()));

headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java

@@ -21,6 +21,8 @@ public class DataModelYamlTpl {
 
     private String tableQuery;
 
+    private String filterSql;
+
     private List<IdentifyYamlTpl> identifiers;
 
     private List<DimensionYamlTpl> dimensions;

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java

@@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.QueryColumn;
 import com.tencent.supersonic.common.pojo.User;
+import com.tencent.supersonic.common.pojo.enums.AuthType;
 import com.tencent.supersonic.common.pojo.enums.EngineType;
 import com.tencent.supersonic.headless.api.pojo.DBColumn;
 import com.tencent.supersonic.headless.api.pojo.enums.DataType;
@@ -79,8 +80,9 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
 
     @Override
     public List<DatabaseResp> getDatabaseList(User user) {
-        List<DatabaseResp> databaseResps =
+        List<DatabaseResp> databaseResps = list().stream().map(DatabaseConverter::convert)
-                list().stream().map(DatabaseConverter::convert).collect(Collectors.toList());
+                .filter(database -> filterByAuth(database, user, AuthType.VIEWER))
+                .collect(Collectors.toList());
         fillPermission(databaseResps, user);
         return databaseResps;
     }
@@ -100,6 +102,43 @@ public class DatabaseServiceImpl extends ServiceImpl<DatabaseDOMapper, DatabaseD
         });
     }
 
+    private boolean filterByAuth(DatabaseResp database, User user, AuthType authType) {
+        if (user.isSuperAdmin() || user.getName().equals(database.getCreatedBy())) {
+            return true;
+        }
+        authType = authType == null ? AuthType.VIEWER : authType;
+        switch (authType) {
+            case ADMIN:
+                return checkAdminPermission(user, database);
+            case VIEWER:
+            default:
+                return checkViewPermission(user, database);
+        }
+    }
+
+    private boolean checkAdminPermission(User user, DatabaseResp database) {
+        List<String> admins = database.getAdmins();
+        if (user.isSuperAdmin()) {
+            return true;
+        }
+        if (admins.contains(user.getName()) || database.getCreatedBy().equals(user.getName())) {
+            return true;
+        }
+        return false;
+    }
+
+    private boolean checkViewPermission(User user, DatabaseResp database) {
+        if (checkAdminPermission(user, database)) {
+            return true;
+        }
+        List<String> viewers = database.getViewers();
+
+        if (viewers.contains(user.getName())) {
+            return true;
+        }
+        return false;
+    }
+
     @Override
     public void deleteDatabase(Long databaseId) {
         ModelFilter modelFilter = new ModelFilter();

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -157,6 +157,7 @@ public class ModelConverter {
             modelDetail.setQueryType(ModelDefineType.TABLE_QUERY.getName());
             modelDetail.setTableQuery(String.format("%s.%s", modelBuildReq.getDb(), tableName));
         }
+        modelDetail.setFilterSql(modelBuildReq.getFilterSql());
         for (ColumnSchema columnSchema : modelSchema.getColumnSchemas()) {
             FieldType fieldType = columnSchema.getFiledType();
             if (getIdentifyType(fieldType) != null) {