From 0e6050e8ce86cc7351d2fa4387559c84015e05a2 Mon Sep 17 00:00:00 2001 From: beat4ocean Date: Wed, 5 Mar 2025 16:30:05 +0800 Subject: [PATCH] [fix][headless] Fix the issue where filterSql is not working. --- .../headless/api/pojo/ModelDetail.java | 2 ++ .../api/pojo/request/ModelBuildReq.java | 2 ++ .../parser/calcite/DataModelNode.java | 34 +++++++++++++------ .../server/manager/ModelYamlManager.java | 1 + .../server/manager/SemanticSchemaManager.java | 1 + .../server/pojo/yaml/DataModelYamlTpl.java | 2 ++ .../headless/server/utils/ModelConverter.java | 1 + 7 files changed, 32 insertions(+), 11 deletions(-) diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index 26a334e27..6538ac3a0 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -24,6 +24,8 @@ public class ModelDetail { private String tableQuery; + private String filterSql; + private List identifiers = Lists.newArrayList(); private List dimensions = Lists.newArrayList(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java index d0d0b48cb..d56cd66b2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java @@ -19,6 +19,8 @@ public class ModelBuildReq { private String sql; + private String filterSql; + private String catalog; private String db; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java index 720a9b1dc..945520e4f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java @@ -40,11 +40,23 @@ public class DataModelNode extends SemanticNode { .equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { String fullTableName = String.join(".public.", dataModel.getModelDetail().getTableQuery().split("\\.")); - sqlTable = "select * from " + fullTableName; + sqlTable = "SELECT * FROM " + fullTableName; } else { - sqlTable = "select * from " + dataModel.getModelDetail().getTableQuery(); + sqlTable = "SELECT * FROM " + dataModel.getModelDetail().getTableQuery(); } } + + // String filterSql = dataModel.getFilterSql(); + String filterSql = dataModel.getModelDetail().getFilterSql(); + if (filterSql != null && !filterSql.isEmpty()) { + boolean sqlContainWhere = sqlTable.toUpperCase().matches("(?s).*\\bWHERE\\b.*"); + if (sqlContainWhere) { + sqlTable = String.format("%s AND %s", sqlTable, filterSql); + } else { + sqlTable = String.format("%s WHERE %s", sqlTable, filterSql); + } + } + if (sqlTable.isEmpty()) { throw new Exception("DataModelNode build error [tableSqlNode not found]"); } @@ -69,7 +81,7 @@ public class DataModelNode extends SemanticNode { } private static void addSchemaTable(SqlValidatorScope scope, ModelResp dataModel, String db, - String tb, Set fields) throws Exception { + String tb, Set fields) throws Exception { Set dateInfo = new HashSet<>(); Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); @@ -106,7 +118,7 @@ public class DataModelNode extends SemanticNode { } public static List getExtendField(Map exprList, - SqlValidatorScope scope, EngineType engineType) throws Exception { + SqlValidatorScope scope, EngineType engineType) throws Exception { List sqlNodeList = new ArrayList<>(); for (String expr : exprList.keySet()) { sqlNodeList.add(parse(expr, scope, engineType)); @@ -127,7 +139,7 @@ public class DataModelNode extends SemanticNode { } public static List getQueryDataModels(Ontology ontology, - OntologyQuery ontologyQuery) { + OntologyQuery ontologyQuery) { // get query measures and dimensions Set queryMeasures = new HashSet<>(); Set queryDimensions = new HashSet<>(); @@ -162,7 +174,7 @@ public class DataModelNode extends SemanticNode { } public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery ontologyQuery, - Set queryDimensions, Set queryMeasures) { + Set queryDimensions, Set queryMeasures) { ontologyQuery.getMetrics().forEach(m -> { if (Objects.nonNull(m.getMetricDefineByMeasureParams())) { m.getMetricDefineByMeasureParams().getMeasures() @@ -215,7 +227,7 @@ public class DataModelNode extends SemanticNode { } private static boolean checkMatch(ModelResp baseDataModel, Set queryMeasures, - Set queryDimension) { + Set queryDimension) { boolean isAllMatch = true; Set baseMeasures = baseDataModel.getMeasures().stream().map(Measure::getName) .collect(Collectors.toSet()); @@ -248,8 +260,8 @@ public class DataModelNode extends SemanticNode { } private static List findRelatedModelsByRelation(Ontology ontology, - OntologyQuery ontologyQuery, ModelResp baseDataModel, Set queryDimensions, - Set queryMeasures) { + OntologyQuery ontologyQuery, ModelResp baseDataModel, Set queryDimensions, + Set queryMeasures) { Set joinDataModelNames = new HashSet<>(); List joinDataModels = new ArrayList<>(); Set before = new HashSet<>(); @@ -333,7 +345,7 @@ public class DataModelNode extends SemanticNode { } private static void sortJoinRelation(List joinRelations, String next, - Set visited, List sortedJoins) { + Set visited, List sortedJoins) { for (JoinRelation link : joinRelations) { if (!visited.contains(link.getId())) { if (link.getLeft().equals(next) || link.getRight().equals(next)) { @@ -348,7 +360,7 @@ public class DataModelNode extends SemanticNode { } private static List findRelatedModelsByIdentifier(Ontology ontology, - ModelResp baseDataModel, Set queryDimension, Set measures) { + ModelResp baseDataModel, Set queryDimension, Set measures) { Set baseIdentifiers = baseDataModel.getModelDetail().getIdentifiers().stream() .map(Identify::getName).collect(Collectors.toSet()); if (baseIdentifiers.isEmpty()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index 9a07684ca..bb0f3a6f7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -36,6 +36,7 @@ public class ModelYamlManager { } else { dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); } + dataModelYamlTpl.setFilterSql(modelDetail.getFilterSql()); dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields()); dataModelYamlTpl.setId(modelResp.getId()); return dataModelYamlTpl; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index b371da13e..9e4633714 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -97,6 +97,7 @@ public class SemanticSchemaManager { modelDetail.setDbType(d.getType()); modelDetail.setSqlQuery(d.getSqlQuery()); modelDetail.setTableQuery(d.getTableQuery()); + modelDetail.setFilterSql(d.getFilterSql()); modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers())); modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures())); modelDetail.getDimensions().addAll(getDimensions(d.getDimensions())); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java index 67813590c..75208392a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java @@ -21,6 +21,8 @@ public class DataModelYamlTpl { private String tableQuery; + private String filterSql; + private List identifiers; private List dimensions; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index b2f23294e..7c2e75382 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -157,6 +157,7 @@ public class ModelConverter { modelDetail.setQueryType(ModelDefineType.TABLE_QUERY.getName()); modelDetail.setTableQuery(String.format("%s.%s", modelBuildReq.getDb(), tableName)); } + modelDetail.setFilterSql(modelBuildReq.getFilterSql()); for (ColumnSchema columnSchema : modelSchema.getColumnSchemas()) { FieldType fieldType = columnSchema.getFiledType(); if (getIdentifyType(fieldType) != null) {