From b6d1525daa5f6472ae71abea6df6be93f679112d Mon Sep 17 00:00:00 2001 From: beat4ocean Date: Wed, 5 Mar 2025 11:03:58 +0800 Subject: [PATCH 1/4] [Fix][Chat]Fix time granularity bug in model editing and dashboard display. --- .../components/ChatMsg/MetricTrend/index.tsx | 12 +++--- .../Datasource/components/ModelFieldForm.tsx | 42 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx b/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx index 8d0b7d4f1..94843c551 100644 --- a/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx @@ -48,7 +48,7 @@ const MetricTrend: React.FC = ({ const { queryColumns, queryResults, aggregateInfo, entityInfo, chatContext } = data; const [chartType, setChartType] = useState('line'); - const dateField: any = queryColumns?.find( + const dateField = queryColumns?.find( (column: any) => column.showType === 'DATE' || column.type === 'DATE' ); const dateColumnName = dateField?.bizName || ''; @@ -80,11 +80,11 @@ const MetricTrend: React.FC = ({ )}
- + {/**/}
{ - handleFieldChange(record, 'timeGranularity', value); - }} - defaultValue={timeGranularity === '' ? undefined : DATE_OPTIONS[0]} - style={{ minWidth: 180 }} - allowClear - > - {DATE_OPTIONS.map((item) => ( - - ))} - - + {/**/} + {/* 时间粒度:*/} + {/* {*/} + {/* handleFieldChange(record, 'timeGranularity', value);*/} + {/* }}*/} + {/* defaultValue={timeGranularity === '' ? undefined : DATE_OPTIONS[0]}*/} + {/* style={{ minWidth: 180 }}*/} + {/* allowClear*/} + {/* >*/} + {/* {DATE_OPTIONS.map((item) => (*/} + {/* */} + {/* ))}*/} + {/* */} + {/**/} ); } From 738093bc886613ed88a3cfa48708c84c680f91a7 Mon Sep 17 00:00:00 2001 From: beat4ocean Date: Wed, 5 Mar 2025 13:23:15 +0800 Subject: [PATCH 2/4] [fix][chat]Fix dashboard display issue when categoryColumnName and dateColumnName are identical. --- .../chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx b/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx index 94843c551..a05c9f866 100644 --- a/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx +++ b/webapp/packages/chat-sdk/src/components/ChatMsg/MetricTrend/index.tsx @@ -52,8 +52,9 @@ const MetricTrend: React.FC = ({ (column: any) => column.showType === 'DATE' || column.type === 'DATE' ); const dateColumnName = dateField?.bizName || ''; - const categoryColumnName = - queryColumns?.find((column: any) => column.showType === 'CATEGORY')?.bizName || ''; + let categoryColumnName = + queryColumns?.find((column: any) => column.showType === 'CATEGORY')?.bizName || ''; + categoryColumnName = categoryColumnName === dateColumnName ? '' : categoryColumnName; const metricFields = queryColumns?.filter((column: any) => column.showType === 'NUMBER'); const currentMetricField = queryColumns?.find((column: any) => column.showType === 'NUMBER'); From 0e6050e8ce86cc7351d2fa4387559c84015e05a2 Mon Sep 17 00:00:00 2001 From: beat4ocean Date: Wed, 5 Mar 2025 16:30:05 +0800 Subject: [PATCH 3/4] [fix][headless] Fix the issue where filterSql is not working. --- .../headless/api/pojo/ModelDetail.java | 2 ++ .../api/pojo/request/ModelBuildReq.java | 2 ++ .../parser/calcite/DataModelNode.java | 34 +++++++++++++------ .../server/manager/ModelYamlManager.java | 1 + .../server/manager/SemanticSchemaManager.java | 1 + .../server/pojo/yaml/DataModelYamlTpl.java | 2 ++ .../headless/server/utils/ModelConverter.java | 1 + 7 files changed, 32 insertions(+), 11 deletions(-) diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index 26a334e27..6538ac3a0 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -24,6 +24,8 @@ public class ModelDetail { private String tableQuery; + private String filterSql; + private List identifiers = Lists.newArrayList(); private List dimensions = Lists.newArrayList(); diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java index d0d0b48cb..d56cd66b2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ModelBuildReq.java @@ -19,6 +19,8 @@ public class ModelBuildReq { private String sql; + private String filterSql; + private String catalog; private String db; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java index 720a9b1dc..945520e4f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java @@ -40,11 +40,23 @@ public class DataModelNode extends SemanticNode { .equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { String fullTableName = String.join(".public.", dataModel.getModelDetail().getTableQuery().split("\\.")); - sqlTable = "select * from " + fullTableName; + sqlTable = "SELECT * FROM " + fullTableName; } else { - sqlTable = "select * from " + dataModel.getModelDetail().getTableQuery(); + sqlTable = "SELECT * FROM " + dataModel.getModelDetail().getTableQuery(); } } + + // String filterSql = dataModel.getFilterSql(); + String filterSql = dataModel.getModelDetail().getFilterSql(); + if (filterSql != null && !filterSql.isEmpty()) { + boolean sqlContainWhere = sqlTable.toUpperCase().matches("(?s).*\\bWHERE\\b.*"); + if (sqlContainWhere) { + sqlTable = String.format("%s AND %s", sqlTable, filterSql); + } else { + sqlTable = String.format("%s WHERE %s", sqlTable, filterSql); + } + } + if (sqlTable.isEmpty()) { throw new Exception("DataModelNode build error [tableSqlNode not found]"); } @@ -69,7 +81,7 @@ public class DataModelNode extends SemanticNode { } private static void addSchemaTable(SqlValidatorScope scope, ModelResp dataModel, String db, - String tb, Set fields) throws Exception { + String tb, Set fields) throws Exception { Set dateInfo = new HashSet<>(); Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); @@ -106,7 +118,7 @@ public class DataModelNode extends SemanticNode { } public static List getExtendField(Map exprList, - SqlValidatorScope scope, EngineType engineType) throws Exception { + SqlValidatorScope scope, EngineType engineType) throws Exception { List sqlNodeList = new ArrayList<>(); for (String expr : exprList.keySet()) { sqlNodeList.add(parse(expr, scope, engineType)); @@ -127,7 +139,7 @@ public class DataModelNode extends SemanticNode { } public static List getQueryDataModels(Ontology ontology, - OntologyQuery ontologyQuery) { + OntologyQuery ontologyQuery) { // get query measures and dimensions Set queryMeasures = new HashSet<>(); Set queryDimensions = new HashSet<>(); @@ -162,7 +174,7 @@ public class DataModelNode extends SemanticNode { } public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery ontologyQuery, - Set queryDimensions, Set queryMeasures) { + Set queryDimensions, Set queryMeasures) { ontologyQuery.getMetrics().forEach(m -> { if (Objects.nonNull(m.getMetricDefineByMeasureParams())) { m.getMetricDefineByMeasureParams().getMeasures() @@ -215,7 +227,7 @@ public class DataModelNode extends SemanticNode { } private static boolean checkMatch(ModelResp baseDataModel, Set queryMeasures, - Set queryDimension) { + Set queryDimension) { boolean isAllMatch = true; Set baseMeasures = baseDataModel.getMeasures().stream().map(Measure::getName) .collect(Collectors.toSet()); @@ -248,8 +260,8 @@ public class DataModelNode extends SemanticNode { } private static List findRelatedModelsByRelation(Ontology ontology, - OntologyQuery ontologyQuery, ModelResp baseDataModel, Set queryDimensions, - Set queryMeasures) { + OntologyQuery ontologyQuery, ModelResp baseDataModel, Set queryDimensions, + Set queryMeasures) { Set joinDataModelNames = new HashSet<>(); List joinDataModels = new ArrayList<>(); Set before = new HashSet<>(); @@ -333,7 +345,7 @@ public class DataModelNode extends SemanticNode { } private static void sortJoinRelation(List joinRelations, String next, - Set visited, List sortedJoins) { + Set visited, List sortedJoins) { for (JoinRelation link : joinRelations) { if (!visited.contains(link.getId())) { if (link.getLeft().equals(next) || link.getRight().equals(next)) { @@ -348,7 +360,7 @@ public class DataModelNode extends SemanticNode { } private static List findRelatedModelsByIdentifier(Ontology ontology, - ModelResp baseDataModel, Set queryDimension, Set measures) { + ModelResp baseDataModel, Set queryDimension, Set measures) { Set baseIdentifiers = baseDataModel.getModelDetail().getIdentifiers().stream() .map(Identify::getName).collect(Collectors.toSet()); if (baseIdentifiers.isEmpty()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index 9a07684ca..bb0f3a6f7 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -36,6 +36,7 @@ public class ModelYamlManager { } else { dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); } + dataModelYamlTpl.setFilterSql(modelDetail.getFilterSql()); dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields()); dataModelYamlTpl.setId(modelResp.getId()); return dataModelYamlTpl; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index b371da13e..9e4633714 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -97,6 +97,7 @@ public class SemanticSchemaManager { modelDetail.setDbType(d.getType()); modelDetail.setSqlQuery(d.getSqlQuery()); modelDetail.setTableQuery(d.getTableQuery()); + modelDetail.setFilterSql(d.getFilterSql()); modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers())); modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures())); modelDetail.getDimensions().addAll(getDimensions(d.getDimensions())); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java index 67813590c..75208392a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DataModelYamlTpl.java @@ -21,6 +21,8 @@ public class DataModelYamlTpl { private String tableQuery; + private String filterSql; + private List identifiers; private List dimensions; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index b2f23294e..7c2e75382 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -157,6 +157,7 @@ public class ModelConverter { modelDetail.setQueryType(ModelDefineType.TABLE_QUERY.getName()); modelDetail.setTableQuery(String.format("%s.%s", modelBuildReq.getDb(), tableName)); } + modelDetail.setFilterSql(modelBuildReq.getFilterSql()); for (ColumnSchema columnSchema : modelSchema.getColumnSchemas()) { FieldType fieldType = columnSchema.getFiledType(); if (getIdentifyType(fieldType) != null) { From 9c10089707a70f5932739f6da5bca0cfafc51e25 Mon Sep 17 00:00:00 2001 From: beat4ocean Date: Wed, 5 Mar 2025 17:09:20 +0800 Subject: [PATCH 4/4] [improvement][headless] Current mainstream large models are not as intelligent, and setting this restriction leads to excessive query failures. --- .../translator/parser/SqlQueryParser.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index b52e9bbea..8bab43edd 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -50,19 +50,19 @@ public class SqlQueryParser implements QueryParser { queryFields.removeAll(queryAliases); Ontology ontology = queryStatement.getOntology(); OntologyQuery ontologyQuery = buildOntologyQuery(ontology, queryFields); - // check if there are fields not matched with any metric or dimension - if (queryFields.size() > ontologyQuery.getMetrics().size() - + ontologyQuery.getDimensions().size()) { - List semanticFields = Lists.newArrayList(); - ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName())); - ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName())); - String errMsg = - String.format("Querying columns[%s] not matched with semantic fields[%s].", - queryFields, semanticFields); - queryStatement.setErrMsg(errMsg); - queryStatement.setStatus(QueryState.INVALID); - return; - } + // // check if there are fields not matched with any metric or dimension + // if (queryFields.size() > ontologyQuery.getMetrics().size() + // + ontologyQuery.getDimensions().size()) { + // List semanticFields = Lists.newArrayList(); + // ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName())); + // ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName())); + // String errMsg = + // String.format("Querying columns[%s] not matched with semantic fields[%s].", + // queryFields, semanticFields); + // queryStatement.setErrMsg(errMsg); + // queryStatement.setStatus(QueryState.INVALID); + // return; + // } queryStatement.setOntologyQuery(ontologyQuery); AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), ontologyQuery.getMetrics());