From 17a3dd052c6e74168ec4b17d9a922e98ac0ee4eb Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Tue, 18 Feb 2025 12:48:26 +0800 Subject: [PATCH 1/4] (improvement)(chat)LLM might output table or column with `` enclose, should handle with it. --- .../tencent/supersonic/common/jsqlparser/SqlSelectHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java index 4f1362cca..63bbf06b2 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlSelectHelper.java @@ -294,7 +294,8 @@ public class SqlSelectHelper { } // do not account in aliases results.removeAll(aliases); - return new ArrayList<>(results); + return new ArrayList<>( + results.stream().map(r -> r.replaceAll("`", "")).collect(Collectors.toList())); } private static List getFieldsByPlainSelect(PlainSelect plainSelect) { From 9aa305ca7a2ec4e6513eef98e89458abecf0a5eb Mon Sep 17 00:00:00 2001 From: weimengdalao <1711952372@qq.com> Date: Tue, 18 Feb 2025 12:49:43 +0800 Subject: [PATCH 2/4] [fix][headless]fix Check if metricFilters is empty not dimensionFilters (#2064) --- .../supersonic/headless/chat/query/rule/RuleSemanticQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java index f7c96745f..ddb66c5ff 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/rule/RuleSemanticQuery.java @@ -241,7 +241,7 @@ public abstract class RuleSemanticQuery extends BaseSemanticQuery { .forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); } List metricFilters = queryStructReq.getMetricFilters(); - if (CollectionUtils.isNotEmpty(dimensionFilters)) { + if (CollectionUtils.isNotEmpty(metricFilters)) { metricFilters.forEach(filter -> filter.setName(bizNameToName.get(filter.getBizName()))); } } From b743585d3e9bdd6ccb70ee464bb820f074bd9fac Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Tue, 18 Feb 2025 19:07:39 +0800 Subject: [PATCH 3/4] (fix)(headless)Fix NPE issue. --- .../tencent/supersonic/headless/server/utils/DictUtils.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 838f9a085..ac90fb2de 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -431,6 +431,10 @@ public class DictUtils { private String generateDictDateFilter(DictItemResp dictItemResp) { ItemValueConfig config = dictItemResp.getConfig(); + if (config == null) { + return ""; + } + if (!partitionedModel(dictItemResp.getModelId())) { return ""; } From d1e5e8777aa1e8e29248d70ad415016a84c97428 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Tue, 18 Feb 2025 21:29:39 +0800 Subject: [PATCH 4/4] (improvement)(chat)Try to find all fields in the same model to avoid unnecessary join. --- .../translator/parser/SqlQueryParser.java | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index be950e75e..b52e9bbea 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -10,6 +10,7 @@ import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; @@ -188,12 +189,10 @@ public class SqlQueryParser implements QueryParser { String modelName = entry.getKey(); entry.getValue().forEach(m -> { if (fields.contains(m.getName()) || fields.contains(m.getBizName())) { - if (!ontologyQuery.getMetricMap().containsKey(modelName)) { - ontologyQuery.getMetricMap().put(modelName, Sets.newHashSet()); - } ontologyQuery.getModelMap().put(modelName, ontology.getModelMap().get(modelName)); - ontologyQuery.getMetricMap().get(modelName).add(m); + ontologyQuery.getMetricMap().computeIfAbsent(modelName, k -> Sets.newHashSet()) + .add(m); fields.remove(m.getName()); fields.remove(m.getBizName()); } @@ -207,33 +206,51 @@ public class SqlQueryParser implements QueryParser { String modelName = entry.getKey(); entry.getValue().forEach(d -> { if (fields.contains(d.getName()) || fields.contains(d.getBizName())) { - if (!ontologyQuery.getDimensionMap().containsKey(entry.getKey())) { - ontologyQuery.getDimensionMap().put(entry.getKey(), - Sets.newHashSet()); - } ontologyQuery.getModelMap().put(modelName, ontology.getModelMap().get(modelName)); - ontologyQuery.getDimensionMap().get(entry.getKey()).add(d); + ontologyQuery.getDimensionMap() + .computeIfAbsent(modelName, k -> Sets.newHashSet()).add(d); fields.remove(d.getName()); fields.remove(d.getBizName()); } }); }); - // if there are still fields not found belonging models, try to find in the models without - // querying metrics. + // second, try to find a model that has all the remaining fields, such that no further join + // is needed. + if (!fields.isEmpty()) { + Map> model2dims = new HashMap<>(); + ontology.getDimensionMap().entrySet().forEach(entry -> { + String modelName = entry.getKey(); + entry.getValue().forEach(d -> { + if (fields.contains(d.getName()) || fields.contains(d.getBizName())) { + model2dims.computeIfAbsent(modelName, k -> Sets.newHashSet()).add(d); + } + }); + }); + Optional>> modelEntry = model2dims.entrySet() + .stream().filter(entry -> entry.getValue().size() == fields.size()).findFirst(); + if (modelEntry.isPresent()) { + ontologyQuery.getDimensionMap().put(modelEntry.get().getKey(), + modelEntry.get().getValue()); + ontologyQuery.getModelMap().put(modelEntry.get().getKey(), + ontology.getModelMap().get(modelEntry.get().getKey())); + fields.clear(); + } + } + + // finally if there are still fields not found belonging models, try to find in the models + // iteratively if (!fields.isEmpty()) { ontology.getDimensionMap().entrySet().forEach(entry -> { String modelName = entry.getKey(); if (!ontologyQuery.getDimensionMap().containsKey(modelName)) { entry.getValue().forEach(d -> { if (fields.contains(d.getName()) || fields.contains(d.getBizName())) { - if (!ontologyQuery.getDimensionMap().containsKey(modelName)) { - ontologyQuery.getDimensionMap().put(modelName, Sets.newHashSet()); - } ontologyQuery.getModelMap().put(modelName, ontology.getModelMap().get(modelName)); - ontologyQuery.getDimensionMap().get(modelName).add(d); + ontologyQuery.getDimensionMap() + .computeIfAbsent(modelName, k -> Sets.newHashSet()).add(d); fields.remove(d.getName()); fields.remove(d.getBizName()); }