From 3dd53bad89181f385570e762d6b54dcfb949ced9 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Mon, 22 Sep 2025 15:59:35 +0800 Subject: [PATCH 1/2] (fix)(headless)Fix concurrent modification exception issue. --- .../headless/chat/knowledge/MultiCustomDictionary.java | 8 +++++--- .../supersonic/headless/chat/knowledge/SearchService.java | 3 ++- .../supersonic/headless/chat/parser/llm/PromptHelper.java | 3 ++- .../headless/core/translator/parser/SqlQueryParser.java | 3 ++- .../server/service/impl/DimensionServiceImpl.java | 8 +++----- .../headless/server/service/impl/MetricServiceImpl.java | 6 ++---- .../headless/server/utils/DataSetSchemaBuilder.java | 8 +++++--- 7 files changed, 21 insertions(+), 18 deletions(-) diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java index 79cefa92f..c6ba9ec5c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java @@ -33,6 +33,7 @@ import java.util.Objects; import java.util.PriorityQueue; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.PriorityBlockingQueue; import static com.hankcs.hanlp.utility.Predefine.logger; @@ -40,7 +41,7 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { public static int MAX_SIZE = 10; public static Boolean removeDuplicates = true; - public static ConcurrentHashMap> NATURE_TO_VALUES = + public static ConcurrentHashMap> NATURE_TO_VALUES = new ConcurrentHashMap<>(); private static boolean addToSuggesterTrie = true; @@ -146,9 +147,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { } for (int i = 0; i < attribute.nature.length; i++) { Nature nature = attribute.nature[i]; - PriorityQueue priorityQueue = NATURE_TO_VALUES.get(nature.toString()); + PriorityBlockingQueue priorityQueue = + NATURE_TO_VALUES.get(nature.toString()); if (Objects.isNull(priorityQueue)) { - priorityQueue = new PriorityQueue<>(MAX_SIZE, + priorityQueue = new PriorityBlockingQueue<>(MAX_SIZE, Comparator.comparingInt(Term::getFrequency).reversed()); NATURE_TO_VALUES.put(nature.toString(), priorityQueue); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java index 40a9504b2..8c14ffa53 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java @@ -24,6 +24,7 @@ import java.util.PriorityQueue; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.PriorityBlockingQueue; import java.util.stream.Collectors; @Slf4j @@ -200,7 +201,7 @@ public class SearchService { public static List getDimensionValue(DimensionValueReq dimensionValueReq) { String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT + dimensionValueReq.getElementID(); - PriorityQueue terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature); + PriorityBlockingQueue terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature); if (CollectionUtils.isEmpty(terms)) { return new ArrayList<>(); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java index 3ce5f4b80..8513d650e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java @@ -144,7 +144,8 @@ public class PromptHelper { dimensionStr.append(" ALIAS '").append(alias).append("'"); } if (Objects.nonNull(dimension.getExtInfo().get(DIMENSION_DATA_TYPE))) { - dimensionStr.append(" DATATYPE '").append(dimension.getExtInfo().get(DIMENSION_DATA_TYPE)).append("'"); + dimensionStr.append(" DATATYPE '") + .append(dimension.getExtInfo().get(DIMENSION_DATA_TYPE)).append("'"); } if (StringUtils.isNotEmpty(dimension.getTimeFormat())) { dimensionStr.append(" FORMAT '").append(dimension.getTimeFormat()).append("'"); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java index 675ff84f6..b6a7206ce 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java @@ -47,7 +47,8 @@ public class SqlQueryParser implements QueryParser { SqlQuery sqlQuery = queryStatement.getSqlQuery(); List queryFields = SqlSelectHelper.getAllSelectFields(sqlQuery.getSql()); Set queryAliases = SqlSelectHelper.getAliasFields(sqlQuery.getSql()); - List> ontologyMetricsDimensionsAndBizName = Collections.synchronizedList(new ArrayList<>()); + List> ontologyMetricsDimensionsAndBizName = + Collections.synchronizedList(new ArrayList<>()); queryFields.removeAll(queryAliases); Ontology ontology = queryStatement.getOntology(); OntologyQuery ontologyQuery = buildOntologyQuery(ontology, queryFields); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java index 7f97af235..14edb7b68 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java @@ -519,7 +519,7 @@ public class DimensionServiceImpl extends ServiceImpl dimensionDOS, EventType eventType) { - List dataItems = - dimensionDOS.stream().map(this::getDataItem) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + List dataItems = dimensionDOS.stream().map(this::getDataItem) + .filter(Objects::nonNull).collect(Collectors.toList()); return new DataEvent(this, dataItems, eventType); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java index 823849fb6..51f06c83b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java @@ -682,10 +682,8 @@ public class MetricServiceImpl extends ServiceImpl } private DataEvent getDataEvent(List metricDOS, EventType eventType) { - List dataItems = - metricDOS.stream().map(this::getDataItem) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + List dataItems = metricDOS.stream().map(this::getDataItem) + .filter(Objects::nonNull).collect(Collectors.toList()); return new DataEvent(this, dataItems, eventType); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java index 6bf4b975c..952996424 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java @@ -96,8 +96,9 @@ public class DataSetSchemaBuilder { Set dimensions = new HashSet<>(); Map> dataTypeMap = Maps.newHashMap(); for (ModelResp modelResp : resp.getModelResps()) { - dataTypeMap.put(modelResp.getId(), modelResp.getModelDetail().getFields().stream() - .collect(Collectors.toMap(Field::getFieldName, Field::getDataType, (k1, k2) -> k2))); + dataTypeMap.put(modelResp.getId(), + modelResp.getModelDetail().getFields().stream().collect(Collectors + .toMap(Field::getFieldName, Field::getDataType, (k1, k2) -> k2))); } for (DimSchemaResp dim : resp.getDimensions()) { @@ -119,7 +120,8 @@ public class DataSetSchemaBuilder { dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_TYPE, dim.getType()); // data type if (dim.getDataType() != null) { - dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE, dim.getDataType()); + dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE, + dim.getDataType()); } else { dimToAdd.getExtInfo().put(DimensionConstants.DIMENSION_DATA_TYPE, dataTypeMap.get(dim.getModelId()).get(dim.getBizName())); From 353c8d8b16bfc4a11cff0ea28d5a73e5800ec7e5 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Mon, 22 Sep 2025 17:02:32 +0800 Subject: [PATCH 2/2] (opt)(headless)Fetching dimension values should use partition time field to do filtering. --- .../headless/server/utils/DictUtils.java | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index f2dbc16f5..a4ab0cfc5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -422,33 +422,25 @@ public class DictUtils { return joiner.toString(); } - public String defaultDateFilter(DateConf dateConf) { - String format = itemValueDateFormat; - String start = LocalDate.now().minusDays(itemValueDateStart) - .format(DateTimeFormatter.ofPattern(format)); - String end = LocalDate.now().minusDays(itemValueDateEnd) - .format(DateTimeFormatter.ofPattern(format)); - if (Objects.nonNull(dateConf)) { - return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start, - dateConf.getDateField(), end); - } else { - return String.format("( %s >= '%s' and %s <= '%s' )", "dt", start, "dt", end); - } - } - private String generateDictDateFilter(DictItemResp dictItemResp) { - ItemValueConfig config = dictItemResp.getConfig(); - if (config == null) { + Dimension partitionTimeDimension = getPartitionTimeDimension(dictItemResp.getModelId()); + // 如果没有设置数据时间维度，则无法做时间分区过滤 + if (partitionTimeDimension == null) { return ""; } - if (!partitionedModel(dictItemResp.getModelId())) { - return ""; - } - // 未进行设置 + ItemValueConfig config = dictItemResp.getConfig(); + // 默认使用数据时间维度进行时间分区过滤 if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) { - return defaultDateFilter(null); + String startDate = LocalDate.now().minusDays(itemValueDateStart) + .format(DateTimeFormatter.ofPattern(partitionTimeDimension.getDateFormat())); + String endDate = LocalDate.now().minusDays(itemValueDateEnd) + .format(DateTimeFormatter.ofPattern(partitionTimeDimension.getDateFormat())); + return String.format("( %s >= '%s' and %s <= '%s' )", + partitionTimeDimension.getBizName(), startDate, + partitionTimeDimension.getBizName(), endDate); } + // 全表扫描 if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) { return ""; @@ -467,15 +459,15 @@ public class DictUtils { return ""; } - private boolean partitionedModel(Long modelId) { + private Dimension getPartitionTimeDimension(Long modelId) { ModelResp model = modelService.getModel(modelId); if (Objects.nonNull(model)) { List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { - return true; + return timeDims.get(0); } } - return false; + return null; } private String generateDictDateFilterRecent(DictItemResp dictItemResp) {