diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java index 79cefa92f..c6ba9ec5c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/MultiCustomDictionary.java @@ -33,6 +33,7 @@ import java.util.Objects; import java.util.PriorityQueue; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.PriorityBlockingQueue; import static com.hankcs.hanlp.utility.Predefine.logger; @@ -40,7 +41,7 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { public static int MAX_SIZE = 10; public static Boolean removeDuplicates = true; - public static ConcurrentHashMap> NATURE_TO_VALUES = + public static ConcurrentHashMap> NATURE_TO_VALUES = new ConcurrentHashMap<>(); private static boolean addToSuggesterTrie = true; @@ -146,9 +147,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { } for (int i = 0; i < attribute.nature.length; i++) { Nature nature = attribute.nature[i]; - PriorityQueue priorityQueue = NATURE_TO_VALUES.get(nature.toString()); + PriorityBlockingQueue priorityQueue = + NATURE_TO_VALUES.get(nature.toString()); if (Objects.isNull(priorityQueue)) { - priorityQueue = new PriorityQueue<>(MAX_SIZE, + priorityQueue = new PriorityBlockingQueue<>(MAX_SIZE, Comparator.comparingInt(Term::getFrequency).reversed()); NATURE_TO_VALUES.put(nature.toString(), priorityQueue); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java index 40a9504b2..8c14ffa53 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/SearchService.java @@ -24,6 +24,7 @@ import java.util.PriorityQueue; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.PriorityBlockingQueue; import java.util.stream.Collectors; @Slf4j @@ -200,7 +201,7 @@ public class SearchService { public static List getDimensionValue(DimensionValueReq dimensionValueReq) { String nature = DictWordType.NATURE_SPILT + dimensionValueReq.getModelId() + DictWordType.NATURE_SPILT + dimensionValueReq.getElementID(); - PriorityQueue terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature); + PriorityBlockingQueue terms = MultiCustomDictionary.NATURE_TO_VALUES.get(nature); if (CollectionUtils.isEmpty(terms)) { return new ArrayList<>(); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index f2dbc16f5..a4ab0cfc5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -422,33 +422,25 @@ public class DictUtils { return joiner.toString(); } - public String defaultDateFilter(DateConf dateConf) { - String format = itemValueDateFormat; - String start = LocalDate.now().minusDays(itemValueDateStart) - .format(DateTimeFormatter.ofPattern(format)); - String end = LocalDate.now().minusDays(itemValueDateEnd) - .format(DateTimeFormatter.ofPattern(format)); - if (Objects.nonNull(dateConf)) { - return String.format("( %s >= '%s' and %s <= '%s' )", dateConf.getDateField(), start, - dateConf.getDateField(), end); - } else { - return String.format("( %s >= '%s' and %s <= '%s' )", "dt", start, "dt", end); - } - } - private String generateDictDateFilter(DictItemResp dictItemResp) { - ItemValueConfig config = dictItemResp.getConfig(); - if (config == null) { + Dimension partitionTimeDimension = getPartitionTimeDimension(dictItemResp.getModelId()); + // 如果没有设置数据时间维度,则无法做时间分区过滤 + if (partitionTimeDimension == null) { return ""; } - if (!partitionedModel(dictItemResp.getModelId())) { - return ""; - } - // 未进行设置 + ItemValueConfig config = dictItemResp.getConfig(); + // 默认使用数据时间维度进行时间分区过滤 if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) { - return defaultDateFilter(null); + String startDate = LocalDate.now().minusDays(itemValueDateStart) + .format(DateTimeFormatter.ofPattern(partitionTimeDimension.getDateFormat())); + String endDate = LocalDate.now().minusDays(itemValueDateEnd) + .format(DateTimeFormatter.ofPattern(partitionTimeDimension.getDateFormat())); + return String.format("( %s >= '%s' and %s <= '%s' )", + partitionTimeDimension.getBizName(), startDate, + partitionTimeDimension.getBizName(), endDate); } + // 全表扫描 if (DateConf.DateMode.ALL.equals(config.getDateConf().getDateMode())) { return ""; @@ -467,15 +459,15 @@ public class DictUtils { return ""; } - private boolean partitionedModel(Long modelId) { + private Dimension getPartitionTimeDimension(Long modelId) { ModelResp model = modelService.getModel(modelId); if (Objects.nonNull(model)) { List timeDims = model.getTimeDimension(); if (!CollectionUtils.isEmpty(timeDims)) { - return true; + return timeDims.get(0); } } - return false; + return null; } private String generateDictDateFilterRecent(DictItemResp dictItemResp) {