topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Opt knowledge data (#1769)

Browse Source
This commit is contained in:
daikon
2024-10-10 14:29:05 +08:00
committed by GitHub
parent febcb01cd3
commit fe3f4c36b5
2 changed files with 23 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/knowledge/file/FileHandlerImpl.java
View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -26,6 +27,10 @@ import java.util.stream.Stream;
@Slf4j
@Component
public class FileHandlerImpl implements FileHandler {
@Value("${dict.value.max.count.page:1000}")
private int dictValueMaxCountPage;
public static final String FILE_SPILT = File.separator;
private final LocalFileConfig localFileConfig;
@@ -92,10 +97,9 @@ public class FileHandlerImpl implements FileHandler {
Long fileLineNum = getFileLineNum(filePath);
Integer startLine = 1;
List<DictValueResp> dictValueRespList =
getFileData(filePath, startLine, fileLineNum.intValue());
dictValueRespList = dictValueRespList.stream()
.filter(dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue()))
.collect(Collectors.toList());
getFileData(filePath, startLine, fileLineNum.intValue()).stream().filter(
dictValue -> dictValue.getValue().contains(dictValueReq.getKeyValue()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(dictValueRespList)) {
dictValueRespPageInfo.setList(new ArrayList<>());
return dictValueRespPageInfo;
@@ -120,7 +124,7 @@ public class FileHandlerImpl implements FileHandler {
DictValueReq dictValueReq) {
PageInfo<DictValueResp> dictValueRespPageInfo = new PageInfo<>();
String filePath = localFileConfig.getDictDirectoryLatest() + FILE_SPILT + fileName;
Long fileLineNum = getFileLineNum(filePath);
Long fileLineNum = Math.min(dictValueMaxCountPage, getFileLineNum(filePath));
Integer startLine = 1;
Integer endLine = Integer.valueOf(
Math.min(dictValueReq.getCurrent() * dictValueReq.getPageSize(), fileLineNum) + "");

14
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
View File

@@ -467,6 +467,9 @@ public class DictUtils {
private String generateDictDateFilter(DictItemResp dictItemResp) {
ItemValueConfig config = dictItemResp.getConfig();
if (!partitionedModel(dictItemResp.getModelId())) {
return "";
}
// 未进行设置
if (Objects.isNull(config) || Objects.isNull(config.getDateConf())) {
return defaultDateFilter();
@@ -489,6 +492,17 @@ public class DictUtils {
return "";
}
private boolean partitionedModel(Long modelId) {
ModelResp model = modelService.getModel(modelId);
if (Objects.nonNull(model)) {
List<Dim> timeDims = model.getTimeDimension();
if (!CollectionUtils.isEmpty(timeDims)) {
return true;
}
}
return false;
}
private String generateDictDateFilterRecent(DictItemResp dictItemResp) {
ModelResp model = modelService.getModel(dictItemResp.getModelId());
if (Objects.nonNull(model)) {