mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-20 06:34:55 +00:00
[knowledge](improve) add knowledge base dimension value task manage (#88)
This commit is contained in:
daikon
@@ -3,22 +3,24 @@ package com.tencent.supersonic.chat.rest;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
|
||||
import com.tencent.supersonic.chat.service.DictionaryService;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
|
||||
import com.tencent.supersonic.chat.service.ChatKnowledgeService;
|
||||
import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
|
||||
|
||||
import java.util.List;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.validation.Valid;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PutMapping;
|
||||
|
||||
@@ -28,13 +30,14 @@ import org.springframework.web.bind.annotation.PutMapping;
|
||||
public class KnowledgeController {
|
||||
|
||||
@Autowired
|
||||
private DictionaryService dictApplicationService;
|
||||
private ChatKnowledgeService knowledgeService;
|
||||
|
||||
@Autowired
|
||||
private ApplicationStartedListener applicationStartedListener;
|
||||
|
||||
/**
|
||||
* addDictInfo
|
||||
* write specific dimension values to the knowledge base
|
||||
*
|
||||
* @param dimValue2DictCommend
|
||||
*/
|
||||
@@ -43,20 +46,21 @@ public class KnowledgeController {
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dictApplicationService.addDictTask(dimValue2DictCommend, user);
|
||||
return knowledgeService.addDictTask(dimValue2DictCommend, user);
|
||||
}
|
||||
|
||||
/**
|
||||
* deleteDictInfo
|
||||
* remove specific dimension values from the knowledge base
|
||||
*
|
||||
* @param dimValue2DictCommend
|
||||
*/
|
||||
@DeleteMapping("/task")
|
||||
@PostMapping("/task/delete")
|
||||
public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dictApplicationService.deleteDictTask(dimValue2DictCommend, user);
|
||||
return knowledgeService.deleteDictTask(dimValue2DictCommend, user);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -65,19 +69,44 @@ public class KnowledgeController {
|
||||
* @param filter
|
||||
*/
|
||||
@PostMapping("/task/search")
|
||||
public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilter filter,
|
||||
public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilterReq filter,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return dictApplicationService.searchDictTaskList(filter, user);
|
||||
return knowledgeService.searchDictTaskList(filter, user);
|
||||
}
|
||||
|
||||
/**
|
||||
* searchDictLatestTaskList
|
||||
*/
|
||||
@PostMapping("/task/search/latest")
|
||||
public List<DictLatestTaskResp> searchDictLatestTaskList(@RequestBody @Valid DictLatestTaskReq filter,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
User user = UserHolder.findUser(request, response);
|
||||
return knowledgeService.searchDictLatestTaskList(filter, user);
|
||||
}
|
||||
|
||||
/**
|
||||
* getDictRootPath
|
||||
* get knowledge base file directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
@GetMapping("/rootPath")
|
||||
public String getDictRootPath(HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
return dictApplicationService.getDictRootPath();
|
||||
return knowledgeService.getDictRootPath();
|
||||
}
|
||||
|
||||
/**
|
||||
* updateDimValue
|
||||
* update in-memory dictionary files in real time
|
||||
*
|
||||
* @param request
|
||||
* @param response
|
||||
* @return
|
||||
*/
|
||||
@PutMapping("/knowledge/dimValue")
|
||||
public Boolean updateDimValue(HttpServletRequest request,
|
||||
HttpServletResponse response) {
|
||||
|
||||
@@ -1,21 +1,25 @@
|
||||
package com.tencent.supersonic.chat.service;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface DictionaryService {
|
||||
public interface ChatKnowledgeService {
|
||||
Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
|
||||
|
||||
Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
|
||||
|
||||
List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter, User user);
|
||||
List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user);
|
||||
|
||||
DictConfig getDictInfoByModelId(Long modelId);
|
||||
|
||||
String getDictRootPath();
|
||||
|
||||
List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq filter, User user);
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package com.tencent.supersonic.chat.service.impl;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
|
||||
import com.tencent.supersonic.chat.config.DefaultMetric;
|
||||
import com.tencent.supersonic.chat.config.Dim4Dict;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
|
||||
import com.tencent.supersonic.chat.service.ChatKnowledgeService;
|
||||
import com.tencent.supersonic.chat.utils.DictMetaHelper;
|
||||
import com.tencent.supersonic.chat.utils.DictQueryHelper;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.common.util.JsonUtil;
|
||||
import com.tencent.supersonic.knowledge.dictionary.FileHandler;
|
||||
import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
|
||||
import com.tencent.supersonic.knowledge.persistence.repository.DictRepository;
|
||||
|
||||
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.Map;
|
||||
import java.util.HashMap;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ChatKnowledgeServiceImpl implements ChatKnowledgeService {
|
||||
|
||||
private final DictMetaHelper metaUtils;
|
||||
private final DictQueryHelper dictQueryHelper;
|
||||
private final FileHandler fileHandler;
|
||||
private final DictRepository dictRepository;
|
||||
private final ApplicationStartedListener applicationStartedListener;
|
||||
|
||||
@Value("${dict.flush.enable:true}")
|
||||
private Boolean dictFlushEnable;
|
||||
@Value("${dict.flush.daily.enable:true}")
|
||||
private Boolean dictFlushDailyEnable;
|
||||
@Value("${dict.file.type:txt}")
|
||||
private String dictFileType;
|
||||
private String dimValue = "DimValue_%d_%d";
|
||||
|
||||
public ChatKnowledgeServiceImpl(DictMetaHelper metaUtils,
|
||||
DictQueryHelper dictQueryHelper,
|
||||
FileHandler fileHandler,
|
||||
DictRepository dictRepository,
|
||||
ApplicationStartedListener applicationStartedListener) {
|
||||
this.metaUtils = metaUtils;
|
||||
this.dictQueryHelper = dictQueryHelper;
|
||||
this.fileHandler = fileHandler;
|
||||
this.dictRepository = dictRepository;
|
||||
this.applicationStartedListener = applicationStartedListener;
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}")
|
||||
public Boolean dailyDictTask() {
|
||||
log.info("[dailyDictTask] start");
|
||||
if (!dictFlushDailyEnable) {
|
||||
log.info("dictFlushDailyEnable is false, now finish dailyDictTask");
|
||||
}
|
||||
DimValue2DictCommand dimValue2DictCommend = new DimValue2DictCommand();
|
||||
dimValue2DictCommend.setUpdateMode(DictUpdateMode.OFFLINE_FULL);
|
||||
|
||||
User user = User.getFakeUser();
|
||||
addDictTask(dimValue2DictCommend, user);
|
||||
log.info("[dailyDictTask] finish");
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
|
||||
if (!dictFlushEnable) {
|
||||
return 0L;
|
||||
}
|
||||
|
||||
if (DictUpdateMode.REALTIME_DELETE.equals(dimValue2DictCommend.getUpdateMode())) {
|
||||
return deleteDictTask(dimValue2DictCommend, user);
|
||||
}
|
||||
|
||||
DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommend, user);
|
||||
log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
|
||||
// todo check dimension can not be searched
|
||||
|
||||
dictRepository.createDimValueDictTask(dictTaskDO);
|
||||
runDictTask(dictTaskDO, user);
|
||||
|
||||
return dictTaskDO.getId();
|
||||
}
|
||||
|
||||
public Long runDictTask(DictTaskDO dictTaskDO, User user) {
|
||||
if (Objects.isNull(dictTaskDO)) {
|
||||
return -1L;
|
||||
}
|
||||
DimValue2DictCommand command = JsonUtil.toObject(dictTaskDO.getCommand(), DimValue2DictCommand.class);
|
||||
try {
|
||||
//1. construct internal dictionary requirements
|
||||
List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(command);
|
||||
Set<Long> dimIds = generateDimSet(dimValueDOList);
|
||||
dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
|
||||
dictRepository.updateDictTaskStatus(TaskStatusEnum.RUNNING.getCode(), dictTaskDO);
|
||||
log.debug("dimValueDOList:{}", dimValueDOList);
|
||||
//2. query dimension value information
|
||||
for (DimValueDO dimValueDO : dimValueDOList) {
|
||||
Long modelId = dimValueDO.getModelId();
|
||||
DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
|
||||
for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
|
||||
List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
|
||||
//3. local file changes
|
||||
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
|
||||
dim4Dict.getDimId());
|
||||
fileHandler.writeFile(data, fileName, false);
|
||||
}
|
||||
}
|
||||
applicationStartedListener.updateKnowledgeDimValue();
|
||||
log.debug("updateDictTaskStatus to SUCCESS");
|
||||
dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
|
||||
} catch (Exception e) {
|
||||
log.warn("addDictInfo exception:", e);
|
||||
dictRepository.updateDictTaskStatus(TaskStatusEnum.ERROR.getCode(), dictTaskDO);
|
||||
}
|
||||
return 1L;
|
||||
}
|
||||
|
||||
private Set<Long> generateDimSet(List<DimValueDO> dimValueDOList) {
|
||||
Set<Long> dimIds = new HashSet<>();
|
||||
if (!CollectionUtils.isEmpty(dimValueDOList)) {
|
||||
dimValueDOList.stream().forEach(dimValueDO -> {
|
||||
if (!CollectionUtils.isEmpty(dimValueDO.getDimensions())) {
|
||||
dimValueDO.getDimensions().stream().forEach(dim4Dict -> dimIds.add(dim4Dict.getDimId()));
|
||||
}
|
||||
});
|
||||
}
|
||||
return dimIds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommand, User user) {
|
||||
if (!dictFlushEnable) {
|
||||
return 0L;
|
||||
}
|
||||
if (Objects.isNull(dimValue2DictCommand) || !DictUpdateMode.REALTIME_DELETE.equals(
|
||||
dimValue2DictCommand.getUpdateMode())) {
|
||||
throw new RuntimeException("illegal parameter");
|
||||
}
|
||||
|
||||
DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommand, user);
|
||||
log.info("[deleteDictTask] dictTaskDO:{}", dictTaskDO);
|
||||
Set<Long> dimIds = generateDimSetFromCommand(dimValue2DictCommand.getModelAndDimPair());
|
||||
dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
|
||||
dictRepository.createDimValueDictTask(dictTaskDO);
|
||||
|
||||
Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommand.getModelAndDimPair();
|
||||
if (CollectionUtils.isEmpty(modelAndDimPair)) {
|
||||
return 0L;
|
||||
}
|
||||
for (Long modelId : modelAndDimPair.keySet()) {
|
||||
if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
|
||||
continue;
|
||||
}
|
||||
for (Long dimId : modelAndDimPair.get(modelId)) {
|
||||
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
|
||||
fileHandler.deleteDictFile(fileName);
|
||||
}
|
||||
}
|
||||
applicationStartedListener.updateKnowledgeDimValue();
|
||||
dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
|
||||
applicationStartedListener.updateKnowledgeDimValue();
|
||||
|
||||
return 1L;
|
||||
}
|
||||
|
||||
private Set<Long> generateDimSetFromCommand(Map<Long, List<Long>> modelAndDimPair) {
|
||||
Set<Long> dimIds = new HashSet<>();
|
||||
if (!CollectionUtils.isEmpty(modelAndDimPair)) {
|
||||
modelAndDimPair.forEach((k, v) -> dimIds.addAll(v));
|
||||
}
|
||||
return dimIds;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDictRootPath() {
|
||||
return fileHandler.getDictRootPath();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq latestFilter, User user) {
|
||||
DictTaskFilterReq filter = new DictTaskFilterReq();
|
||||
BeanUtils.copyProperties(latestFilter, filter);
|
||||
List<DimValueDictInfo> dimValueDictInfoList = searchDictTaskList(filter, user);
|
||||
return extractLatestTask(dimValueDictInfoList, latestFilter.getDimIds());
|
||||
}
|
||||
|
||||
private List<DictLatestTaskResp> extractLatestTask(List<DimValueDictInfo> dimValueDictInfoList, List<Long> dimIds) {
|
||||
List<DictLatestTaskResp> dictLatestTaskRespList = new ArrayList<>();
|
||||
Map<Long, DictLatestTaskResp> dimAndTaskPair = new HashMap<>(50);
|
||||
for (DimValueDictInfo dimValueDictInfo : dimValueDictInfoList) {
|
||||
//1. filter
|
||||
if (Objects.isNull(dimValueDictInfo) || CollectionUtils.isEmpty(dimValueDictInfo.getDimIds())) {
|
||||
continue;
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(dimIds)) {
|
||||
Set<Long> tmp = dimValueDictInfo.getDimIds();
|
||||
tmp.retainAll(dimIds);
|
||||
dimValueDictInfo.setDimIds(tmp);
|
||||
if (tmp.size() <= 0) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. extract
|
||||
Set<Long> dimIdList = dimValueDictInfo.getDimIds();
|
||||
for (Long dimId : dimIdList) {
|
||||
DictLatestTaskResp dictLatestTaskResp = new DictLatestTaskResp();
|
||||
if (!dimAndTaskPair.containsKey(dimId)) {
|
||||
BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
|
||||
dictLatestTaskResp.setDimId(dimId);
|
||||
} else {
|
||||
DictLatestTaskResp dictLatestTaskExist = dimAndTaskPair.get(dimId);
|
||||
if (dictLatestTaskExist.getCreatedAt().before(dimValueDictInfo.getCreatedAt())) {
|
||||
BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
|
||||
dictLatestTaskResp.setDimId(dimId);
|
||||
} else {
|
||||
dictLatestTaskResp = dictLatestTaskExist;
|
||||
}
|
||||
}
|
||||
dimAndTaskPair.put(dimId, dictLatestTaskResp);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (dimAndTaskPair.size() >= 0 && !CollectionUtils.isEmpty(dimAndTaskPair.values())) {
|
||||
dimAndTaskPair.values().stream()
|
||||
.filter(v -> !v.getCommand().contains(DictUpdateMode.REALTIME_DELETE.name()))
|
||||
.forEach(v -> dictLatestTaskRespList.add(v));
|
||||
}
|
||||
|
||||
|
||||
return dictLatestTaskRespList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user) {
|
||||
return dictRepository.searchDictTaskList(filter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DictConfig getDictInfoByModelId(Long modelId) {
|
||||
return dictRepository.getDictInfoByModelId(modelId);
|
||||
}
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
package com.tencent.supersonic.chat.service.impl;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.config.DefaultMetric;
|
||||
import com.tencent.supersonic.chat.config.Dim4Dict;
|
||||
import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
|
||||
import com.tencent.supersonic.chat.service.DictionaryService;
|
||||
import com.tencent.supersonic.chat.utils.DictMetaHelper;
|
||||
import com.tencent.supersonic.chat.utils.DictQueryHelper;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
|
||||
import com.tencent.supersonic.knowledge.dictionary.FileHandler;
|
||||
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
|
||||
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
|
||||
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
|
||||
import com.tencent.supersonic.knowledge.persistence.repository.DictRepository;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DictionaryServiceImpl implements DictionaryService {
|
||||
|
||||
private final DictMetaHelper metaUtils;
|
||||
private final DictQueryHelper dictQueryHelper;
|
||||
private final FileHandler fileHandler;
|
||||
private final DictRepository dictRepository;
|
||||
@Value("${dict.flush.enable:true}")
|
||||
private Boolean dictFlushEnable;
|
||||
@Value("${dict.file.type:txt}")
|
||||
private String dictFileType;
|
||||
private String dimValue = "DimValue_%d_%d";
|
||||
|
||||
public DictionaryServiceImpl(DictMetaHelper metaUtils,
|
||||
DictQueryHelper dictQueryHelper,
|
||||
FileHandler fileHandler,
|
||||
DictRepository dictRepository) {
|
||||
this.metaUtils = metaUtils;
|
||||
this.dictQueryHelper = dictQueryHelper;
|
||||
this.fileHandler = fileHandler;
|
||||
this.dictRepository = dictRepository;
|
||||
}
|
||||
|
||||
public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
|
||||
if (!dictFlushEnable) {
|
||||
return 0L;
|
||||
}
|
||||
DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskPO(dimValue2DictCommend,
|
||||
user);
|
||||
log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
|
||||
dictRepository.createDimValueDictTask(dictTaskDO);
|
||||
TaskStatusEnum finalStatus = TaskStatusEnum.SUCCESS;
|
||||
try {
|
||||
//1. construct internal dictionary requirements
|
||||
List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(dimValue2DictCommend);
|
||||
log.info("dimValueDOList:{}", dimValueDOList);
|
||||
//2. query dimension value information
|
||||
for (DimValueDO dimValueDO : dimValueDOList) {
|
||||
Long modelId = dimValueDO.getModelId();
|
||||
DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
|
||||
for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
|
||||
List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
|
||||
//3. local file changes
|
||||
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
|
||||
dim4Dict.getDimId());
|
||||
fileHandler.writeFile(data, fileName, false);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("addDictInfo exception:", e);
|
||||
finalStatus = TaskStatusEnum.ERROR;
|
||||
}
|
||||
dictRepository.updateDictTaskStatus(finalStatus.getCode(),
|
||||
dictTaskDO);
|
||||
return 1L;
|
||||
}
|
||||
|
||||
|
||||
public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
|
||||
if (!dictFlushEnable) {
|
||||
return 0L;
|
||||
}
|
||||
if (Objects.isNull(dimValue2DictCommend) || DictUpdateMode.REALTIME_DELETE.equals(
|
||||
dimValue2DictCommend.getUpdateMode())) {
|
||||
throw new RuntimeException("illegal parameter");
|
||||
}
|
||||
Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommend.getModelAndDimPair();
|
||||
if (CollectionUtils.isEmpty(modelAndDimPair)) {
|
||||
return 0L;
|
||||
}
|
||||
for (Long modelId : modelAndDimPair.keySet()) {
|
||||
if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
|
||||
continue;
|
||||
}
|
||||
for (Long dimId : modelAndDimPair.get(modelId)) {
|
||||
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
|
||||
fileHandler.deleteDictFile(fileName);
|
||||
}
|
||||
}
|
||||
|
||||
return 1L;
|
||||
}
|
||||
|
||||
public String getDictRootPath() {
|
||||
return fileHandler.getDictRootPath();
|
||||
}
|
||||
|
||||
public List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter, User user) {
|
||||
return dictRepository.searchDictTaskList(filter);
|
||||
}
|
||||
|
||||
public DictConfig getDictInfoByModelId(Long modelId) {
|
||||
return dictRepository.getDictInfoByModelId(modelId);
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,8 @@ public class DictMetaHelper {
|
||||
private ConfigService configService;
|
||||
@Value("${model.internal.metric.suffix:internal_cnt}")
|
||||
private String internalMetricNameSuffix;
|
||||
@Value("${model.internal.day.number:2}")
|
||||
private Integer internalMetricDays;
|
||||
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
|
||||
|
||||
public List<DimValueDO> generateDimValueInfo(DimValue2DictCommand dimValue2DictCommend) {
|
||||
@@ -134,14 +136,21 @@ public class DictMetaHelper {
|
||||
|
||||
ChatDefaultRichConfigResp chatDefaultConfig =
|
||||
chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
|
||||
|
||||
KnowledgeAdvancedConfig globalKnowledgeConfigAgg = chaConfigRichDesc.getChatAggRichConfig()
|
||||
.getGlobalKnowledgeConfig();
|
||||
List<KnowledgeInfoReq> knowledgeAggInfo =
|
||||
chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos();
|
||||
|
||||
KnowledgeAdvancedConfig globalKnowledgeConfigDetail = chaConfigRichDesc.getChatDetailRichConfig()
|
||||
.getGlobalKnowledgeConfig();
|
||||
List<KnowledgeInfoReq> knowledgeDetailInfo =
|
||||
chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos();
|
||||
|
||||
fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
|
||||
fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
|
||||
fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair,
|
||||
modelId, globalKnowledgeConfigDetail);
|
||||
fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair,
|
||||
modelId, globalKnowledgeConfigAgg);
|
||||
|
||||
|
||||
}
|
||||
@@ -150,7 +159,8 @@ public class DictMetaHelper {
|
||||
private void fillKnowledgeDimValue(List<KnowledgeInfoReq> knowledgeInfos,
|
||||
ChatDefaultRichConfigResp chatDefaultConfig,
|
||||
List<DimValueDO> dimValueDOList,
|
||||
Map<Long, SchemaElement> dimIdAndDescPair, Long modelId) {
|
||||
Map<Long, SchemaElement> dimIdAndDescPair, Long modelId,
|
||||
KnowledgeAdvancedConfig globalKnowledgeConfigDetail) {
|
||||
if (!CollectionUtils.isEmpty(knowledgeInfos)) {
|
||||
List<Dim4Dict> dimensions = new ArrayList<>();
|
||||
List<DefaultMetric> defaultMetricDescList = new ArrayList<>();
|
||||
@@ -159,36 +169,41 @@ public class DictMetaHelper {
|
||||
&& !CollectionUtils.isEmpty(dimIdAndDescPair)
|
||||
&& dimIdAndDescPair.containsKey(knowledgeInfo.getItemId()))
|
||||
.forEach(knowledgeInfo -> {
|
||||
if (dimIdAndDescPair.containsKey(knowledgeInfo.getItemId())) {
|
||||
SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId());
|
||||
|
||||
//default cnt
|
||||
if (Objects.isNull(chatDefaultConfig)
|
||||
|| CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
|
||||
String datasourceBizName = dimensionDesc.getBizName();
|
||||
if (Strings.isNotEmpty(datasourceBizName)) {
|
||||
String internalMetricName =
|
||||
datasourceBizName + UNDERLINE + internalMetricNameSuffix;
|
||||
defaultMetricDescList.add(new DefaultMetric(internalMetricName, 2, DAY));
|
||||
}
|
||||
} else {
|
||||
SchemaElement schemaItem = chatDefaultConfig.getMetrics().get(0);
|
||||
defaultMetricDescList.add(new DefaultMetric(schemaItem.getBizName(),
|
||||
chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod()));
|
||||
SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId());
|
||||
|
||||
//default cnt
|
||||
if (Objects.isNull(chatDefaultConfig)
|
||||
|| CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
|
||||
String datasourceBizName = dimensionDesc.getBizName();
|
||||
if (Strings.isNotEmpty(datasourceBizName)) {
|
||||
String internalMetricName =
|
||||
datasourceBizName + UNDERLINE + internalMetricNameSuffix;
|
||||
defaultMetricDescList.add(new DefaultMetric(internalMetricName,
|
||||
internalMetricDays, DAY));
|
||||
}
|
||||
} else {
|
||||
SchemaElement schemaItem = chatDefaultConfig.getMetrics().get(0);
|
||||
defaultMetricDescList.add(new DefaultMetric(schemaItem.getBizName(),
|
||||
chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod()));
|
||||
|
||||
String bizName = dimensionDesc.getBizName();
|
||||
Dim4Dict dim4Dict = new Dim4Dict();
|
||||
dim4Dict.setDimId(knowledgeInfo.getItemId());
|
||||
dim4Dict.setBizName(bizName);
|
||||
if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) {
|
||||
KnowledgeAdvancedConfig knowledgeAdvancedConfig
|
||||
= knowledgeInfo.getKnowledgeAdvancedConfig();
|
||||
BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict);
|
||||
}
|
||||
dimensions.add(dim4Dict);
|
||||
}
|
||||
|
||||
String bizName = dimensionDesc.getBizName();
|
||||
Dim4Dict dim4Dict = new Dim4Dict();
|
||||
dim4Dict.setDimId(knowledgeInfo.getItemId());
|
||||
dim4Dict.setBizName(bizName);
|
||||
if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) {
|
||||
KnowledgeAdvancedConfig knowledgeAdvancedConfig
|
||||
= knowledgeInfo.getKnowledgeAdvancedConfig();
|
||||
BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict);
|
||||
|
||||
if (Objects.nonNull(globalKnowledgeConfigDetail)
|
||||
&& !CollectionUtils.isEmpty(globalKnowledgeConfigDetail.getRuleList())) {
|
||||
dim4Dict.getRuleList().addAll(globalKnowledgeConfigDetail.getRuleList());
|
||||
}
|
||||
}
|
||||
dimensions.add(dim4Dict);
|
||||
|
||||
});
|
||||
|
||||
if (!CollectionUtils.isEmpty(dimensions)) {
|
||||
|
||||
@@ -1,10 +1,5 @@
|
||||
package com.tencent.supersonic.chat.utils;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticLayer;
|
||||
import com.tencent.supersonic.chat.config.DefaultMetric;
|
||||
@@ -34,6 +29,12 @@ import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
|
||||
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class DictQueryHelper {
|
||||
@@ -46,6 +47,8 @@ public class DictQueryHelper {
|
||||
private Integer printDataShow;
|
||||
@Value("${dimension.max.limit:3000000}")
|
||||
private Long dimMaxLimit;
|
||||
@Value("${dimension.white.weight:60000000}")
|
||||
private Long dimensionWhiteWeight;
|
||||
|
||||
public List<String> fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
|
||||
User user) {
|
||||
@@ -53,10 +56,11 @@ public class DictQueryHelper {
|
||||
QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict);
|
||||
try {
|
||||
QueryResultWithSchemaResp queryResultWithColumns = semanticLayer.queryByStruct(queryStructCmd, user);
|
||||
log.info("fetchDimValueSingle sql:{}", queryResultWithColumns.getSql());
|
||||
String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId());
|
||||
String dimNameRewrite = rewriteDimName(queryResultWithColumns.getColumns(), dim4Dict.getBizName());
|
||||
data = generateFileData(queryResultWithColumns.getResultList(), nature, dimNameRewrite,
|
||||
defaultMetricDesc.getBizName());
|
||||
defaultMetricDesc.getBizName(), dim4Dict);
|
||||
if (!CollectionUtils.isEmpty(data)) {
|
||||
int size = (data.size() > printDataShow) ? printDataShow : data.size();
|
||||
log.info("data:{}", data.subList(0, size - 1));
|
||||
@@ -91,7 +95,7 @@ public class DictQueryHelper {
|
||||
}
|
||||
|
||||
private List<String> generateFileData(List<Map<String, Object>> resultList, String nature, String dimName,
|
||||
String metricName) {
|
||||
String metricName, Dim4Dict dim4Dict) {
|
||||
List<String> data = new ArrayList<>();
|
||||
if (CollectionUtils.isEmpty(resultList)) {
|
||||
return data;
|
||||
@@ -111,17 +115,26 @@ public class DictQueryHelper {
|
||||
}
|
||||
|
||||
}
|
||||
constructDataLines(valueAndFrequencyPair, nature, data);
|
||||
constructDataLines(valueAndFrequencyPair, nature, data, dim4Dict);
|
||||
return data;
|
||||
}
|
||||
|
||||
private void constructDataLines(Map<String, Long> valueAndFrequencyPair, String nature, List<String> data) {
|
||||
private void constructDataLines(Map<String, Long> valueAndFrequencyPair, String nature,
|
||||
List<String> data, Dim4Dict dim4Dict) {
|
||||
valueAndFrequencyPair.forEach((dimValue, metric) -> {
|
||||
if (metric > MAX_FREQUENCY) {
|
||||
metric = MAX_FREQUENCY;
|
||||
}
|
||||
if (Strings.isNotEmpty(dimValue) && dimValue.contains(SPACE)) {
|
||||
dimValue = dimValue.replace(SPACE, "#");
|
||||
}
|
||||
data.add(String.format("%s %s %s", dimValue, nature, metric));
|
||||
});
|
||||
|
||||
if (Objects.nonNull(dim4Dict) && !CollectionUtils.isEmpty(dim4Dict.getWhiteList())) {
|
||||
dim4Dict.getWhiteList().stream()
|
||||
.forEach(white -> data.add(String.format("%s %s %s", white, nature, dimensionWhiteWeight)));
|
||||
}
|
||||
}
|
||||
|
||||
private void mergeMultivaluedValue(Map<String, Long> valueAndFrequencyPair, String dimValue, Long metric) {
|
||||
@@ -185,7 +198,7 @@ public class DictQueryHelper {
|
||||
if (Objects.isNull(dim4Dict)) {
|
||||
return "";
|
||||
}
|
||||
StringJoiner joiner = new StringJoiner(AND_UPPER);
|
||||
StringJoiner joiner = new StringJoiner(SPACE + AND_UPPER + SPACE);
|
||||
|
||||
String dimName = dim4Dict.getBizName();
|
||||
if (!CollectionUtils.isEmpty(dim4Dict.getBlackList())) {
|
||||
|
||||
Reference in New Issue
Block a user