diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java new file mode 100644 index 000000000..f38167c13 --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java @@ -0,0 +1,23 @@ +package com.tencent.supersonic.chat.api.pojo.request; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import javax.validation.constraints.NotNull; +import java.util.List; + +import static java.time.LocalDate.now; + +@ToString +@Data +@NoArgsConstructor +public class DictLatestTaskReq { + + @NotNull + private Long modelId; + + private List dimIds; + + private String createdAt = now().plusDays(-4).toString(); +} \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java new file mode 100644 index 000000000..d58364ff3 --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.chat.api.pojo.request; + +import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import lombok.Data; +import lombok.ToString; + +@ToString +@Data +public class DictTaskFilterReq { + + private Long id; + + private String name; + + private String createdBy; + + private String createdAt; + + private TaskStatusEnum status; +} \ No newline at end of file diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java new file mode 100644 index 000000000..d2a2df6ab --- /dev/null +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java @@ -0,0 +1,30 @@ +package com.tencent.supersonic.chat.api.pojo.response; + +import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@ToString +@Data +public class DictLatestTaskResp { + + private Long dimId; + + private Long id; + + private String name; + + private String description; + + private String command; + + private TaskStatusEnum status; + + private String createdBy; + + private Date createdAt; + + private Long elapsedMs; +} \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java index 1cbf74fff..a46f53108 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java @@ -3,22 +3,24 @@ package com.tencent.supersonic.chat.rest; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; -import com.tencent.supersonic.chat.service.DictionaryService; +import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq; +import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp; +import com.tencent.supersonic.chat.service.ChatKnowledgeService; import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener; -import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter; +import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand; import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo; import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PutMapping; @@ -28,13 +30,14 @@ import org.springframework.web.bind.annotation.PutMapping; public class KnowledgeController { @Autowired - private DictionaryService dictApplicationService; + private ChatKnowledgeService knowledgeService; @Autowired private ApplicationStartedListener applicationStartedListener; /** * addDictInfo + * write specific dimension values to the knowledge base * * @param dimValue2DictCommend */ @@ -43,20 +46,21 @@ public class KnowledgeController { HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return dictApplicationService.addDictTask(dimValue2DictCommend, user); + return knowledgeService.addDictTask(dimValue2DictCommend, user); } /** * deleteDictInfo + * remove specific dimension values from the knowledge base * * @param dimValue2DictCommend */ - @DeleteMapping("/task") + @PostMapping("/task/delete") public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return dictApplicationService.deleteDictTask(dimValue2DictCommend, user); + return knowledgeService.deleteDictTask(dimValue2DictCommend, user); } /** @@ -65,19 +69,44 @@ public class KnowledgeController { * @param filter */ @PostMapping("/task/search") - public List searchDictTaskList(@RequestBody DictTaskFilter filter, + public List searchDictTaskList(@RequestBody DictTaskFilterReq filter, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); - return dictApplicationService.searchDictTaskList(filter, user); + return knowledgeService.searchDictTaskList(filter, user); } + /** + * searchDictLatestTaskList + */ + @PostMapping("/task/search/latest") + public List searchDictLatestTaskList(@RequestBody @Valid DictLatestTaskReq filter, + HttpServletRequest request, + HttpServletResponse response) { + User user = UserHolder.findUser(request, response); + return knowledgeService.searchDictLatestTaskList(filter, user); + } + + /** + * getDictRootPath + * get knowledge base file directory + * + * @return + */ @GetMapping("/rootPath") public String getDictRootPath(HttpServletRequest request, HttpServletResponse response) { - return dictApplicationService.getDictRootPath(); + return knowledgeService.getDictRootPath(); } + /** + * updateDimValue + * update in-memory dictionary files in real time + * + * @param request + * @param response + * @return + */ @PutMapping("/knowledge/dimValue") public Boolean updateDimValue(HttpServletRequest request, HttpServletResponse response) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/DictionaryService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatKnowledgeService.java similarity index 57% rename from chat/core/src/main/java/com/tencent/supersonic/chat/service/DictionaryService.java rename to chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatKnowledgeService.java index 786e79796..c57b1d3e9 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/DictionaryService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatKnowledgeService.java @@ -1,21 +1,25 @@ package com.tencent.supersonic.chat.service; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq; +import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp; import com.tencent.supersonic.knowledge.dictionary.DictConfig; -import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter; +import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand; import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo; import java.util.List; -public interface DictionaryService { +public interface ChatKnowledgeService { Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user); Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user); - List searchDictTaskList(DictTaskFilter filter, User user); + List searchDictTaskList(DictTaskFilterReq filter, User user); DictConfig getDictInfoByModelId(Long modelId); String getDictRootPath(); + + List searchDictLatestTaskList(DictLatestTaskReq filter, User user); } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatKnowledgeServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatKnowledgeServiceImpl.java new file mode 100644 index 000000000..eccdb5fb5 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatKnowledgeServiceImpl.java @@ -0,0 +1,268 @@ +package com.tencent.supersonic.chat.service.impl; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq; +import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp; +import com.tencent.supersonic.chat.config.DefaultMetric; +import com.tencent.supersonic.chat.config.Dim4Dict; +import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO; +import com.tencent.supersonic.chat.service.ChatKnowledgeService; +import com.tencent.supersonic.chat.utils.DictMetaHelper; +import com.tencent.supersonic.chat.utils.DictQueryHelper; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.knowledge.dictionary.FileHandler; +import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener; +import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO; +import com.tencent.supersonic.knowledge.utils.DictTaskConverter; +import com.tencent.supersonic.knowledge.dictionary.DictConfig; +import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; +import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode; +import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand; +import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo; +import com.tencent.supersonic.knowledge.persistence.repository.DictRepository; + + + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.ArrayList; +import java.util.Objects; +import java.util.Set; +import java.util.Map; +import java.util.HashMap; + +@Slf4j +@Service +public class ChatKnowledgeServiceImpl implements ChatKnowledgeService { + + private final DictMetaHelper metaUtils; + private final DictQueryHelper dictQueryHelper; + private final FileHandler fileHandler; + private final DictRepository dictRepository; + private final ApplicationStartedListener applicationStartedListener; + + @Value("${dict.flush.enable:true}") + private Boolean dictFlushEnable; + @Value("${dict.flush.daily.enable:true}") + private Boolean dictFlushDailyEnable; + @Value("${dict.file.type:txt}") + private String dictFileType; + private String dimValue = "DimValue_%d_%d"; + + public ChatKnowledgeServiceImpl(DictMetaHelper metaUtils, + DictQueryHelper dictQueryHelper, + FileHandler fileHandler, + DictRepository dictRepository, + ApplicationStartedListener applicationStartedListener) { + this.metaUtils = metaUtils; + this.dictQueryHelper = dictQueryHelper; + this.fileHandler = fileHandler; + this.dictRepository = dictRepository; + this.applicationStartedListener = applicationStartedListener; + } + + @Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}") + public Boolean dailyDictTask() { + log.info("[dailyDictTask] start"); + if (!dictFlushDailyEnable) { + log.info("dictFlushDailyEnable is false, now finish dailyDictTask"); + } + DimValue2DictCommand dimValue2DictCommend = new DimValue2DictCommand(); + dimValue2DictCommend.setUpdateMode(DictUpdateMode.OFFLINE_FULL); + + User user = User.getFakeUser(); + addDictTask(dimValue2DictCommend, user); + log.info("[dailyDictTask] finish"); + return true; + } + + @Override + public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) { + if (!dictFlushEnable) { + return 0L; + } + + if (DictUpdateMode.REALTIME_DELETE.equals(dimValue2DictCommend.getUpdateMode())) { + return deleteDictTask(dimValue2DictCommend, user); + } + + DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommend, user); + log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); + // todo check dimension can not be searched + + dictRepository.createDimValueDictTask(dictTaskDO); + runDictTask(dictTaskDO, user); + + return dictTaskDO.getId(); + } + + public Long runDictTask(DictTaskDO dictTaskDO, User user) { + if (Objects.isNull(dictTaskDO)) { + return -1L; + } + DimValue2DictCommand command = JsonUtil.toObject(dictTaskDO.getCommand(), DimValue2DictCommand.class); + try { + //1. construct internal dictionary requirements + List dimValueDOList = metaUtils.generateDimValueInfo(command); + Set dimIds = generateDimSet(dimValueDOList); + dictTaskDO.setDimIds(JsonUtil.toString(dimIds)); + dictRepository.updateDictTaskStatus(TaskStatusEnum.RUNNING.getCode(), dictTaskDO); + log.debug("dimValueDOList:{}", dimValueDOList); + //2. query dimension value information + for (DimValueDO dimValueDO : dimValueDOList) { + Long modelId = dimValueDO.getModelId(); + DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0); + for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) { + List data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user); + //3. local file changes + String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, + dim4Dict.getDimId()); + fileHandler.writeFile(data, fileName, false); + } + } + applicationStartedListener.updateKnowledgeDimValue(); + log.debug("updateDictTaskStatus to SUCCESS"); + dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO); + } catch (Exception e) { + log.warn("addDictInfo exception:", e); + dictRepository.updateDictTaskStatus(TaskStatusEnum.ERROR.getCode(), dictTaskDO); + } + return 1L; + } + + private Set generateDimSet(List dimValueDOList) { + Set dimIds = new HashSet<>(); + if (!CollectionUtils.isEmpty(dimValueDOList)) { + dimValueDOList.stream().forEach(dimValueDO -> { + if (!CollectionUtils.isEmpty(dimValueDO.getDimensions())) { + dimValueDO.getDimensions().stream().forEach(dim4Dict -> dimIds.add(dim4Dict.getDimId())); + } + }); + } + return dimIds; + } + + @Override + public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommand, User user) { + if (!dictFlushEnable) { + return 0L; + } + if (Objects.isNull(dimValue2DictCommand) || !DictUpdateMode.REALTIME_DELETE.equals( + dimValue2DictCommand.getUpdateMode())) { + throw new RuntimeException("illegal parameter"); + } + + DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommand, user); + log.info("[deleteDictTask] dictTaskDO:{}", dictTaskDO); + Set dimIds = generateDimSetFromCommand(dimValue2DictCommand.getModelAndDimPair()); + dictTaskDO.setDimIds(JsonUtil.toString(dimIds)); + dictRepository.createDimValueDictTask(dictTaskDO); + + Map> modelAndDimPair = dimValue2DictCommand.getModelAndDimPair(); + if (CollectionUtils.isEmpty(modelAndDimPair)) { + return 0L; + } + for (Long modelId : modelAndDimPair.keySet()) { + if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) { + continue; + } + for (Long dimId : modelAndDimPair.get(modelId)) { + String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId); + fileHandler.deleteDictFile(fileName); + } + } + applicationStartedListener.updateKnowledgeDimValue(); + dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO); + applicationStartedListener.updateKnowledgeDimValue(); + + return 1L; + } + + private Set generateDimSetFromCommand(Map> modelAndDimPair) { + Set dimIds = new HashSet<>(); + if (!CollectionUtils.isEmpty(modelAndDimPair)) { + modelAndDimPair.forEach((k, v) -> dimIds.addAll(v)); + } + return dimIds; + } + + @Override + public String getDictRootPath() { + return fileHandler.getDictRootPath(); + } + + @Override + public List searchDictLatestTaskList(DictLatestTaskReq latestFilter, User user) { + DictTaskFilterReq filter = new DictTaskFilterReq(); + BeanUtils.copyProperties(latestFilter, filter); + List dimValueDictInfoList = searchDictTaskList(filter, user); + return extractLatestTask(dimValueDictInfoList, latestFilter.getDimIds()); + } + + private List extractLatestTask(List dimValueDictInfoList, List dimIds) { + List dictLatestTaskRespList = new ArrayList<>(); + Map dimAndTaskPair = new HashMap<>(50); + for (DimValueDictInfo dimValueDictInfo : dimValueDictInfoList) { + //1. filter + if (Objects.isNull(dimValueDictInfo) || CollectionUtils.isEmpty(dimValueDictInfo.getDimIds())) { + continue; + } + if (!CollectionUtils.isEmpty(dimIds)) { + Set tmp = dimValueDictInfo.getDimIds(); + tmp.retainAll(dimIds); + dimValueDictInfo.setDimIds(tmp); + if (tmp.size() <= 0) { + continue; + } + } + + // 2. extract + Set dimIdList = dimValueDictInfo.getDimIds(); + for (Long dimId : dimIdList) { + DictLatestTaskResp dictLatestTaskResp = new DictLatestTaskResp(); + if (!dimAndTaskPair.containsKey(dimId)) { + BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp); + dictLatestTaskResp.setDimId(dimId); + } else { + DictLatestTaskResp dictLatestTaskExist = dimAndTaskPair.get(dimId); + if (dictLatestTaskExist.getCreatedAt().before(dimValueDictInfo.getCreatedAt())) { + BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp); + dictLatestTaskResp.setDimId(dimId); + } else { + dictLatestTaskResp = dictLatestTaskExist; + } + } + dimAndTaskPair.put(dimId, dictLatestTaskResp); + } + + } + + if (dimAndTaskPair.size() >= 0 && !CollectionUtils.isEmpty(dimAndTaskPair.values())) { + dimAndTaskPair.values().stream() + .filter(v -> !v.getCommand().contains(DictUpdateMode.REALTIME_DELETE.name())) + .forEach(v -> dictLatestTaskRespList.add(v)); + } + + + return dictLatestTaskRespList; + } + + @Override + public List searchDictTaskList(DictTaskFilterReq filter, User user) { + return dictRepository.searchDictTaskList(filter); + } + + @Override + public DictConfig getDictInfoByModelId(Long modelId) { + return dictRepository.getDictInfoByModelId(modelId); + } +} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/DictionaryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/DictionaryServiceImpl.java deleted file mode 100644 index ce11ad509..000000000 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/DictionaryServiceImpl.java +++ /dev/null @@ -1,124 +0,0 @@ -package com.tencent.supersonic.chat.service.impl; - -import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.chat.config.DefaultMetric; -import com.tencent.supersonic.chat.config.Dim4Dict; -import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO; -import com.tencent.supersonic.chat.service.DictionaryService; -import com.tencent.supersonic.chat.utils.DictMetaHelper; -import com.tencent.supersonic.chat.utils.DictQueryHelper; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; -import com.tencent.supersonic.knowledge.dictionary.FileHandler; -import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO; -import com.tencent.supersonic.knowledge.utils.DictTaskConverter; -import com.tencent.supersonic.knowledge.dictionary.DictConfig; -import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter; -import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode; -import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand; -import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo; -import com.tencent.supersonic.knowledge.persistence.repository.DictRepository; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -@Slf4j -@Service -public class DictionaryServiceImpl implements DictionaryService { - - private final DictMetaHelper metaUtils; - private final DictQueryHelper dictQueryHelper; - private final FileHandler fileHandler; - private final DictRepository dictRepository; - @Value("${dict.flush.enable:true}") - private Boolean dictFlushEnable; - @Value("${dict.file.type:txt}") - private String dictFileType; - private String dimValue = "DimValue_%d_%d"; - - public DictionaryServiceImpl(DictMetaHelper metaUtils, - DictQueryHelper dictQueryHelper, - FileHandler fileHandler, - DictRepository dictRepository) { - this.metaUtils = metaUtils; - this.dictQueryHelper = dictQueryHelper; - this.fileHandler = fileHandler; - this.dictRepository = dictRepository; - } - - public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) { - if (!dictFlushEnable) { - return 0L; - } - DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskPO(dimValue2DictCommend, - user); - log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); - dictRepository.createDimValueDictTask(dictTaskDO); - TaskStatusEnum finalStatus = TaskStatusEnum.SUCCESS; - try { - //1. construct internal dictionary requirements - List dimValueDOList = metaUtils.generateDimValueInfo(dimValue2DictCommend); - log.info("dimValueDOList:{}", dimValueDOList); - //2. query dimension value information - for (DimValueDO dimValueDO : dimValueDOList) { - Long modelId = dimValueDO.getModelId(); - DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0); - for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) { - List data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user); - //3. local file changes - String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, - dim4Dict.getDimId()); - fileHandler.writeFile(data, fileName, false); - } - } - } catch (Exception e) { - log.warn("addDictInfo exception:", e); - finalStatus = TaskStatusEnum.ERROR; - } - dictRepository.updateDictTaskStatus(finalStatus.getCode(), - dictTaskDO); - return 1L; - } - - - public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user) { - if (!dictFlushEnable) { - return 0L; - } - if (Objects.isNull(dimValue2DictCommend) || DictUpdateMode.REALTIME_DELETE.equals( - dimValue2DictCommend.getUpdateMode())) { - throw new RuntimeException("illegal parameter"); - } - Map> modelAndDimPair = dimValue2DictCommend.getModelAndDimPair(); - if (CollectionUtils.isEmpty(modelAndDimPair)) { - return 0L; - } - for (Long modelId : modelAndDimPair.keySet()) { - if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) { - continue; - } - for (Long dimId : modelAndDimPair.get(modelId)) { - String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId); - fileHandler.deleteDictFile(fileName); - } - } - - return 1L; - } - - public String getDictRootPath() { - return fileHandler.getDictRootPath(); - } - - public List searchDictTaskList(DictTaskFilter filter, User user) { - return dictRepository.searchDictTaskList(filter); - } - - public DictConfig getDictInfoByModelId(Long modelId) { - return dictRepository.getDictInfoByModelId(modelId); - } -} diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java index 87173e509..6c3436790 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java @@ -40,6 +40,8 @@ public class DictMetaHelper { private ConfigService configService; @Value("${model.internal.metric.suffix:internal_cnt}") private String internalMetricNameSuffix; + @Value("${model.internal.day.number:2}") + private Integer internalMetricDays; private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer(); public List generateDimValueInfo(DimValue2DictCommand dimValue2DictCommend) { @@ -134,14 +136,21 @@ public class DictMetaHelper { ChatDefaultRichConfigResp chatDefaultConfig = chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig(); + + KnowledgeAdvancedConfig globalKnowledgeConfigAgg = chaConfigRichDesc.getChatAggRichConfig() + .getGlobalKnowledgeConfig(); List knowledgeAggInfo = chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos(); + KnowledgeAdvancedConfig globalKnowledgeConfigDetail = chaConfigRichDesc.getChatDetailRichConfig() + .getGlobalKnowledgeConfig(); List knowledgeDetailInfo = chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos(); - fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId); - fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId); + fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, + modelId, globalKnowledgeConfigDetail); + fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, + modelId, globalKnowledgeConfigAgg); } @@ -150,7 +159,8 @@ public class DictMetaHelper { private void fillKnowledgeDimValue(List knowledgeInfos, ChatDefaultRichConfigResp chatDefaultConfig, List dimValueDOList, - Map dimIdAndDescPair, Long modelId) { + Map dimIdAndDescPair, Long modelId, + KnowledgeAdvancedConfig globalKnowledgeConfigDetail) { if (!CollectionUtils.isEmpty(knowledgeInfos)) { List dimensions = new ArrayList<>(); List defaultMetricDescList = new ArrayList<>(); @@ -159,36 +169,41 @@ public class DictMetaHelper { && !CollectionUtils.isEmpty(dimIdAndDescPair) && dimIdAndDescPair.containsKey(knowledgeInfo.getItemId())) .forEach(knowledgeInfo -> { - if (dimIdAndDescPair.containsKey(knowledgeInfo.getItemId())) { - SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId()); - - //default cnt - if (Objects.isNull(chatDefaultConfig) - || CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) { - String datasourceBizName = dimensionDesc.getBizName(); - if (Strings.isNotEmpty(datasourceBizName)) { - String internalMetricName = - datasourceBizName + UNDERLINE + internalMetricNameSuffix; - defaultMetricDescList.add(new DefaultMetric(internalMetricName, 2, DAY)); - } - } else { - SchemaElement schemaItem = chatDefaultConfig.getMetrics().get(0); - defaultMetricDescList.add(new DefaultMetric(schemaItem.getBizName(), - chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod())); + SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId()); + //default cnt + if (Objects.isNull(chatDefaultConfig) + || CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) { + String datasourceBizName = dimensionDesc.getBizName(); + if (Strings.isNotEmpty(datasourceBizName)) { + String internalMetricName = + datasourceBizName + UNDERLINE + internalMetricNameSuffix; + defaultMetricDescList.add(new DefaultMetric(internalMetricName, + internalMetricDays, DAY)); } + } else { + SchemaElement schemaItem = chatDefaultConfig.getMetrics().get(0); + defaultMetricDescList.add(new DefaultMetric(schemaItem.getBizName(), + chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod())); - String bizName = dimensionDesc.getBizName(); - Dim4Dict dim4Dict = new Dim4Dict(); - dim4Dict.setDimId(knowledgeInfo.getItemId()); - dim4Dict.setBizName(bizName); - if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) { - KnowledgeAdvancedConfig knowledgeAdvancedConfig - = knowledgeInfo.getKnowledgeAdvancedConfig(); - BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict); - } - dimensions.add(dim4Dict); } + + String bizName = dimensionDesc.getBizName(); + Dim4Dict dim4Dict = new Dim4Dict(); + dim4Dict.setDimId(knowledgeInfo.getItemId()); + dim4Dict.setBizName(bizName); + if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) { + KnowledgeAdvancedConfig knowledgeAdvancedConfig + = knowledgeInfo.getKnowledgeAdvancedConfig(); + BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict); + + if (Objects.nonNull(globalKnowledgeConfigDetail) + && !CollectionUtils.isEmpty(globalKnowledgeConfigDetail.getRuleList())) { + dim4Dict.getRuleList().addAll(globalKnowledgeConfigDetail.getRuleList()); + } + } + dimensions.add(dim4Dict); + }); if (!CollectionUtils.isEmpty(dimensions)) { diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java index 539ddc726..35102e1c0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java @@ -1,10 +1,5 @@ package com.tencent.supersonic.chat.utils; -import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; -import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; -import static com.tencent.supersonic.common.pojo.Constants.COMMA; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE; - import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticLayer; import com.tencent.supersonic.chat.config.DefaultMetric; @@ -34,6 +29,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import static com.tencent.supersonic.common.pojo.Constants.SPACE; +import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; +import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE; + @Slf4j @Component public class DictQueryHelper { @@ -46,6 +47,8 @@ public class DictQueryHelper { private Integer printDataShow; @Value("${dimension.max.limit:3000000}") private Long dimMaxLimit; + @Value("${dimension.white.weight:60000000}") + private Long dimensionWhiteWeight; public List fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict, User user) { @@ -53,10 +56,11 @@ public class DictQueryHelper { QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict); try { QueryResultWithSchemaResp queryResultWithColumns = semanticLayer.queryByStruct(queryStructCmd, user); + log.info("fetchDimValueSingle sql:{}", queryResultWithColumns.getSql()); String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId()); String dimNameRewrite = rewriteDimName(queryResultWithColumns.getColumns(), dim4Dict.getBizName()); data = generateFileData(queryResultWithColumns.getResultList(), nature, dimNameRewrite, - defaultMetricDesc.getBizName()); + defaultMetricDesc.getBizName(), dim4Dict); if (!CollectionUtils.isEmpty(data)) { int size = (data.size() > printDataShow) ? printDataShow : data.size(); log.info("data:{}", data.subList(0, size - 1)); @@ -91,7 +95,7 @@ public class DictQueryHelper { } private List generateFileData(List> resultList, String nature, String dimName, - String metricName) { + String metricName, Dim4Dict dim4Dict) { List data = new ArrayList<>(); if (CollectionUtils.isEmpty(resultList)) { return data; @@ -111,17 +115,26 @@ public class DictQueryHelper { } } - constructDataLines(valueAndFrequencyPair, nature, data); + constructDataLines(valueAndFrequencyPair, nature, data, dim4Dict); return data; } - private void constructDataLines(Map valueAndFrequencyPair, String nature, List data) { + private void constructDataLines(Map valueAndFrequencyPair, String nature, + List data, Dim4Dict dim4Dict) { valueAndFrequencyPair.forEach((dimValue, metric) -> { if (metric > MAX_FREQUENCY) { metric = MAX_FREQUENCY; } + if (Strings.isNotEmpty(dimValue) && dimValue.contains(SPACE)) { + dimValue = dimValue.replace(SPACE, "#"); + } data.add(String.format("%s %s %s", dimValue, nature, metric)); }); + + if (Objects.nonNull(dim4Dict) && !CollectionUtils.isEmpty(dim4Dict.getWhiteList())) { + dim4Dict.getWhiteList().stream() + .forEach(white -> data.add(String.format("%s %s %s", white, nature, dimensionWhiteWeight))); + } } private void mergeMultivaluedValue(Map valueAndFrequencyPair, String dimValue, Long metric) { @@ -185,7 +198,7 @@ public class DictQueryHelper { if (Objects.isNull(dim4Dict)) { return ""; } - StringJoiner joiner = new StringJoiner(AND_UPPER); + StringJoiner joiner = new StringJoiner(SPACE + AND_UPPER + SPACE); String dimName = dim4Dict.getBizName(); if (!CollectionUtils.isEmpty(dim4Dict.getBlackList())) { diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictTaskFilter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictTaskFilter.java deleted file mode 100644 index c5758e8cc..000000000 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictTaskFilter.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.tencent.supersonic.knowledge.dictionary; - - -public class DictTaskFilter { - - private Long id; - - private String name; - - private String createdBy; - - private String createdAt; - - private Integer status; -} \ No newline at end of file diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DimValueDictInfo.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DimValueDictInfo.java index 153e6afdd..a93dd07fb 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DimValueDictInfo.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DimValueDictInfo.java @@ -2,7 +2,10 @@ package com.tencent.supersonic.knowledge.dictionary; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; + import java.util.Date; +import java.util.Set; + import lombok.Data; @Data @@ -23,4 +26,6 @@ public class DimValueDictInfo { private Date createdAt; private Long elapsedMs; + + private Set dimIds; } \ No newline at end of file diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/LocalFileHandler.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/LocalFileHandler.java index 2752b0cca..26adb9579 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/LocalFileHandler.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/LocalFileHandler.java @@ -10,6 +10,7 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.List; + import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -37,7 +38,7 @@ public class LocalFileHandler implements FileHandler { Path targetPath = Paths.get(target); try { Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); - log.info("File copied successfully!"); + log.info("backupFile successfully! path:{}", targetPath.toAbsolutePath()); } catch (IOException e) { log.info("Failed to copy file: " + e.getMessage()); } @@ -62,7 +63,7 @@ public class LocalFileHandler implements FileHandler { Files.delete(path); log.info("File:{} deleted successfully!", getAbsolutePath(filePath)); } catch (IOException e) { - log.info("Failed to delete file:{}, e:", getAbsolutePath(filePath), e); + log.warn("Failed to delete file:{}, e:", getAbsolutePath(filePath), e); } } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java index a9f1e8161..44973ac28 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java @@ -13,6 +13,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import java.util.List; +import java.util.concurrent.CompletableFuture; @Slf4j @Component @@ -47,6 +48,14 @@ public class ApplicationStartedListener implements ApplicationListener { + updateKnowledgeDimValue(); + return null; + }); + return true; + } + /*** * reload knowledge task */ diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/dataobject/DictTaskDO.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/dataobject/DictTaskDO.java index 12ff115bc..44a5776fb 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/dataobject/DictTaskDO.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/dataobject/DictTaskDO.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.knowledge.persistence.dataobject; import java.util.Date; + import lombok.Data; import lombok.ToString; import org.apache.commons.codec.digest.DigestUtils; @@ -19,6 +20,8 @@ public class DictTaskDO { private String commandMd5; + private String dimIds; + private Integer status; private String createdBy; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/mapper/DictTaskMapper.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/mapper/DictTaskMapper.java index a63926643..ab83cb29d 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/mapper/DictTaskMapper.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/mapper/DictTaskMapper.java @@ -1,8 +1,10 @@ package com.tencent.supersonic.knowledge.persistence.mapper; import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO; -import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter; +import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; + import java.util.List; + import org.apache.ibatis.annotations.Mapper; @Mapper @@ -12,5 +14,5 @@ public interface DictTaskMapper { Boolean updateTaskStatus(DictTaskDO dictTaskDO); - List searchDictTaskList(DictTaskFilter filter); + List searchDictTaskList(DictTaskFilterReq filter); } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepository.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepository.java index ac029aed9..556e96e9b 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepository.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepository.java @@ -3,7 +3,7 @@ package com.tencent.supersonic.knowledge.persistence.repository; import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.knowledge.dictionary.DictConfig; -import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter; +import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo; import java.util.List; @@ -14,7 +14,7 @@ public interface DictRepository { Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO); - List searchDictTaskList(DictTaskFilter filter); + List searchDictTaskList(DictTaskFilterReq filter); DictConfig getDictInfoByModelId(Long modelId); } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepositoryImpl.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepositoryImpl.java index f460bcf30..ad24f2896 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepositoryImpl.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepositoryImpl.java @@ -1,27 +1,32 @@ package com.tencent.supersonic.knowledge.persistence.repository; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; +import com.tencent.supersonic.common.util.JsonUtil; import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO; import com.tencent.supersonic.knowledge.utils.DictTaskConverter; import com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO; import com.tencent.supersonic.knowledge.dictionary.DictConfig; -import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter; +import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq; import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo; import com.tencent.supersonic.knowledge.persistence.mapper.DictConfMapper; import com.tencent.supersonic.knowledge.persistence.mapper.DictTaskMapper; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Repository; import org.springframework.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.Objects; + @Repository +@Slf4j public class DictRepositoryImpl implements DictRepository { private final DictTaskMapper dictTaskMapper; @@ -46,22 +51,24 @@ public class DictRepositoryImpl implements DictRepository { Date createdAt = dictTaskDO.getCreatedAt(); long elapsedMs = System.currentTimeMillis() - createdAt.getTime(); dictTaskDO.setElapsedMs(elapsedMs); - CompletableFuture.supplyAsync(() -> { - dictTaskMapper.updateTaskStatus(dictTaskDO); - return null; - }); + dictTaskMapper.updateTaskStatus(dictTaskDO); return true; } @Override - public List searchDictTaskList(DictTaskFilter filter) { + public List searchDictTaskList(DictTaskFilterReq filter) { List dimValueDictDescList = new ArrayList<>(); + log.info("filter:{}", filter); List dictTaskDOList = dictTaskMapper.searchDictTaskList(filter); if (!CollectionUtils.isEmpty(dictTaskDOList)) { - dictTaskDOList.stream().forEach(dictTaskPO -> { + dictTaskDOList.stream().forEach(dictTaskDO -> { DimValueDictInfo dimValueDictDesc = new DimValueDictInfo(); - BeanUtils.copyProperties(dictTaskPO, dimValueDictDesc); - dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskPO.getStatus())); + BeanUtils.copyProperties(dictTaskDO, dimValueDictDesc); + dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskDO.getStatus())); + if (StringUtils.isNotEmpty(dictTaskDO.getDimIds())) { + Set dimIds = JsonUtil.toSet(dictTaskDO.getDimIds(), Long.class); + dimValueDictDesc.setDimIds(dimIds); + } dimValueDictDescList.add(dimValueDictDesc); }); } diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/DictTaskConverter.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/DictTaskConverter.java index 67ce6cf34..821dd7041 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/DictTaskConverter.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/DictTaskConverter.java @@ -19,7 +19,7 @@ public class DictTaskConverter { private static String dateTimeFormatter = "yyyyMMddHHmmss"; - public static DictTaskDO generateDimValueDictTaskPO(DimValue2DictCommand dimValue2DictCommend, User user) { + public static DictTaskDO generateDimValueDictTaskDO(DimValue2DictCommand dimValue2DictCommend, User user) { DictTaskDO taskPO = new DictTaskDO(); Date createAt = new Date(); String date = DateTimeFormatter.ofPattern(dateTimeFormatter) @@ -31,7 +31,7 @@ public class DictTaskConverter { taskPO.setCreatedAt(createAt); taskPO.setCommand(JsonUtil.toString(dimValue2DictCommend)); - taskPO.setStatus(TaskStatusEnum.RUNNING.getCode()); + taskPO.setStatus(TaskStatusEnum.PENDING.getCode()); taskPO.setCreatedBy(creator); return taskPO; diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/FileHelper.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/FileHelper.java index 3ad816b90..219342925 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/FileHelper.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/FileHelper.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.knowledge.utils; import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath; import com.hankcs.hanlp.dictionary.DynamicCustomDictionary; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -25,7 +26,7 @@ public class FileHelper { for (File file : customSubFiles) { try { file.delete(); - log.info("customPath:{},delete cache file:{}", customPath, file); + log.info("customPath:{},delete file:{}", customPath, file); } catch (Exception e) { log.error("delete " + file, e); } @@ -70,7 +71,7 @@ public class FileHelper { } } - log.info("CustomDictionaryPath:{}", fileList); + log.debug("CustomDictionaryPath:{}", fileList); CustomDictionaryPath = fileList.toArray(new String[0]); customDictionary.path = (CustomDictionaryPath == null || CustomDictionaryPath.length == 0) ? path : CustomDictionaryPath; diff --git a/chat/knowledge/src/main/resources/mapper/DictTaskMapper.xml b/chat/knowledge/src/main/resources/mapper/DictTaskMapper.xml index 2f26482a4..e378f7a80 100644 --- a/chat/knowledge/src/main/resources/mapper/DictTaskMapper.xml +++ b/chat/knowledge/src/main/resources/mapper/DictTaskMapper.xml @@ -11,6 +11,7 @@ + @@ -18,11 +19,11 @@ - + insert into s2_dictionary_task - (`name`, description, command, command_md5, status, created_by, progress, elapsed_ms) + (`name`, description, command, command_md5, dimension_ids, status, created_by, progress, elapsed_ms) values - (#{name}, #{description}, #{command}, #{commandMd5}, #{status}, #{createdBy}, #{progress}, #{elapsedMs}) + (#{name}, #{description}, #{command}, #{commandMd5}, #{dimIds}, #{status}, #{createdBy}, #{progress}, #{elapsedMs}) @@ -34,6 +35,9 @@ status = #{status}, + + dimension_ids = #{dimIds}, + progress = #{progress}, @@ -42,8 +46,7 @@ - where name = #{name} - and status = 0 + where id = #{id} \ No newline at end of file diff --git a/chat/knowledge/src/main/resources/sql.ddl/s2_dictionary_task.sql b/chat/knowledge/src/main/resources/sql.ddl/s2_dictionary_task.sql index 3744f3106..1b9cb8dbf 100644 --- a/chat/knowledge/src/main/resources/sql.ddl/s2_dictionary_task.sql +++ b/chat/knowledge/src/main/resources/sql.ddl/s2_dictionary_task.sql @@ -3,9 +3,11 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( `name` varchar(255) NOT NULL COMMENT '任务名称', `description` varchar(255) NOT NULL COMMENT '任务描述', `command` mediumtext NOT NULL COMMENT '任务请求参数', + `dimension_ids` mediumtext NULL COMMENT '本次执行维度列表', `status` int(10) NOT NULL COMMENT '任务最终运行状态', `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `created_by` varchar(100) NOT NULL COMMENT '创建人', `elapsed_ms` bigint(10) DEFAULT NULL COMMENT '任务耗时', + `message` mediumtext COLLATE utf8mb4_unicode_ci COMMENT '备注相关信息', PRIMARY KEY (`id`) )COMMENT='字典任务信息表' diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java index 21e8861ca..03f3349ff 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java @@ -2,13 +2,15 @@ package com.tencent.supersonic.common.pojo.enums; public enum TaskStatusEnum { - RUNNING("running", 0), - - SUCCESS("success", 1), - ERROR("error", -1), - UNKNOWN("UNKNOWN", 2); + PENDING("pending", 0), + + RUNNING("running", 1), + + SUCCESS("success", 2), + + UNKNOWN("UNKNOWN", 3); private String status; private Integer code; diff --git a/launchers/chat/src/main/resources/db/chat-schema-h2.sql b/launchers/chat/src/main/resources/db/chat-schema-h2.sql index bf83e93ef..8bcad7b6d 100644 --- a/launchers/chat/src/main/resources/db/chat-schema-h2.sql +++ b/launchers/chat/src/main/resources/db/chat-schema-h2.sql @@ -118,6 +118,7 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( `description` varchar(255) , `command`LONGVARCHAR NOT NULL , -- task Request Parameters `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 + `dimension_ids` varchar(500) , `status` INT NOT NULL , -- the final status of the task `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `created_by` varchar(100) NOT NULL , diff --git a/launchers/standalone/pom.xml b/launchers/standalone/pom.xml index 0a82f3708..bc9d9d245 100644 --- a/launchers/standalone/pom.xml +++ b/launchers/standalone/pom.xml @@ -28,6 +28,11 @@ chat-core ${project.version} + + com.tencent.supersonic + chat-api + ${project.version} + com.tencent.supersonic diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index f433756a2..b8d090f07 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -400,6 +400,7 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` ( `command`LONGVARCHAR NOT NULL , -- task Request Parameters `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5 `status` INT NOT NULL , -- the final status of the task + `dimension_ids` varchar(500) NULL , `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , `created_by` varchar(100) NOT NULL , `progress` DOUBLE default 0.00 , -- task real-time progress diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index 6347a92e2..da3579178 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -183,6 +183,7 @@ CREATE TABLE `s2_dictionary_task` ( `description` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '任务描述', `command` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数', `command_md5` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数', + `dimension_ids` mediumtext NULL COMMENT '本次执行维度列表', `status` int(10) NOT NULL COMMENT '任务最终运行状态', `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `created_by` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '创建人',