topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[knowledge](improve) add knowledge base dimension value task manage (#88)

Browse Source
This commit is contained in:
daikon
2023-09-14 14:01:38 +08:00
committed by GitHub
parent 157c2999dc
commit 592870f397
26 changed files with 536 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java Normal file
View File

@@ -0,0 +1,23 @@
package com.tencent.supersonic.chat.api.pojo.request;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import javax.validation.constraints.NotNull;
import java.util.List;
import static java.time.LocalDate.now;
@ToString
@Data
@NoArgsConstructor
public class DictLatestTaskReq {
@NotNull
private Long modelId;
private List<Long> dimIds;
private String createdAt = now().plusDays(-4).toString();
}

20
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java Normal file
View File

@@ -0,0 +1,20 @@
package com.tencent.supersonic.chat.api.pojo.request;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import lombok.Data;
import lombok.ToString;
@ToString
@Data
public class DictTaskFilterReq {
private Long id;
private String name;
private String createdBy;
private String createdAt;
private TaskStatusEnum status;
}

30
chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java Normal file
View File

@@ -0,0 +1,30 @@
package com.tencent.supersonic.chat.api.pojo.response;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
@ToString
@Data
public class DictLatestTaskResp {
private Long dimId;
private Long id;
private String name;
private String description;
private String command;
private TaskStatusEnum status;
private String createdBy;
private Date createdAt;
private Long elapsedMs;
}

49
chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java
View File

@@ -3,22 +3,24 @@ package com.tencent.supersonic.chat.rest;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.chat.service.DictionaryService;
import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
import com.tencent.supersonic.chat.service.ChatKnowledgeService;
import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PutMapping;
@@ -28,13 +30,14 @@ import org.springframework.web.bind.annotation.PutMapping;
public class KnowledgeController {
@Autowired
private DictionaryService dictApplicationService;
private ChatKnowledgeService knowledgeService;
@Autowired
private ApplicationStartedListener applicationStartedListener;
/**
* addDictInfo
* write specific dimension values to the knowledge base
*
* @param dimValue2DictCommend
*/
@@ -43,20 +46,21 @@ public class KnowledgeController {
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return dictApplicationService.addDictTask(dimValue2DictCommend, user);
return knowledgeService.addDictTask(dimValue2DictCommend, user);
}
/**
* deleteDictInfo
* remove specific dimension values from the knowledge base
*
* @param dimValue2DictCommend
*/
@DeleteMapping("/task")
@PostMapping("/task/delete")
public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return dictApplicationService.deleteDictTask(dimValue2DictCommend, user);
return knowledgeService.deleteDictTask(dimValue2DictCommend, user);
}
/**
@@ -65,19 +69,44 @@ public class KnowledgeController {
* @param filter
*/
@PostMapping("/task/search")
public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilter filter,
public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilterReq filter,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return dictApplicationService.searchDictTaskList(filter, user);
return knowledgeService.searchDictTaskList(filter, user);
}
/**
* searchDictLatestTaskList
*/
@PostMapping("/task/search/latest")
public List<DictLatestTaskResp> searchDictLatestTaskList(@RequestBody @Valid DictLatestTaskReq filter,
HttpServletRequest request,
HttpServletResponse response) {
User user = UserHolder.findUser(request, response);
return knowledgeService.searchDictLatestTaskList(filter, user);
}
/**
* getDictRootPath
* get knowledge base file directory
*
* @return
*/
@GetMapping("/rootPath")
public String getDictRootPath(HttpServletRequest request,
HttpServletResponse response) {
return dictApplicationService.getDictRootPath();
return knowledgeService.getDictRootPath();
}
/**
* updateDimValue
* update in-memory dictionary files in real time
*
* @param request
* @param response
* @return
*/
@PutMapping("/knowledge/dimValue")
public Boolean updateDimValue(HttpServletRequest request,
HttpServletResponse response) {

10
chat/core/src/main/java/com/tencent/supersonic/chat/service/DictionaryService.java → chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatKnowledgeService.java
View File

@@ -1,21 +1,25 @@
package com.tencent.supersonic.chat.service;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import java.util.List;
public interface DictionaryService {
public interface ChatKnowledgeService {
Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter, User user);
List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user);
DictConfig getDictInfoByModelId(Long modelId);
String getDictRootPath();
List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq filter, User user);
}

268
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatKnowledgeServiceImpl.java Normal file
View File

@@ -0,0 +1,268 @@
package com.tencent.supersonic.chat.service.impl;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
import com.tencent.supersonic.chat.config.DefaultMetric;
import com.tencent.supersonic.chat.config.Dim4Dict;
import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
import com.tencent.supersonic.chat.service.ChatKnowledgeService;
import com.tencent.supersonic.chat.utils.DictMetaHelper;
import com.tencent.supersonic.chat.utils.DictQueryHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.knowledge.dictionary.FileHandler;
import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import com.tencent.supersonic.knowledge.persistence.repository.DictRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;
import java.util.Objects;
import java.util.Set;
import java.util.Map;
import java.util.HashMap;
@Slf4j
@Service
public class ChatKnowledgeServiceImpl implements ChatKnowledgeService {
private final DictMetaHelper metaUtils;
private final DictQueryHelper dictQueryHelper;
private final FileHandler fileHandler;
private final DictRepository dictRepository;
private final ApplicationStartedListener applicationStartedListener;
@Value("${dict.flush.enable:true}")
private Boolean dictFlushEnable;
@Value("${dict.flush.daily.enable:true}")
private Boolean dictFlushDailyEnable;
@Value("${dict.file.type:txt}")
private String dictFileType;
private String dimValue = "DimValue_%d_%d";
public ChatKnowledgeServiceImpl(DictMetaHelper metaUtils,
DictQueryHelper dictQueryHelper,
FileHandler fileHandler,
DictRepository dictRepository,
ApplicationStartedListener applicationStartedListener) {
this.metaUtils = metaUtils;
this.dictQueryHelper = dictQueryHelper;
this.fileHandler = fileHandler;
this.dictRepository = dictRepository;
this.applicationStartedListener = applicationStartedListener;
}
@Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}")
public Boolean dailyDictTask() {
log.info("[dailyDictTask] start");
if (!dictFlushDailyEnable) {
log.info("dictFlushDailyEnable is false, now finish dailyDictTask");
}
DimValue2DictCommand dimValue2DictCommend = new DimValue2DictCommand();
dimValue2DictCommend.setUpdateMode(DictUpdateMode.OFFLINE_FULL);
User user = User.getFakeUser();
addDictTask(dimValue2DictCommend, user);
log.info("[dailyDictTask] finish");
return true;
}
@Override
public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
if (!dictFlushEnable) {
return 0L;
}
if (DictUpdateMode.REALTIME_DELETE.equals(dimValue2DictCommend.getUpdateMode())) {
return deleteDictTask(dimValue2DictCommend, user);
}
DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommend, user);
log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
// todo check dimension can not be searched
dictRepository.createDimValueDictTask(dictTaskDO);
runDictTask(dictTaskDO, user);
return dictTaskDO.getId();
}
public Long runDictTask(DictTaskDO dictTaskDO, User user) {
if (Objects.isNull(dictTaskDO)) {
return -1L;
}
DimValue2DictCommand command = JsonUtil.toObject(dictTaskDO.getCommand(), DimValue2DictCommand.class);
try {
//1. construct internal dictionary requirements
List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(command);
Set<Long> dimIds = generateDimSet(dimValueDOList);
dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
dictRepository.updateDictTaskStatus(TaskStatusEnum.RUNNING.getCode(), dictTaskDO);
log.debug("dimValueDOList:{}", dimValueDOList);
//2. query dimension value information
for (DimValueDO dimValueDO : dimValueDOList) {
Long modelId = dimValueDO.getModelId();
DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
//3. local file changes
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
dim4Dict.getDimId());
fileHandler.writeFile(data, fileName, false);
}
}
applicationStartedListener.updateKnowledgeDimValue();
log.debug("updateDictTaskStatus to SUCCESS");
dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
} catch (Exception e) {
log.warn("addDictInfo exception:", e);
dictRepository.updateDictTaskStatus(TaskStatusEnum.ERROR.getCode(), dictTaskDO);
}
return 1L;
}
private Set<Long> generateDimSet(List<DimValueDO> dimValueDOList) {
Set<Long> dimIds = new HashSet<>();
if (!CollectionUtils.isEmpty(dimValueDOList)) {
dimValueDOList.stream().forEach(dimValueDO -> {
if (!CollectionUtils.isEmpty(dimValueDO.getDimensions())) {
dimValueDO.getDimensions().stream().forEach(dim4Dict -> dimIds.add(dim4Dict.getDimId()));
}
});
}
return dimIds;
}
@Override
public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommand, User user) {
if (!dictFlushEnable) {
return 0L;
}
if (Objects.isNull(dimValue2DictCommand) || !DictUpdateMode.REALTIME_DELETE.equals(
dimValue2DictCommand.getUpdateMode())) {
throw new RuntimeException("illegal parameter");
}
DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommand, user);
log.info("[deleteDictTask] dictTaskDO:{}", dictTaskDO);
Set<Long> dimIds = generateDimSetFromCommand(dimValue2DictCommand.getModelAndDimPair());
dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
dictRepository.createDimValueDictTask(dictTaskDO);
Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommand.getModelAndDimPair();
if (CollectionUtils.isEmpty(modelAndDimPair)) {
return 0L;
}
for (Long modelId : modelAndDimPair.keySet()) {
if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
continue;
}
for (Long dimId : modelAndDimPair.get(modelId)) {
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
fileHandler.deleteDictFile(fileName);
}
}
applicationStartedListener.updateKnowledgeDimValue();
dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
applicationStartedListener.updateKnowledgeDimValue();
return 1L;
}
private Set<Long> generateDimSetFromCommand(Map<Long, List<Long>> modelAndDimPair) {
Set<Long> dimIds = new HashSet<>();
if (!CollectionUtils.isEmpty(modelAndDimPair)) {
modelAndDimPair.forEach((k, v) -> dimIds.addAll(v));
}
return dimIds;
}
@Override
public String getDictRootPath() {
return fileHandler.getDictRootPath();
}
@Override
public List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq latestFilter, User user) {
DictTaskFilterReq filter = new DictTaskFilterReq();
BeanUtils.copyProperties(latestFilter, filter);
List<DimValueDictInfo> dimValueDictInfoList = searchDictTaskList(filter, user);
return extractLatestTask(dimValueDictInfoList, latestFilter.getDimIds());
}
private List<DictLatestTaskResp> extractLatestTask(List<DimValueDictInfo> dimValueDictInfoList, List<Long> dimIds) {
List<DictLatestTaskResp> dictLatestTaskRespList = new ArrayList<>();
Map<Long, DictLatestTaskResp> dimAndTaskPair = new HashMap<>(50);
for (DimValueDictInfo dimValueDictInfo : dimValueDictInfoList) {
//1. filter
if (Objects.isNull(dimValueDictInfo) || CollectionUtils.isEmpty(dimValueDictInfo.getDimIds())) {
continue;
}
if (!CollectionUtils.isEmpty(dimIds)) {
Set<Long> tmp = dimValueDictInfo.getDimIds();
tmp.retainAll(dimIds);
dimValueDictInfo.setDimIds(tmp);
if (tmp.size() <= 0) {
continue;
}
}
// 2. extract
Set<Long> dimIdList = dimValueDictInfo.getDimIds();
for (Long dimId : dimIdList) {
DictLatestTaskResp dictLatestTaskResp = new DictLatestTaskResp();
if (!dimAndTaskPair.containsKey(dimId)) {
BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
dictLatestTaskResp.setDimId(dimId);
} else {
DictLatestTaskResp dictLatestTaskExist = dimAndTaskPair.get(dimId);
if (dictLatestTaskExist.getCreatedAt().before(dimValueDictInfo.getCreatedAt())) {
BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
dictLatestTaskResp.setDimId(dimId);
} else {
dictLatestTaskResp = dictLatestTaskExist;
}
}
dimAndTaskPair.put(dimId, dictLatestTaskResp);
}
}
if (dimAndTaskPair.size() >= 0 && !CollectionUtils.isEmpty(dimAndTaskPair.values())) {
dimAndTaskPair.values().stream()
.filter(v -> !v.getCommand().contains(DictUpdateMode.REALTIME_DELETE.name()))
.forEach(v -> dictLatestTaskRespList.add(v));
}
return dictLatestTaskRespList;
}
@Override
public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user) {
return dictRepository.searchDictTaskList(filter);
}
@Override
public DictConfig getDictInfoByModelId(Long modelId) {
return dictRepository.getDictInfoByModelId(modelId);
}
}

124
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/DictionaryServiceImpl.java
View File

@@ -1,124 +0,0 @@
package com.tencent.supersonic.chat.service.impl;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.config.DefaultMetric;
import com.tencent.supersonic.chat.config.Dim4Dict;
import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
import com.tencent.supersonic.chat.service.DictionaryService;
import com.tencent.supersonic.chat.utils.DictMetaHelper;
import com.tencent.supersonic.chat.utils.DictQueryHelper;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.knowledge.dictionary.FileHandler;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import com.tencent.supersonic.knowledge.persistence.repository.DictRepository;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j
@Service
public class DictionaryServiceImpl implements DictionaryService {
private final DictMetaHelper metaUtils;
private final DictQueryHelper dictQueryHelper;
private final FileHandler fileHandler;
private final DictRepository dictRepository;
@Value("${dict.flush.enable:true}")
private Boolean dictFlushEnable;
@Value("${dict.file.type:txt}")
private String dictFileType;
private String dimValue = "DimValue_%d_%d";
public DictionaryServiceImpl(DictMetaHelper metaUtils,
DictQueryHelper dictQueryHelper,
FileHandler fileHandler,
DictRepository dictRepository) {
this.metaUtils = metaUtils;
this.dictQueryHelper = dictQueryHelper;
this.fileHandler = fileHandler;
this.dictRepository = dictRepository;
}
public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
if (!dictFlushEnable) {
return 0L;
}
DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskPO(dimValue2DictCommend,
user);
log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
dictRepository.createDimValueDictTask(dictTaskDO);
TaskStatusEnum finalStatus = TaskStatusEnum.SUCCESS;
try {
//1. construct internal dictionary requirements
List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(dimValue2DictCommend);
log.info("dimValueDOList:{}", dimValueDOList);
//2. query dimension value information
for (DimValueDO dimValueDO : dimValueDOList) {
Long modelId = dimValueDO.getModelId();
DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
//3. local file changes
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
dim4Dict.getDimId());
fileHandler.writeFile(data, fileName, false);
}
}
} catch (Exception e) {
log.warn("addDictInfo exception:", e);
finalStatus = TaskStatusEnum.ERROR;
}
dictRepository.updateDictTaskStatus(finalStatus.getCode(),
dictTaskDO);
return 1L;
}
public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
if (!dictFlushEnable) {
return 0L;
}
if (Objects.isNull(dimValue2DictCommend) || DictUpdateMode.REALTIME_DELETE.equals(
dimValue2DictCommend.getUpdateMode())) {
throw new RuntimeException("illegal parameter");
}
Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommend.getModelAndDimPair();
if (CollectionUtils.isEmpty(modelAndDimPair)) {
return 0L;
}
for (Long modelId : modelAndDimPair.keySet()) {
if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
continue;
}
for (Long dimId : modelAndDimPair.get(modelId)) {
String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
fileHandler.deleteDictFile(fileName);
}
}
return 1L;
}
public String getDictRootPath() {
return fileHandler.getDictRootPath();
}
public List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter, User user) {
return dictRepository.searchDictTaskList(filter);
}
public DictConfig getDictInfoByModelId(Long modelId) {
return dictRepository.getDictInfoByModelId(modelId);
}
}

73
chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java
View File

@@ -40,6 +40,8 @@ public class DictMetaHelper {
private ConfigService configService;
@Value("${model.internal.metric.suffix:internal_cnt}")
private String internalMetricNameSuffix;
@Value("${model.internal.day.number:2}")
private Integer internalMetricDays;
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
public List<DimValueDO> generateDimValueInfo(DimValue2DictCommand dimValue2DictCommend) {
@@ -134,14 +136,21 @@ public class DictMetaHelper {
ChatDefaultRichConfigResp chatDefaultConfig =
chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
KnowledgeAdvancedConfig globalKnowledgeConfigAgg = chaConfigRichDesc.getChatAggRichConfig()
.getGlobalKnowledgeConfig();
List<KnowledgeInfoReq> knowledgeAggInfo =
chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos();
KnowledgeAdvancedConfig globalKnowledgeConfigDetail = chaConfigRichDesc.getChatDetailRichConfig()
.getGlobalKnowledgeConfig();
List<KnowledgeInfoReq> knowledgeDetailInfo =
chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos();
fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair,
modelId, globalKnowledgeConfigDetail);
fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair,
modelId, globalKnowledgeConfigAgg);
}
@@ -150,7 +159,8 @@ public class DictMetaHelper {
private void fillKnowledgeDimValue(List<KnowledgeInfoReq> knowledgeInfos,
ChatDefaultRichConfigResp chatDefaultConfig,
List<DimValueDO> dimValueDOList,
Map<Long, SchemaElement> dimIdAndDescPair, Long modelId) {
Map<Long, SchemaElement> dimIdAndDescPair, Long modelId,
KnowledgeAdvancedConfig globalKnowledgeConfigDetail) {
if (!CollectionUtils.isEmpty(knowledgeInfos)) {
List<Dim4Dict> dimensions = new ArrayList<>();
List<DefaultMetric> defaultMetricDescList = new ArrayList<>();
@@ -159,36 +169,41 @@ public class DictMetaHelper {
&& !CollectionUtils.isEmpty(dimIdAndDescPair)
&& dimIdAndDescPair.containsKey(knowledgeInfo.getItemId()))
.forEach(knowledgeInfo -> {
if (dimIdAndDescPair.containsKey(knowledgeInfo.getItemId())) {
SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId());
//default cnt
if (Objects.isNull(chatDefaultConfig)
|| CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
String datasourceBizName = dimensionDesc.getBizName();
if (Strings.isNotEmpty(datasourceBizName)) {
String internalMetricName =
datasourceBizName + UNDERLINE + internalMetricNameSuffix;
defaultMetricDescList.add(new DefaultMetric(internalMetricName, 2, DAY));
}
} else {
SchemaElement schemaItem = chatDefaultConfig.getMetrics().get(0);
defaultMetricDescList.add(new DefaultMetric(schemaItem.getBizName(),
chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod()));
SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId());
//default cnt
if (Objects.isNull(chatDefaultConfig)
|| CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
String datasourceBizName = dimensionDesc.getBizName();
if (Strings.isNotEmpty(datasourceBizName)) {
String internalMetricName =
datasourceBizName + UNDERLINE + internalMetricNameSuffix;
defaultMetricDescList.add(new DefaultMetric(internalMetricName,
internalMetricDays, DAY));
}
} else {
SchemaElement schemaItem = chatDefaultConfig.getMetrics().get(0);
defaultMetricDescList.add(new DefaultMetric(schemaItem.getBizName(),
chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod()));
String bizName = dimensionDesc.getBizName();
Dim4Dict dim4Dict = new Dim4Dict();
dim4Dict.setDimId(knowledgeInfo.getItemId());
dim4Dict.setBizName(bizName);
if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) {
KnowledgeAdvancedConfig knowledgeAdvancedConfig
= knowledgeInfo.getKnowledgeAdvancedConfig();
BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict);
}
dimensions.add(dim4Dict);
}
String bizName = dimensionDesc.getBizName();
Dim4Dict dim4Dict = new Dim4Dict();
dim4Dict.setDimId(knowledgeInfo.getItemId());
dim4Dict.setBizName(bizName);
if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) {
KnowledgeAdvancedConfig knowledgeAdvancedConfig
= knowledgeInfo.getKnowledgeAdvancedConfig();
BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict);
if (Objects.nonNull(globalKnowledgeConfigDetail)
&& !CollectionUtils.isEmpty(globalKnowledgeConfigDetail.getRuleList())) {
dim4Dict.getRuleList().addAll(globalKnowledgeConfigDetail.getRuleList());
}
}
dimensions.add(dim4Dict);
});
if (!CollectionUtils.isEmpty(dimensions)) {

33
chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java
View File

@@ -1,10 +1,5 @@
package com.tencent.supersonic.chat.utils;
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.config.DefaultMetric;
@@ -34,6 +29,12 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import static com.tencent.supersonic.common.pojo.Constants.SPACE;
import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
import static com.tencent.supersonic.common.pojo.Constants.COMMA;
import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE;
@Slf4j
@Component
public class DictQueryHelper {
@@ -46,6 +47,8 @@ public class DictQueryHelper {
private Integer printDataShow;
@Value("${dimension.max.limit:3000000}")
private Long dimMaxLimit;
@Value("${dimension.white.weight:60000000}")
private Long dimensionWhiteWeight;
public List<String> fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
User user) {
@@ -53,10 +56,11 @@ public class DictQueryHelper {
QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict);
try {
QueryResultWithSchemaResp queryResultWithColumns = semanticLayer.queryByStruct(queryStructCmd, user);
log.info("fetchDimValueSingle sql:{}", queryResultWithColumns.getSql());
String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId());
String dimNameRewrite = rewriteDimName(queryResultWithColumns.getColumns(), dim4Dict.getBizName());
data = generateFileData(queryResultWithColumns.getResultList(), nature, dimNameRewrite,
defaultMetricDesc.getBizName());
defaultMetricDesc.getBizName(), dim4Dict);
if (!CollectionUtils.isEmpty(data)) {
int size = (data.size() > printDataShow) ? printDataShow : data.size();
log.info("data:{}", data.subList(0, size - 1));
@@ -91,7 +95,7 @@ public class DictQueryHelper {
}
private List<String> generateFileData(List<Map<String, Object>> resultList, String nature, String dimName,
String metricName) {
String metricName, Dim4Dict dim4Dict) {
List<String> data = new ArrayList<>();
if (CollectionUtils.isEmpty(resultList)) {
return data;
@@ -111,17 +115,26 @@ public class DictQueryHelper {
}
}
constructDataLines(valueAndFrequencyPair, nature, data);
constructDataLines(valueAndFrequencyPair, nature, data, dim4Dict);
return data;
}
private void constructDataLines(Map<String, Long> valueAndFrequencyPair, String nature, List<String> data) {
private void constructDataLines(Map<String, Long> valueAndFrequencyPair, String nature,
List<String> data, Dim4Dict dim4Dict) {
valueAndFrequencyPair.forEach((dimValue, metric) -> {
if (metric > MAX_FREQUENCY) {
metric = MAX_FREQUENCY;
}
if (Strings.isNotEmpty(dimValue) && dimValue.contains(SPACE)) {
dimValue = dimValue.replace(SPACE, "#");
}
data.add(String.format("%s %s %s", dimValue, nature, metric));
});
if (Objects.nonNull(dim4Dict) && !CollectionUtils.isEmpty(dim4Dict.getWhiteList())) {
dim4Dict.getWhiteList().stream()
.forEach(white -> data.add(String.format("%s %s %s", white, nature, dimensionWhiteWeight)));
}
}
private void mergeMultivaluedValue(Map<String, Long> valueAndFrequencyPair, String dimValue, Long metric) {
@@ -185,7 +198,7 @@ public class DictQueryHelper {
if (Objects.isNull(dim4Dict)) {
return "";
}
StringJoiner joiner = new StringJoiner(AND_UPPER);
StringJoiner joiner = new StringJoiner(SPACE + AND_UPPER + SPACE);
String dimName = dim4Dict.getBizName();
if (!CollectionUtils.isEmpty(dim4Dict.getBlackList())) {

15
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictTaskFilter.java
View File

@@ -1,15 +0,0 @@
package com.tencent.supersonic.knowledge.dictionary;
public class DictTaskFilter {
private Long id;
private String name;
private String createdBy;
private String createdAt;
private Integer status;
}

5
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DimValueDictInfo.java
View File

@@ -2,7 +2,10 @@ package com.tencent.supersonic.knowledge.dictionary;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import java.util.Date;
import java.util.Set;
import lombok.Data;
@Data
@@ -23,4 +26,6 @@ public class DimValueDictInfo {
private Date createdAt;
private Long elapsedMs;
private Set<Long> dimIds;
}

5
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/LocalFileHandler.java
View File

@@ -10,6 +10,7 @@ import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@@ -37,7 +38,7 @@ public class LocalFileHandler implements FileHandler {
Path targetPath = Paths.get(target);
try {
Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
log.info("File copied successfully!");
log.info("backupFile successfully! path:{}", targetPath.toAbsolutePath());
} catch (IOException e) {
log.info("Failed to copy file: " + e.getMessage());
}
@@ -62,7 +63,7 @@ public class LocalFileHandler implements FileHandler {
Files.delete(path);
log.info("File:{} deleted successfully!", getAbsolutePath(filePath));
} catch (IOException e) {
log.info("Failed to delete file:{}, e:", getAbsolutePath(filePath), e);
log.warn("Failed to delete file:{}, e:", getAbsolutePath(filePath), e);
}
}

9
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java
View File

@@ -13,6 +13,7 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@Slf4j
@Component
@@ -47,6 +48,14 @@ public class ApplicationStartedListener implements ApplicationListener<Applicati
return isOk;
}
public Boolean updateKnowledgeDimValueAsync() {
CompletableFuture.supplyAsync(() -> {
updateKnowledgeDimValue();
return null;
});
return true;
}
/***
* reload knowledge task
*/

3
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/dataobject/DictTaskDO.java
View File

@@ -1,6 +1,7 @@
package com.tencent.supersonic.knowledge.persistence.dataobject;
import java.util.Date;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.codec.digest.DigestUtils;
@@ -19,6 +20,8 @@ public class DictTaskDO {
private String commandMd5;
private String dimIds;
private Integer status;
private String createdBy;

6
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/mapper/DictTaskMapper.java
View File

@@ -1,8 +1,10 @@
package com.tencent.supersonic.knowledge.persistence.mapper;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
import java.util.List;
import org.apache.ibatis.annotations.Mapper;
@Mapper
@@ -12,5 +14,5 @@ public interface DictTaskMapper {
Boolean updateTaskStatus(DictTaskDO dictTaskDO);
List<DictTaskDO> searchDictTaskList(DictTaskFilter filter);
List<DictTaskDO> searchDictTaskList(DictTaskFilterReq filter);
}

4
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepository.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.knowledge.persistence.repository;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import java.util.List;
@@ -14,7 +14,7 @@ public interface DictRepository {
Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO);
List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter);
List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter);
DictConfig getDictInfoByModelId(Long modelId);
}

35
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepositoryImpl.java
View File

@@ -1,27 +1,32 @@
package com.tencent.supersonic.knowledge.persistence.repository;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
import com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO;
import com.tencent.supersonic.knowledge.dictionary.DictConfig;
import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
import com.tencent.supersonic.knowledge.persistence.mapper.DictConfMapper;
import com.tencent.supersonic.knowledge.persistence.mapper.DictTaskMapper;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Repository;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.Objects;
@Repository
@Slf4j
public class DictRepositoryImpl implements DictRepository {
private final DictTaskMapper dictTaskMapper;
@@ -46,22 +51,24 @@ public class DictRepositoryImpl implements DictRepository {
Date createdAt = dictTaskDO.getCreatedAt();
long elapsedMs = System.currentTimeMillis() - createdAt.getTime();
dictTaskDO.setElapsedMs(elapsedMs);
CompletableFuture.supplyAsync(() -> {
dictTaskMapper.updateTaskStatus(dictTaskDO);
return null;
});
dictTaskMapper.updateTaskStatus(dictTaskDO);
return true;
}
@Override
public List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter) {
public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter) {
List<DimValueDictInfo> dimValueDictDescList = new ArrayList<>();
log.info("filter:{}", filter);
List<DictTaskDO> dictTaskDOList = dictTaskMapper.searchDictTaskList(filter);
if (!CollectionUtils.isEmpty(dictTaskDOList)) {
dictTaskDOList.stream().forEach(dictTaskPO -> {
dictTaskDOList.stream().forEach(dictTaskDO -> {
DimValueDictInfo dimValueDictDesc = new DimValueDictInfo();
BeanUtils.copyProperties(dictTaskPO, dimValueDictDesc);
dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskPO.getStatus()));
BeanUtils.copyProperties(dictTaskDO, dimValueDictDesc);
dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskDO.getStatus()));
if (StringUtils.isNotEmpty(dictTaskDO.getDimIds())) {
Set<Long> dimIds = JsonUtil.toSet(dictTaskDO.getDimIds(), Long.class);
dimValueDictDesc.setDimIds(dimIds);
}
dimValueDictDescList.add(dimValueDictDesc);
});
}

4
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/DictTaskConverter.java
View File

@@ -19,7 +19,7 @@ public class DictTaskConverter {
private static String dateTimeFormatter = "yyyyMMddHHmmss";
public static DictTaskDO generateDimValueDictTaskPO(DimValue2DictCommand dimValue2DictCommend, User user) {
public static DictTaskDO generateDimValueDictTaskDO(DimValue2DictCommand dimValue2DictCommend, User user) {
DictTaskDO taskPO = new DictTaskDO();
Date createAt = new Date();
String date = DateTimeFormatter.ofPattern(dateTimeFormatter)
@@ -31,7 +31,7 @@ public class DictTaskConverter {
taskPO.setCreatedAt(createAt);
taskPO.setCommand(JsonUtil.toString(dimValue2DictCommend));
taskPO.setStatus(TaskStatusEnum.RUNNING.getCode());
taskPO.setStatus(TaskStatusEnum.PENDING.getCode());
taskPO.setCreatedBy(creator);
return taskPO;

5
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/FileHelper.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.knowledge.utils;
import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;
import com.hankcs.hanlp.dictionary.DynamicCustomDictionary;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -25,7 +26,7 @@ public class FileHelper {
for (File file : customSubFiles) {
try {
file.delete();
log.info("customPath:{},delete cache file:{}", customPath, file);
log.info("customPath:{},delete file:{}", customPath, file);
} catch (Exception e) {
log.error("delete " + file, e);
}
@@ -70,7 +71,7 @@ public class FileHelper {
}
}
log.info("CustomDictionaryPath:{}", fileList);
log.debug("CustomDictionaryPath:{}", fileList);
CustomDictionaryPath = fileList.toArray(new String[0]);
customDictionary.path = (CustomDictionaryPath == null || CustomDictionaryPath.length == 0) ? path
: CustomDictionaryPath;

22
chat/knowledge/src/main/resources/mapper/DictTaskMapper.xml
View File

@@ -11,6 +11,7 @@
<result column="description" property="description"/>
<result column="command" property="command"/>
<result column="command_md5" property="commandMd5"/>
<result column="dimension_ids" property="dimIds"/>
<result column="status" property="status"/>
<result column="created_by" property="createdBy"/>
<result column="created_at" property="createdAt"/>
@@ -18,11 +19,11 @@
<result column="elapsed_ms" property="elapsedMs"/>
</resultMap>
<insert id="createDimValueTask">
<insert id="createDimValueTask" useGeneratedKeys="true" keyProperty="id">
insert into s2_dictionary_task
(`name`, description, command, command_md5, status, created_by, progress, elapsed_ms)
(`name`, description, command, command_md5, dimension_ids, status, created_by, progress, elapsed_ms)
values
(#{name}, #{description}, #{command}, #{commandMd5}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
(#{name}, #{description}, #{command}, #{commandMd5}, #{dimIds}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
</insert>
<update id="updateTaskStatus">
@@ -34,6 +35,9 @@
<if test="status != null">
status = #{status},
</if>
<if test="dimIds != null">
dimension_ids = #{dimIds},
</if>
<if test="progress != null">
progress = #{progress},
</if>
@@ -42,8 +46,7 @@
</if>
</set>
where name = #{name}
and status = 0
where id = #{id}
</update>
<select id="searchDictTaskList" resultMap="DimValueDictTaskPO">
@@ -51,10 +54,10 @@
from s2_dictionary_task
<where>
<if test="id != null and id != ''">
and id >= #{id}
and id = #{id}
</if>
<if test="name != null and name !=''">
and `name` like "%"#{name}"%"
and `name` like CONCAT('%', #{name}, '%')
</if>
<if test="createdBy != null and createdBy !=''">
and created_by = #{createdBy}
@@ -62,10 +65,11 @@
<if test="createdAt != null and createdAt !=''">
and created_at &gt;= #{createdAt}
</if>
<if test="status != null and status !=''">
and status= #{status}
<if test="status != null">
and status= #{status.code}
</if>
</where>
order by created_at desc
</select>
</mapper>

2
chat/knowledge/src/main/resources/sql.ddl/s2_dictionary_task.sql
View File

@@ -3,9 +3,11 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`name` varchar(255) NOT NULL COMMENT '任务名称',
`description` varchar(255) NOT NULL COMMENT '任务描述',
`command` mediumtext NOT NULL COMMENT '任务请求参数',
`dimension_ids` mediumtext NULL COMMENT '本次执行维度列表',
`status` int(10) NOT NULL COMMENT '任务最终运行状态',
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT '任务耗时',
`message` mediumtext COLLATE utf8mb4_unicode_ci COMMENT '备注相关信息',
PRIMARY KEY (`id`)
)COMMENT='字典任务信息表'

12
common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java
View File

@@ -2,13 +2,15 @@ package com.tencent.supersonic.common.pojo.enums;
public enum TaskStatusEnum {
RUNNING("running", 0),
SUCCESS("success", 1),
ERROR("error", -1),
UNKNOWN("UNKNOWN", 2);
PENDING("pending", 0),
RUNNING("running", 1),
SUCCESS("success", 2),
UNKNOWN("UNKNOWN", 3);
private String status;
private Integer code;

1
launchers/chat/src/main/resources/db/chat-schema-h2.sql
View File

@@ -118,6 +118,7 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`description` varchar(255) ,
`command`LONGVARCHAR NOT NULL , -- task Request Parameters
`command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
`dimension_ids` varchar(500) ,
`status` INT NOT NULL , -- the final status of the task
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,

5
launchers/standalone/pom.xml
View File

@@ -28,6 +28,11 @@
<artifactId>chat-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>chat-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>

1
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -400,6 +400,7 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`command`LONGVARCHAR NOT NULL , -- task Request Parameters
`command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
`status` INT NOT NULL , -- the final status of the task
`dimension_ids` varchar(500) NULL ,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
`progress` DOUBLE default 0.00 , -- task real-time progress

1
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -183,6 +183,7 @@ CREATE TABLE `s2_dictionary_task` (
`description` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '任务描述',
`command` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数',
`command_md5` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数',
`dimension_ids` mediumtext NULL COMMENT '本次执行维度列表',
`status` int(10) NOT NULL COMMENT '任务最终运行状态',
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`created_by` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '创建人',