[knowledge](improve) add knowledge base dimension value task manage (#88)

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java

@@ -0,0 +1,23 @@
+package com.tencent.supersonic.chat.api.pojo.request;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+import static java.time.LocalDate.now;
+
+@ToString
+@Data
+@NoArgsConstructor
+public class DictLatestTaskReq {
+
+    @NotNull
+    private Long modelId;
+
+    private List<Long> dimIds;
+
+    private String createdAt = now().plusDays(-4).toString();
+}

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java

@@ -0,0 +1,20 @@
+package com.tencent.supersonic.chat.api.pojo.request;
+
+import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import lombok.Data;
+import lombok.ToString;
+
+@ToString
+@Data
+public class DictTaskFilterReq {
+
+    private Long id;
+
+    private String name;
+
+    private String createdBy;
+
+    private String createdAt;
+
+    private TaskStatusEnum status;
+}

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/DictLatestTaskResp.java

@@ -0,0 +1,30 @@
+package com.tencent.supersonic.chat.api.pojo.response;
+
+import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Date;
+
+@ToString
+@Data
+public class DictLatestTaskResp {
+
+    private Long dimId;
+
+    private Long id;
+
+    private String name;
+
+    private String description;
+
+    private String command;
+
+    private TaskStatusEnum status;
+
+    private String createdBy;
+
+    private Date createdAt;
+
+    private Long elapsedMs;
+}

chat/core/src/main/java/com/tencent/supersonic/chat/rest/KnowledgeController.java

@@ -3,22 +3,24 @@ package com.tencent.supersonic.chat.rest;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
-import com.tencent.supersonic.chat.service.DictionaryService;
+import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
+import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
+import com.tencent.supersonic.chat.service.ChatKnowledgeService;
 import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
 import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
 import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
 
 import java.util.List;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import javax.validation.Valid;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 
@@ -28,13 +30,14 @@ import org.springframework.web.bind.annotation.PutMapping;
 public class KnowledgeController {
 
     @Autowired
-    private DictionaryService dictApplicationService;
+    private ChatKnowledgeService knowledgeService;
 
     @Autowired
     private ApplicationStartedListener applicationStartedListener;
 
     /**
      * addDictInfo
+     * write specific dimension values to the knowledge base
      *
      * @param dimValue2DictCommend
      */
@@ -43,20 +46,21 @@ public class KnowledgeController {
                             HttpServletRequest request,
                             HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return dictApplicationService.addDictTask(dimValue2DictCommend, user);
+        return knowledgeService.addDictTask(dimValue2DictCommend, user);
     }
 
     /**
      * deleteDictInfo
+     * remove specific dimension values from the knowledge base
      *
      * @param dimValue2DictCommend
      */
-    @DeleteMapping("/task")
+    @PostMapping("/task/delete")
     public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend,
                                HttpServletRequest request,
                                HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return dictApplicationService.deleteDictTask(dimValue2DictCommend, user);
+        return knowledgeService.deleteDictTask(dimValue2DictCommend, user);
     }
 
     /**
@@ -65,19 +69,44 @@ public class KnowledgeController {
      * @param filter
      */
     @PostMapping("/task/search")
-    public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilter filter,
+    public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilterReq filter,
                                                      HttpServletRequest request,
                                                      HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
-        return dictApplicationService.searchDictTaskList(filter, user);
+        return knowledgeService.searchDictTaskList(filter, user);
     }
 
+    /**
+     * searchDictLatestTaskList
+     */
+    @PostMapping("/task/search/latest")
+    public List<DictLatestTaskResp> searchDictLatestTaskList(@RequestBody @Valid DictLatestTaskReq filter,
+                                                             HttpServletRequest request,
+                                                             HttpServletResponse response) {
+        User user = UserHolder.findUser(request, response);
+        return knowledgeService.searchDictLatestTaskList(filter, user);
+    }
+
+    /**
+     * getDictRootPath
+     * get knowledge base file directory
+     *
+     * @return
+     */
     @GetMapping("/rootPath")
     public String getDictRootPath(HttpServletRequest request,
                                   HttpServletResponse response) {
-        return dictApplicationService.getDictRootPath();
+        return knowledgeService.getDictRootPath();
     }
 
+    /**
+     * updateDimValue
+     * update in-memory dictionary files in real time
+     *
+     * @param request
+     * @param response
+     * @return
+     */
     @PutMapping("/knowledge/dimValue")
     public Boolean updateDimValue(HttpServletRequest request,
                                   HttpServletResponse response) {

chat/core/src/main/java/com/tencent/supersonic/chat/service/ChatKnowledgeService.java

@@ -1,21 +1,25 @@
 package com.tencent.supersonic.chat.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
+import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
 import com.tencent.supersonic.knowledge.dictionary.DictConfig;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
 import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
 import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
 
 import java.util.List;
 
-public interface DictionaryService {
+public interface ChatKnowledgeService {
     Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
 
     Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
 
-    List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter, User user);
+    List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user);
 
     DictConfig getDictInfoByModelId(Long modelId);
 
     String getDictRootPath();
+
+    List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq filter, User user);
 }

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ChatKnowledgeServiceImpl.java

@@ -0,0 +1,268 @@
+package com.tencent.supersonic.chat.service.impl;
+
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
+import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
+import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
+import com.tencent.supersonic.chat.config.DefaultMetric;
+import com.tencent.supersonic.chat.config.Dim4Dict;
+import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
+import com.tencent.supersonic.chat.service.ChatKnowledgeService;
+import com.tencent.supersonic.chat.utils.DictMetaHelper;
+import com.tencent.supersonic.chat.utils.DictQueryHelper;
+import com.tencent.supersonic.common.pojo.Constants;
+import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.JsonUtil;
+import com.tencent.supersonic.knowledge.dictionary.FileHandler;
+import com.tencent.supersonic.knowledge.listener.ApplicationStartedListener;
+import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
+import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
+import com.tencent.supersonic.knowledge.dictionary.DictConfig;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
+import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
+import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
+import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
+import com.tencent.supersonic.knowledge.persistence.repository.DictRepository;
+
+
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Objects;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+
+@Slf4j
+@Service
+public class ChatKnowledgeServiceImpl implements ChatKnowledgeService {
+
+    private final DictMetaHelper metaUtils;
+    private final DictQueryHelper dictQueryHelper;
+    private final FileHandler fileHandler;
+    private final DictRepository dictRepository;
+    private final ApplicationStartedListener applicationStartedListener;
+
+    @Value("${dict.flush.enable:true}")
+    private Boolean dictFlushEnable;
+    @Value("${dict.flush.daily.enable:true}")
+    private Boolean dictFlushDailyEnable;
+    @Value("${dict.file.type:txt}")
+    private String dictFileType;
+    private String dimValue = "DimValue_%d_%d";
+
+    public ChatKnowledgeServiceImpl(DictMetaHelper metaUtils,
+                                    DictQueryHelper dictQueryHelper,
+                                    FileHandler fileHandler,
+                                    DictRepository dictRepository,
+                                    ApplicationStartedListener applicationStartedListener) {
+        this.metaUtils = metaUtils;
+        this.dictQueryHelper = dictQueryHelper;
+        this.fileHandler = fileHandler;
+        this.dictRepository = dictRepository;
+        this.applicationStartedListener = applicationStartedListener;
+    }
+
+    @Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}")
+    public Boolean dailyDictTask() {
+        log.info("[dailyDictTask] start");
+        if (!dictFlushDailyEnable) {
+            log.info("dictFlushDailyEnable is false, now finish dailyDictTask");
+        }
+        DimValue2DictCommand dimValue2DictCommend = new DimValue2DictCommand();
+        dimValue2DictCommend.setUpdateMode(DictUpdateMode.OFFLINE_FULL);
+
+        User user = User.getFakeUser();
+        addDictTask(dimValue2DictCommend, user);
+        log.info("[dailyDictTask] finish");
+        return true;
+    }
+
+    @Override
+    public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
+        if (!dictFlushEnable) {
+            return 0L;
+        }
+
+        if (DictUpdateMode.REALTIME_DELETE.equals(dimValue2DictCommend.getUpdateMode())) {
+            return deleteDictTask(dimValue2DictCommend, user);
+        }
+
+        DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommend, user);
+        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
+        // todo check dimension can not be searched
+
+        dictRepository.createDimValueDictTask(dictTaskDO);
+        runDictTask(dictTaskDO, user);
+
+        return dictTaskDO.getId();
+    }
+
+    public Long runDictTask(DictTaskDO dictTaskDO, User user) {
+        if (Objects.isNull(dictTaskDO)) {
+            return -1L;
+        }
+        DimValue2DictCommand command = JsonUtil.toObject(dictTaskDO.getCommand(), DimValue2DictCommand.class);
+        try {
+            //1. construct internal dictionary requirements
+            List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(command);
+            Set<Long> dimIds = generateDimSet(dimValueDOList);
+            dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
+            dictRepository.updateDictTaskStatus(TaskStatusEnum.RUNNING.getCode(), dictTaskDO);
+            log.debug("dimValueDOList:{}", dimValueDOList);
+            //2. query dimension value information
+            for (DimValueDO dimValueDO : dimValueDOList) {
+                Long modelId = dimValueDO.getModelId();
+                DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
+                for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
+                    List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
+                    //3. local file changes
+                    String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
+                            dim4Dict.getDimId());
+                    fileHandler.writeFile(data, fileName, false);
+                }
+            }
+            applicationStartedListener.updateKnowledgeDimValue();
+            log.debug("updateDictTaskStatus to SUCCESS");
+            dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
+        } catch (Exception e) {
+            log.warn("addDictInfo exception:", e);
+            dictRepository.updateDictTaskStatus(TaskStatusEnum.ERROR.getCode(), dictTaskDO);
+        }
+        return 1L;
+    }
+
+    private Set<Long> generateDimSet(List<DimValueDO> dimValueDOList) {
+        Set<Long> dimIds = new HashSet<>();
+        if (!CollectionUtils.isEmpty(dimValueDOList)) {
+            dimValueDOList.stream().forEach(dimValueDO -> {
+                if (!CollectionUtils.isEmpty(dimValueDO.getDimensions())) {
+                    dimValueDO.getDimensions().stream().forEach(dim4Dict -> dimIds.add(dim4Dict.getDimId()));
+                }
+            });
+        }
+        return dimIds;
+    }
+
+    @Override
+    public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommand, User user) {
+        if (!dictFlushEnable) {
+            return 0L;
+        }
+        if (Objects.isNull(dimValue2DictCommand) || !DictUpdateMode.REALTIME_DELETE.equals(
+                dimValue2DictCommand.getUpdateMode())) {
+            throw new RuntimeException("illegal parameter");
+        }
+
+        DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommand, user);
+        log.info("[deleteDictTask] dictTaskDO:{}", dictTaskDO);
+        Set<Long> dimIds = generateDimSetFromCommand(dimValue2DictCommand.getModelAndDimPair());
+        dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
+        dictRepository.createDimValueDictTask(dictTaskDO);
+
+        Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommand.getModelAndDimPair();
+        if (CollectionUtils.isEmpty(modelAndDimPair)) {
+            return 0L;
+        }
+        for (Long modelId : modelAndDimPair.keySet()) {
+            if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
+                continue;
+            }
+            for (Long dimId : modelAndDimPair.get(modelId)) {
+                String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
+                fileHandler.deleteDictFile(fileName);
+            }
+        }
+        applicationStartedListener.updateKnowledgeDimValue();
+        dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
+        applicationStartedListener.updateKnowledgeDimValue();
+
+        return 1L;
+    }
+
+    private Set<Long> generateDimSetFromCommand(Map<Long, List<Long>> modelAndDimPair) {
+        Set<Long> dimIds = new HashSet<>();
+        if (!CollectionUtils.isEmpty(modelAndDimPair)) {
+            modelAndDimPair.forEach((k, v) -> dimIds.addAll(v));
+        }
+        return dimIds;
+    }
+
+    @Override
+    public String getDictRootPath() {
+        return fileHandler.getDictRootPath();
+    }
+
+    @Override
+    public List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq latestFilter, User user) {
+        DictTaskFilterReq filter = new DictTaskFilterReq();
+        BeanUtils.copyProperties(latestFilter, filter);
+        List<DimValueDictInfo> dimValueDictInfoList = searchDictTaskList(filter, user);
+        return extractLatestTask(dimValueDictInfoList, latestFilter.getDimIds());
+    }
+
+    private List<DictLatestTaskResp> extractLatestTask(List<DimValueDictInfo> dimValueDictInfoList, List<Long> dimIds) {
+        List<DictLatestTaskResp> dictLatestTaskRespList = new ArrayList<>();
+        Map<Long, DictLatestTaskResp> dimAndTaskPair = new HashMap<>(50);
+        for (DimValueDictInfo dimValueDictInfo : dimValueDictInfoList) {
+            //1. filter
+            if (Objects.isNull(dimValueDictInfo) || CollectionUtils.isEmpty(dimValueDictInfo.getDimIds())) {
+                continue;
+            }
+            if (!CollectionUtils.isEmpty(dimIds)) {
+                Set<Long> tmp = dimValueDictInfo.getDimIds();
+                tmp.retainAll(dimIds);
+                dimValueDictInfo.setDimIds(tmp);
+                if (tmp.size() <= 0) {
+                    continue;
+                }
+            }
+
+            // 2. extract
+            Set<Long> dimIdList = dimValueDictInfo.getDimIds();
+            for (Long dimId : dimIdList) {
+                DictLatestTaskResp dictLatestTaskResp = new DictLatestTaskResp();
+                if (!dimAndTaskPair.containsKey(dimId)) {
+                    BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
+                    dictLatestTaskResp.setDimId(dimId);
+                } else {
+                    DictLatestTaskResp dictLatestTaskExist = dimAndTaskPair.get(dimId);
+                    if (dictLatestTaskExist.getCreatedAt().before(dimValueDictInfo.getCreatedAt())) {
+                        BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
+                        dictLatestTaskResp.setDimId(dimId);
+                    } else {
+                        dictLatestTaskResp = dictLatestTaskExist;
+                    }
+                }
+                dimAndTaskPair.put(dimId, dictLatestTaskResp);
+            }
+
+        }
+
+        if (dimAndTaskPair.size() >= 0 && !CollectionUtils.isEmpty(dimAndTaskPair.values())) {
+            dimAndTaskPair.values().stream()
+                    .filter(v -> !v.getCommand().contains(DictUpdateMode.REALTIME_DELETE.name()))
+                    .forEach(v -> dictLatestTaskRespList.add(v));
+        }
+
+
+        return dictLatestTaskRespList;
+    }
+
+    @Override
+    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user) {
+        return dictRepository.searchDictTaskList(filter);
+    }
+
+    @Override
+    public DictConfig getDictInfoByModelId(Long modelId) {
+        return dictRepository.getDictInfoByModelId(modelId);
+    }
+}

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/DictionaryServiceImpl.java

@@ -1,124 +0,0 @@
-package com.tencent.supersonic.chat.service.impl;
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.config.DefaultMetric;
-import com.tencent.supersonic.chat.config.Dim4Dict;
-import com.tencent.supersonic.chat.persistence.dataobject.DimValueDO;
-import com.tencent.supersonic.chat.service.DictionaryService;
-import com.tencent.supersonic.chat.utils.DictMetaHelper;
-import com.tencent.supersonic.chat.utils.DictQueryHelper;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.knowledge.dictionary.FileHandler;
-import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
-import com.tencent.supersonic.knowledge.dictionary.DictConfig;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
-import com.tencent.supersonic.knowledge.dictionary.DictUpdateMode;
-import com.tencent.supersonic.knowledge.dictionary.DimValue2DictCommand;
-import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
-import com.tencent.supersonic.knowledge.persistence.repository.DictRepository;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
-
-@Slf4j
-@Service
-public class DictionaryServiceImpl implements DictionaryService {
-
-    private final DictMetaHelper metaUtils;
-    private final DictQueryHelper dictQueryHelper;
-    private final FileHandler fileHandler;
-    private final DictRepository dictRepository;
-    @Value("${dict.flush.enable:true}")
-    private Boolean dictFlushEnable;
-    @Value("${dict.file.type:txt}")
-    private String dictFileType;
-    private String dimValue = "DimValue_%d_%d";
-
-    public DictionaryServiceImpl(DictMetaHelper metaUtils,
-                                 DictQueryHelper dictQueryHelper,
-                                 FileHandler fileHandler,
-                                 DictRepository dictRepository) {
-        this.metaUtils = metaUtils;
-        this.dictQueryHelper = dictQueryHelper;
-        this.fileHandler = fileHandler;
-        this.dictRepository = dictRepository;
-    }
-
-    public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
-        if (!dictFlushEnable) {
-            return 0L;
-        }
-        DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskPO(dimValue2DictCommend,
-                user);
-        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
-        dictRepository.createDimValueDictTask(dictTaskDO);
-        TaskStatusEnum finalStatus = TaskStatusEnum.SUCCESS;
-        try {
-            //1. construct internal dictionary requirements
-            List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(dimValue2DictCommend);
-            log.info("dimValueDOList:{}", dimValueDOList);
-            //2. query dimension value information
-            for (DimValueDO dimValueDO : dimValueDOList) {
-                Long modelId = dimValueDO.getModelId();
-                DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
-                for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
-                    List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
-                    //3. local file changes
-                    String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
-                            dim4Dict.getDimId());
-                    fileHandler.writeFile(data, fileName, false);
-                }
-            }
-        } catch (Exception e) {
-            log.warn("addDictInfo exception:", e);
-            finalStatus = TaskStatusEnum.ERROR;
-        }
-        dictRepository.updateDictTaskStatus(finalStatus.getCode(),
-                dictTaskDO);
-        return 1L;
-    }
-
-
-    public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
-        if (!dictFlushEnable) {
-            return 0L;
-        }
-        if (Objects.isNull(dimValue2DictCommend) || DictUpdateMode.REALTIME_DELETE.equals(
-                dimValue2DictCommend.getUpdateMode())) {
-            throw new RuntimeException("illegal parameter");
-        }
-        Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommend.getModelAndDimPair();
-        if (CollectionUtils.isEmpty(modelAndDimPair)) {
-            return 0L;
-        }
-        for (Long modelId : modelAndDimPair.keySet()) {
-            if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
-                continue;
-            }
-            for (Long dimId : modelAndDimPair.get(modelId)) {
-                String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
-                fileHandler.deleteDictFile(fileName);
-            }
-        }
-
-        return 1L;
-    }
-
-    public String getDictRootPath() {
-        return fileHandler.getDictRootPath();
-    }
-
-    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter, User user) {
-        return dictRepository.searchDictTaskList(filter);
-    }
-
-    public DictConfig getDictInfoByModelId(Long modelId) {
-        return dictRepository.getDictInfoByModelId(modelId);
-    }
-}

chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictMetaHelper.java

@@ -40,6 +40,8 @@ public class DictMetaHelper {
     private ConfigService configService;
     @Value("${model.internal.metric.suffix:internal_cnt}")
     private String internalMetricNameSuffix;
+    @Value("${model.internal.day.number:2}")
+    private Integer internalMetricDays;
     private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
 
     public List<DimValueDO> generateDimValueInfo(DimValue2DictCommand dimValue2DictCommend) {
@@ -134,14 +136,21 @@ public class DictMetaHelper {
 
             ChatDefaultRichConfigResp chatDefaultConfig =
                     chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
+
+            KnowledgeAdvancedConfig globalKnowledgeConfigAgg = chaConfigRichDesc.getChatAggRichConfig()
+                    .getGlobalKnowledgeConfig();
             List<KnowledgeInfoReq> knowledgeAggInfo =
                     chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos();
 
+            KnowledgeAdvancedConfig globalKnowledgeConfigDetail = chaConfigRichDesc.getChatDetailRichConfig()
+                    .getGlobalKnowledgeConfig();
             List<KnowledgeInfoReq> knowledgeDetailInfo =
                     chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos();
 
-            fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
+            fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair,
-            fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
+                    modelId, globalKnowledgeConfigDetail);
+            fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair,
+                    modelId, globalKnowledgeConfigAgg);
 
 
         }
@@ -150,7 +159,8 @@ public class DictMetaHelper {
     private void fillKnowledgeDimValue(List<KnowledgeInfoReq> knowledgeInfos,
                                        ChatDefaultRichConfigResp chatDefaultConfig,
                                        List<DimValueDO> dimValueDOList,
-                                       Map<Long, SchemaElement> dimIdAndDescPair, Long modelId) {
+                                       Map<Long, SchemaElement> dimIdAndDescPair, Long modelId,
+                                       KnowledgeAdvancedConfig globalKnowledgeConfigDetail) {
         if (!CollectionUtils.isEmpty(knowledgeInfos)) {
             List<Dim4Dict> dimensions = new ArrayList<>();
             List<DefaultMetric> defaultMetricDescList = new ArrayList<>();
@@ -159,7 +169,6 @@ public class DictMetaHelper {
                             && !CollectionUtils.isEmpty(dimIdAndDescPair)
                             && dimIdAndDescPair.containsKey(knowledgeInfo.getItemId()))
                     .forEach(knowledgeInfo -> {
-                        if (dimIdAndDescPair.containsKey(knowledgeInfo.getItemId())) {
                         SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId());
 
                         //default cnt
@@ -169,7 +178,8 @@ public class DictMetaHelper {
                             if (Strings.isNotEmpty(datasourceBizName)) {
                                 String internalMetricName =
                                         datasourceBizName + UNDERLINE + internalMetricNameSuffix;
-                                    defaultMetricDescList.add(new DefaultMetric(internalMetricName, 2, DAY));
+                                defaultMetricDescList.add(new DefaultMetric(internalMetricName,
+                                        internalMetricDays, DAY));
                             }
                         } else {
                             SchemaElement schemaItem = chatDefaultConfig.getMetrics().get(0);
@@ -186,9 +196,14 @@ public class DictMetaHelper {
                             KnowledgeAdvancedConfig knowledgeAdvancedConfig
                                     = knowledgeInfo.getKnowledgeAdvancedConfig();
                             BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict);
+
+                            if (Objects.nonNull(globalKnowledgeConfigDetail)
+                                    && !CollectionUtils.isEmpty(globalKnowledgeConfigDetail.getRuleList())) {
+                                dim4Dict.getRuleList().addAll(globalKnowledgeConfigDetail.getRuleList());
+                            }
                         }
                         dimensions.add(dim4Dict);
-                        }
+
                     });
 
             if (!CollectionUtils.isEmpty(dimensions)) {

chat/core/src/main/java/com/tencent/supersonic/chat/utils/DictQueryHelper.java

@@ -1,10 +1,5 @@
 package com.tencent.supersonic.chat.utils;
 
-import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
-import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
-import static com.tencent.supersonic.common.pojo.Constants.COMMA;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE;
-
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.component.SemanticLayer;
 import com.tencent.supersonic.chat.config.DefaultMetric;
@@ -34,6 +29,12 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
+import static com.tencent.supersonic.common.pojo.Constants.SPACE;
+import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
+import static com.tencent.supersonic.common.pojo.Constants.COMMA;
+import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE;
+
 @Slf4j
 @Component
 public class DictQueryHelper {
@@ -46,6 +47,8 @@ public class DictQueryHelper {
     private Integer printDataShow;
     @Value("${dimension.max.limit:3000000}")
     private Long dimMaxLimit;
+    @Value("${dimension.white.weight:60000000}")
+    private Long dimensionWhiteWeight;
 
     public List<String> fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
                                             User user) {
@@ -53,10 +56,11 @@ public class DictQueryHelper {
         QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict);
         try {
             QueryResultWithSchemaResp queryResultWithColumns = semanticLayer.queryByStruct(queryStructCmd, user);
+            log.info("fetchDimValueSingle sql:{}", queryResultWithColumns.getSql());
             String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId());
             String dimNameRewrite = rewriteDimName(queryResultWithColumns.getColumns(), dim4Dict.getBizName());
             data = generateFileData(queryResultWithColumns.getResultList(), nature, dimNameRewrite,
-                    defaultMetricDesc.getBizName());
+                    defaultMetricDesc.getBizName(), dim4Dict);
             if (!CollectionUtils.isEmpty(data)) {
                 int size = (data.size() > printDataShow) ? printDataShow : data.size();
                 log.info("data:{}", data.subList(0, size - 1));
@@ -91,7 +95,7 @@ public class DictQueryHelper {
     }
 
     private List<String> generateFileData(List<Map<String, Object>> resultList, String nature, String dimName,
-                                          String metricName) {
+                                          String metricName, Dim4Dict dim4Dict) {
         List<String> data = new ArrayList<>();
         if (CollectionUtils.isEmpty(resultList)) {
             return data;
@@ -111,17 +115,26 @@ public class DictQueryHelper {
             }
 
         }
-        constructDataLines(valueAndFrequencyPair, nature, data);
+        constructDataLines(valueAndFrequencyPair, nature, data, dim4Dict);
         return data;
     }
 
-    private void constructDataLines(Map<String, Long> valueAndFrequencyPair, String nature, List<String> data) {
+    private void constructDataLines(Map<String, Long> valueAndFrequencyPair, String nature,
+                                    List<String> data, Dim4Dict dim4Dict) {
         valueAndFrequencyPair.forEach((dimValue, metric) -> {
             if (metric > MAX_FREQUENCY) {
                 metric = MAX_FREQUENCY;
             }
+            if (Strings.isNotEmpty(dimValue) && dimValue.contains(SPACE)) {
+                dimValue = dimValue.replace(SPACE, "#");
+            }
             data.add(String.format("%s %s %s", dimValue, nature, metric));
         });
+
+        if (Objects.nonNull(dim4Dict) && !CollectionUtils.isEmpty(dim4Dict.getWhiteList())) {
+            dim4Dict.getWhiteList().stream()
+                    .forEach(white -> data.add(String.format("%s %s %s", white, nature, dimensionWhiteWeight)));
+        }
     }
 
     private void mergeMultivaluedValue(Map<String, Long> valueAndFrequencyPair, String dimValue, Long metric) {
@@ -185,7 +198,7 @@ public class DictQueryHelper {
         if (Objects.isNull(dim4Dict)) {
             return "";
         }
-        StringJoiner joiner = new StringJoiner(AND_UPPER);
+        StringJoiner joiner = new StringJoiner(SPACE + AND_UPPER + SPACE);
 
         String dimName = dim4Dict.getBizName();
         if (!CollectionUtils.isEmpty(dim4Dict.getBlackList())) {

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictTaskFilter.java

@@ -1,15 +0,0 @@
-package com.tencent.supersonic.knowledge.dictionary;
-
-
-public class DictTaskFilter {
-
-    private Long id;
-
-    private String name;
-
-    private String createdBy;
-
-    private String createdAt;
-
-    private Integer status;
-}

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DimValueDictInfo.java

@@ -2,7 +2,10 @@ package com.tencent.supersonic.knowledge.dictionary;
 
 
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+
 import java.util.Date;
+import java.util.Set;
+
 import lombok.Data;
 
 @Data
@@ -23,4 +26,6 @@ public class DimValueDictInfo {
     private Date createdAt;
 
     private Long elapsedMs;
+
+    private Set<Long> dimIds;
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/LocalFileHandler.java

@@ -10,6 +10,7 @@ import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.util.List;
+
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
@@ -37,7 +38,7 @@ public class LocalFileHandler implements FileHandler {
         Path targetPath = Paths.get(target);
         try {
             Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
-            log.info("File copied successfully!");
+            log.info("backupFile successfully! path:{}", targetPath.toAbsolutePath());
         } catch (IOException e) {
             log.info("Failed to copy file: " + e.getMessage());
         }
@@ -62,7 +63,7 @@ public class LocalFileHandler implements FileHandler {
             Files.delete(path);
             log.info("File:{} deleted successfully!", getAbsolutePath(filePath));
         } catch (IOException e) {
-            log.info("Failed to delete file:{}, e:", getAbsolutePath(filePath), e);
+            log.warn("Failed to delete file:{}, e:", getAbsolutePath(filePath), e);
         }
     }
 

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java

@@ -13,6 +13,7 @@ import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 
 @Slf4j
 @Component
@@ -47,6 +48,14 @@ public class ApplicationStartedListener implements ApplicationListener<Applicati
         return isOk;
     }
 
+    public Boolean updateKnowledgeDimValueAsync() {
+        CompletableFuture.supplyAsync(() -> {
+            updateKnowledgeDimValue();
+            return null;
+        });
+        return true;
+    }
+
     /***
      * reload knowledge task
      */

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/dataobject/DictTaskDO.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.knowledge.persistence.dataobject;
 
 import java.util.Date;
+
 import lombok.Data;
 import lombok.ToString;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -19,6 +20,8 @@ public class DictTaskDO {
 
     private String commandMd5;
 
+    private String dimIds;
+
     private Integer status;
 
     private String createdBy;

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/mapper/DictTaskMapper.java

@@ -1,8 +1,10 @@
 package com.tencent.supersonic.knowledge.persistence.mapper;
 
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
+
 import java.util.List;
+
 import org.apache.ibatis.annotations.Mapper;
 
 @Mapper
@@ -12,5 +14,5 @@ public interface DictTaskMapper {
 
     Boolean updateTaskStatus(DictTaskDO dictTaskDO);
 
-    List<DictTaskDO> searchDictTaskList(DictTaskFilter filter);
+    List<DictTaskDO> searchDictTaskList(DictTaskFilterReq filter);
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepository.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.knowledge.persistence.repository;
 
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
 import com.tencent.supersonic.knowledge.dictionary.DictConfig;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
 import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
 
 import java.util.List;
@@ -14,7 +14,7 @@ public interface DictRepository {
 
     Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO);
 
-    List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter);
+    List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter);
 
     DictConfig getDictInfoByModelId(Long modelId);
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepositoryImpl.java

@@ -1,27 +1,32 @@
 package com.tencent.supersonic.knowledge.persistence.repository;
 
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
 import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO;
 import com.tencent.supersonic.knowledge.dictionary.DictConfig;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
 import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
 import com.tencent.supersonic.knowledge.persistence.mapper.DictConfMapper;
 import com.tencent.supersonic.knowledge.persistence.mapper.DictTaskMapper;
 
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
 
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.CollectionUtils;
 
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+import java.util.Objects;
+
 
 @Repository
+@Slf4j
 public class DictRepositoryImpl implements DictRepository {
 
     private final DictTaskMapper dictTaskMapper;
@@ -46,22 +51,24 @@ public class DictRepositoryImpl implements DictRepository {
         Date createdAt = dictTaskDO.getCreatedAt();
         long elapsedMs = System.currentTimeMillis() - createdAt.getTime();
         dictTaskDO.setElapsedMs(elapsedMs);
-        CompletableFuture.supplyAsync(() -> {
         dictTaskMapper.updateTaskStatus(dictTaskDO);
-            return null;
-        });
         return true;
     }
 
     @Override
-    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter) {
+    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter) {
         List<DimValueDictInfo> dimValueDictDescList = new ArrayList<>();
+        log.info("filter:{}", filter);
         List<DictTaskDO> dictTaskDOList = dictTaskMapper.searchDictTaskList(filter);
         if (!CollectionUtils.isEmpty(dictTaskDOList)) {
-            dictTaskDOList.stream().forEach(dictTaskPO -> {
+            dictTaskDOList.stream().forEach(dictTaskDO -> {
                 DimValueDictInfo dimValueDictDesc = new DimValueDictInfo();
-                BeanUtils.copyProperties(dictTaskPO, dimValueDictDesc);
+                BeanUtils.copyProperties(dictTaskDO, dimValueDictDesc);
-                dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskPO.getStatus()));
+                dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskDO.getStatus()));
+                if (StringUtils.isNotEmpty(dictTaskDO.getDimIds())) {
+                    Set<Long> dimIds = JsonUtil.toSet(dictTaskDO.getDimIds(), Long.class);
+                    dimValueDictDesc.setDimIds(dimIds);
+                }
                 dimValueDictDescList.add(dimValueDictDesc);
             });
         }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/DictTaskConverter.java

@@ -19,7 +19,7 @@ public class DictTaskConverter {
 
     private static String dateTimeFormatter = "yyyyMMddHHmmss";
 
-    public static DictTaskDO generateDimValueDictTaskPO(DimValue2DictCommand dimValue2DictCommend, User user) {
+    public static DictTaskDO generateDimValueDictTaskDO(DimValue2DictCommand dimValue2DictCommend, User user) {
         DictTaskDO taskPO = new DictTaskDO();
         Date createAt = new Date();
         String date = DateTimeFormatter.ofPattern(dateTimeFormatter)
@@ -31,7 +31,7 @@ public class DictTaskConverter {
 
         taskPO.setCreatedAt(createAt);
         taskPO.setCommand(JsonUtil.toString(dimValue2DictCommend));
-        taskPO.setStatus(TaskStatusEnum.RUNNING.getCode());
+        taskPO.setStatus(TaskStatusEnum.PENDING.getCode());
         taskPO.setCreatedBy(creator);
 
         return taskPO;

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/FileHelper.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.knowledge.utils;
 import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;
 
 import com.hankcs.hanlp.dictionary.DynamicCustomDictionary;
+
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -25,7 +26,7 @@ public class FileHelper {
         for (File file : customSubFiles) {
             try {
                 file.delete();
-                log.info("customPath:{},delete cache file:{}", customPath, file);
+                log.info("customPath:{},delete file:{}", customPath, file);
             } catch (Exception e) {
                 log.error("delete " + file, e);
             }
@@ -70,7 +71,7 @@ public class FileHelper {
             }
         }
 
-        log.info("CustomDictionaryPath:{}", fileList);
+        log.debug("CustomDictionaryPath:{}", fileList);
         CustomDictionaryPath = fileList.toArray(new String[0]);
         customDictionary.path = (CustomDictionaryPath == null || CustomDictionaryPath.length == 0) ? path
                 : CustomDictionaryPath;

chat/knowledge/src/main/resources/mapper/DictTaskMapper.xml

@@ -11,6 +11,7 @@
         <result column="description" property="description"/>
         <result column="command" property="command"/>
         <result column="command_md5" property="commandMd5"/>
+        <result column="dimension_ids" property="dimIds"/>
         <result column="status" property="status"/>
         <result column="created_by" property="createdBy"/>
         <result column="created_at" property="createdAt"/>
@@ -18,11 +19,11 @@
         <result column="elapsed_ms" property="elapsedMs"/>
     </resultMap>
 
-    <insert id="createDimValueTask">
+    <insert id="createDimValueTask" useGeneratedKeys="true" keyProperty="id">
         insert into s2_dictionary_task
-        (`name`, description, command, command_md5, status, created_by, progress, elapsed_ms)
+        (`name`, description, command, command_md5, dimension_ids, status, created_by, progress, elapsed_ms)
         values
-        (#{name}, #{description}, #{command}, #{commandMd5}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
+        (#{name}, #{description}, #{command}, #{commandMd5}, #{dimIds}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
     </insert>
 
     <update id="updateTaskStatus">
@@ -34,6 +35,9 @@
             <if test="status != null">
                 status = #{status},
             </if>
+            <if test="dimIds != null">
+                dimension_ids = #{dimIds},
+            </if>
             <if test="progress != null">
                 progress = #{progress},
             </if>
@@ -42,8 +46,7 @@
             </if>
 
         </set>
-        where name = #{name}
+        where id = #{id}
-        and status = 0
     </update>
 
     <select id="searchDictTaskList" resultMap="DimValueDictTaskPO">
@@ -51,10 +54,10 @@
         from s2_dictionary_task
         <where>
             <if test="id != null and id != ''">
-                and id >= #{id}
+                and id = #{id}
             </if>
             <if test="name != null and name !=''">
-                and `name` like  "%"#{name}"%"
+                and `name` like  CONCAT('%', #{name}, '%')
             </if>
             <if test="createdBy != null and createdBy !=''">
                 and created_by = #{createdBy}
@@ -62,10 +65,11 @@
             <if test="createdAt != null and createdAt !=''">
                 and created_at &gt;= #{createdAt}
             </if>
-            <if test="status != null and status !=''">
+            <if test="status != null">
-                and status= #{status}
+                and status= #{status.code}
             </if>
         </where>
+        order by created_at desc
     </select>
 
 </mapper>

chat/knowledge/src/main/resources/sql.ddl/s2_dictionary_task.sql

@@ -3,9 +3,11 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
   `name` varchar(255) NOT NULL COMMENT '任务名称',
   `description` varchar(255) NOT NULL COMMENT '任务描述',
   `command` mediumtext  NOT NULL COMMENT '任务请求参数',
+  `dimension_ids` mediumtext  NULL COMMENT '本次执行维度列表',
   `status` int(10) NOT NULL COMMENT '任务最终运行状态',
   `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
   `created_by` varchar(100) NOT NULL COMMENT '创建人',
   `elapsed_ms` bigint(10) DEFAULT NULL COMMENT '任务耗时',
+  `message` mediumtext COLLATE utf8mb4_unicode_ci COMMENT '备注相关信息',
   PRIMARY KEY (`id`)
 )COMMENT='字典任务信息表'

common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java

@@ -2,13 +2,15 @@ package com.tencent.supersonic.common.pojo.enums;
 
 public enum TaskStatusEnum {
 
-    RUNNING("running", 0),
-
-    SUCCESS("success", 1),
-
     ERROR("error", -1),
 
-    UNKNOWN("UNKNOWN", 2);
+    PENDING("pending", 0),
+
+    RUNNING("running", 1),
+
+    SUCCESS("success", 2),
+
+    UNKNOWN("UNKNOWN", 3);
 
     private String status;
     private Integer code;

launchers/chat/src/main/resources/db/chat-schema-h2.sql

@@ -118,6 +118,7 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
    `description` varchar(255) ,
    `command`LONGVARCHAR  NOT NULL , -- task Request Parameters
    `command_md5` varchar(255)  NOT NULL , -- task Request Parameters md5
+   `dimension_ids` varchar(500) ,
    `status` INT NOT NULL , -- the final status of the task
    `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP  ,
    `created_by` varchar(100) NOT NULL ,

launchers/standalone/pom.xml

@@ -28,6 +28,11 @@
             <artifactId>chat-core</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.tencent.supersonic</groupId>
+            <artifactId>chat-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
 
         <dependency>
             <groupId>com.tencent.supersonic</groupId>

launchers/standalone/src/main/resources/db/schema-h2.sql

@@ -400,6 +400,7 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
    `command`LONGVARCHAR  NOT NULL , -- task Request Parameters
    `command_md5` varchar(255)  NOT NULL , -- task Request Parameters md5
    `status` INT NOT NULL , -- the final status of the task
+   `dimension_ids` varchar(500)  NULL ,
    `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP  ,
    `created_by` varchar(100) NOT NULL ,
    `progress` DOUBLE default 0.00  ,  -- task real-time progress

launchers/standalone/src/main/resources/db/schema-mysql.sql

@@ -183,6 +183,7 @@ CREATE TABLE `s2_dictionary_task` (
                                       `description` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '任务描述',
                                       `command` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数',
                                       `command_md5` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数',
+                                      `dimension_ids` mediumtext  NULL COMMENT '本次执行维度列表',
                                       `status` int(10) NOT NULL COMMENT '任务最终运行状态',
                                       `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                                       `created_by` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '创建人',