[knowledge](improve) add knowledge base dimension value task manage (#88)

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DictTaskFilter.java

@@ -1,15 +0,0 @@
-package com.tencent.supersonic.knowledge.dictionary;
-
-
-public class DictTaskFilter {
-
-    private Long id;
-
-    private String name;
-
-    private String createdBy;
-
-    private String createdAt;
-
-    private Integer status;
-}

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/DimValueDictInfo.java

@@ -2,7 +2,10 @@ package com.tencent.supersonic.knowledge.dictionary;
 
 
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+
 import java.util.Date;
+import java.util.Set;
+
 import lombok.Data;
 
 @Data
@@ -23,4 +26,6 @@ public class DimValueDictInfo {
     private Date createdAt;
 
     private Long elapsedMs;
+
+    private Set<Long> dimIds;
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/LocalFileHandler.java

@@ -10,6 +10,7 @@ import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.util.List;
+
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
@@ -37,7 +38,7 @@ public class LocalFileHandler implements FileHandler {
         Path targetPath = Paths.get(target);
         try {
             Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
-            log.info("File copied successfully!");
+            log.info("backupFile successfully! path:{}", targetPath.toAbsolutePath());
         } catch (IOException e) {
             log.info("Failed to copy file: " + e.getMessage());
         }
@@ -62,7 +63,7 @@ public class LocalFileHandler implements FileHandler {
             Files.delete(path);
             log.info("File:{} deleted successfully!", getAbsolutePath(filePath));
         } catch (IOException e) {
-            log.info("Failed to delete file:{}, e:", getAbsolutePath(filePath), e);
+            log.warn("Failed to delete file:{}, e:", getAbsolutePath(filePath), e);
         }
     }
 

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/listener/ApplicationStartedListener.java

@@ -13,6 +13,7 @@ import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 
 @Slf4j
 @Component
@@ -47,6 +48,14 @@ public class ApplicationStartedListener implements ApplicationListener<Applicati
         return isOk;
     }
 
+    public Boolean updateKnowledgeDimValueAsync() {
+        CompletableFuture.supplyAsync(() -> {
+            updateKnowledgeDimValue();
+            return null;
+        });
+        return true;
+    }
+
     /***
      * reload knowledge task
      */

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/dataobject/DictTaskDO.java

@@ -1,6 +1,7 @@
 package com.tencent.supersonic.knowledge.persistence.dataobject;
 
 import java.util.Date;
+
 import lombok.Data;
 import lombok.ToString;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -19,6 +20,8 @@ public class DictTaskDO {
 
     private String commandMd5;
 
+    private String dimIds;
+
     private Integer status;
 
     private String createdBy;

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/mapper/DictTaskMapper.java

@@ -1,8 +1,10 @@
 package com.tencent.supersonic.knowledge.persistence.mapper;
 
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
+
 import java.util.List;
+
 import org.apache.ibatis.annotations.Mapper;
 
 @Mapper
@@ -12,5 +14,5 @@ public interface DictTaskMapper {
 
     Boolean updateTaskStatus(DictTaskDO dictTaskDO);
 
-    List<DictTaskDO> searchDictTaskList(DictTaskFilter filter);
+    List<DictTaskDO> searchDictTaskList(DictTaskFilterReq filter);
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepository.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.knowledge.persistence.repository;
 
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
 import com.tencent.supersonic.knowledge.dictionary.DictConfig;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
 import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
 
 import java.util.List;
@@ -14,7 +14,7 @@ public interface DictRepository {
 
     Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO);
 
-    List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter);
+    List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter);
 
     DictConfig getDictInfoByModelId(Long modelId);
 }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/persistence/repository/DictRepositoryImpl.java

@@ -1,27 +1,32 @@
 package com.tencent.supersonic.knowledge.persistence.repository;
 
 import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
+import com.tencent.supersonic.common.util.JsonUtil;
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictTaskDO;
 import com.tencent.supersonic.knowledge.utils.DictTaskConverter;
 import com.tencent.supersonic.knowledge.persistence.dataobject.DictConfDO;
 import com.tencent.supersonic.knowledge.dictionary.DictConfig;
-import com.tencent.supersonic.knowledge.dictionary.DictTaskFilter;
+import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
 import com.tencent.supersonic.knowledge.dictionary.DimValueDictInfo;
 import com.tencent.supersonic.knowledge.persistence.mapper.DictConfMapper;
 import com.tencent.supersonic.knowledge.persistence.mapper.DictTaskMapper;
 
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
 
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.CollectionUtils;
 
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+import java.util.Objects;
+
 
 @Repository
+@Slf4j
 public class DictRepositoryImpl implements DictRepository {
 
     private final DictTaskMapper dictTaskMapper;
@@ -46,22 +51,24 @@ public class DictRepositoryImpl implements DictRepository {
         Date createdAt = dictTaskDO.getCreatedAt();
         long elapsedMs = System.currentTimeMillis() - createdAt.getTime();
         dictTaskDO.setElapsedMs(elapsedMs);
-        CompletableFuture.supplyAsync(() -> {
-            dictTaskMapper.updateTaskStatus(dictTaskDO);
-            return null;
-        });
+        dictTaskMapper.updateTaskStatus(dictTaskDO);
         return true;
     }
 
     @Override
-    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilter filter) {
+    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter) {
         List<DimValueDictInfo> dimValueDictDescList = new ArrayList<>();
+        log.info("filter:{}", filter);
         List<DictTaskDO> dictTaskDOList = dictTaskMapper.searchDictTaskList(filter);
         if (!CollectionUtils.isEmpty(dictTaskDOList)) {
-            dictTaskDOList.stream().forEach(dictTaskPO -> {
+            dictTaskDOList.stream().forEach(dictTaskDO -> {
                 DimValueDictInfo dimValueDictDesc = new DimValueDictInfo();
-                BeanUtils.copyProperties(dictTaskPO, dimValueDictDesc);
-                dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskPO.getStatus()));
+                BeanUtils.copyProperties(dictTaskDO, dimValueDictDesc);
+                dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskDO.getStatus()));
+                if (StringUtils.isNotEmpty(dictTaskDO.getDimIds())) {
+                    Set<Long> dimIds = JsonUtil.toSet(dictTaskDO.getDimIds(), Long.class);
+                    dimValueDictDesc.setDimIds(dimIds);
+                }
                 dimValueDictDescList.add(dimValueDictDesc);
             });
         }

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/DictTaskConverter.java

@@ -19,7 +19,7 @@ public class DictTaskConverter {
 
     private static String dateTimeFormatter = "yyyyMMddHHmmss";
 
-    public static DictTaskDO generateDimValueDictTaskPO(DimValue2DictCommand dimValue2DictCommend, User user) {
+    public static DictTaskDO generateDimValueDictTaskDO(DimValue2DictCommand dimValue2DictCommend, User user) {
         DictTaskDO taskPO = new DictTaskDO();
         Date createAt = new Date();
         String date = DateTimeFormatter.ofPattern(dateTimeFormatter)
@@ -31,7 +31,7 @@ public class DictTaskConverter {
 
         taskPO.setCreatedAt(createAt);
         taskPO.setCommand(JsonUtil.toString(dimValue2DictCommend));
-        taskPO.setStatus(TaskStatusEnum.RUNNING.getCode());
+        taskPO.setStatus(TaskStatusEnum.PENDING.getCode());
         taskPO.setCreatedBy(creator);
 
         return taskPO;

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/utils/FileHelper.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.knowledge.utils;
 import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;
 
 import com.hankcs.hanlp.dictionary.DynamicCustomDictionary;
+
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -25,7 +26,7 @@ public class FileHelper {
         for (File file : customSubFiles) {
             try {
                 file.delete();
-                log.info("customPath:{},delete cache file:{}", customPath, file);
+                log.info("customPath:{},delete file:{}", customPath, file);
             } catch (Exception e) {
                 log.error("delete " + file, e);
             }
@@ -70,7 +71,7 @@ public class FileHelper {
             }
         }
 
-        log.info("CustomDictionaryPath:{}", fileList);
+        log.debug("CustomDictionaryPath:{}", fileList);
         CustomDictionaryPath = fileList.toArray(new String[0]);
         customDictionary.path = (CustomDictionaryPath == null || CustomDictionaryPath.length == 0) ? path
                 : CustomDictionaryPath;

chat/knowledge/src/main/resources/mapper/DictTaskMapper.xml

@@ -11,6 +11,7 @@
         <result column="description" property="description"/>
         <result column="command" property="command"/>
         <result column="command_md5" property="commandMd5"/>
+        <result column="dimension_ids" property="dimIds"/>
         <result column="status" property="status"/>
         <result column="created_by" property="createdBy"/>
         <result column="created_at" property="createdAt"/>
@@ -18,11 +19,11 @@
         <result column="elapsed_ms" property="elapsedMs"/>
     </resultMap>
 
-    <insert id="createDimValueTask">
+    <insert id="createDimValueTask" useGeneratedKeys="true" keyProperty="id">
         insert into s2_dictionary_task
-        (`name`, description, command, command_md5, status, created_by, progress, elapsed_ms)
+        (`name`, description, command, command_md5, dimension_ids, status, created_by, progress, elapsed_ms)
         values
-        (#{name}, #{description}, #{command}, #{commandMd5}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
+        (#{name}, #{description}, #{command}, #{commandMd5}, #{dimIds}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
     </insert>
 
     <update id="updateTaskStatus">
@@ -34,6 +35,9 @@
             <if test="status != null">
                 status = #{status},
             </if>
+            <if test="dimIds != null">
+                dimension_ids = #{dimIds},
+            </if>
             <if test="progress != null">
                 progress = #{progress},
             </if>
@@ -42,8 +46,7 @@
             </if>
 
         </set>
-        where name = #{name}
-        and status = 0
+        where id = #{id}
     </update>
 
     <select id="searchDictTaskList" resultMap="DimValueDictTaskPO">
@@ -51,10 +54,10 @@
         from s2_dictionary_task
         <where>
             <if test="id != null and id != ''">
-                and id >= #{id}
+                and id = #{id}
             </if>
             <if test="name != null and name !=''">
-                and `name` like  "%"#{name}"%"
+                and `name` like  CONCAT('%', #{name}, '%')
             </if>
             <if test="createdBy != null and createdBy !=''">
                 and created_by = #{createdBy}
@@ -62,10 +65,11 @@
             <if test="createdAt != null and createdAt !=''">
                 and created_at &gt;= #{createdAt}
             </if>
-            <if test="status != null and status !=''">
-                and status= #{status}
+            <if test="status != null">
+                and status= #{status.code}
             </if>
         </where>
+        order by created_at desc
     </select>
 
 </mapper>

chat/knowledge/src/main/resources/sql.ddl/s2_dictionary_task.sql

@@ -3,9 +3,11 @@ CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
   `name` varchar(255) NOT NULL COMMENT '任务名称',
   `description` varchar(255) NOT NULL COMMENT '任务描述',
   `command` mediumtext  NOT NULL COMMENT '任务请求参数',
+  `dimension_ids` mediumtext  NULL COMMENT '本次执行维度列表',
   `status` int(10) NOT NULL COMMENT '任务最终运行状态',
   `created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
   `created_by` varchar(100) NOT NULL COMMENT '创建人',
   `elapsed_ms` bigint(10) DEFAULT NULL COMMENT '任务耗时',
+  `message` mediumtext COLLATE utf8mb4_unicode_ci COMMENT '备注相关信息',
   PRIMARY KEY (`id`)
 )COMMENT='字典任务信息表'