Dict headless opt (#764)

* [improvement](headless) Enhance dictionary execution function,support clear dict data recording

* [fix](headless) fix dict value support space

---------

Co-authored-by: kanedai <kanedai@tencent.com>

common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java

@@ -11,6 +11,7 @@ public class Constants {
     public static final String AT_SYMBOL = "@";
     public static final String DOT = ".";
     public static String SPACE = " ";
+    public static String POUND = "#";
     public static final String COLON = ":";
     public static final String MINUS = "-";
     public static final String UNDERLINE = "_";

common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java

@@ -2,6 +2,8 @@ package com.tencent.supersonic.common.pojo.enums;
 
 public enum TaskStatusEnum {
 
+    INITIAL("initial", -2),
+
     ERROR("error", -1),
 
     PENDING("pending", 0),
@@ -10,7 +12,7 @@ public enum TaskStatusEnum {
 
     SUCCESS("success", 2),
 
-    UNKNOWN("UNKNOWN", 3);
+    UNKNOWN("unknown", 3);
 
     private String status;
     private Integer code;

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java

@@ -5,14 +5,11 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import lombok.Builder;
 import lombok.Data;
 
-import javax.validation.constraints.NotNull;
-
 @Data
 @Builder
 public class DictItemFilter {
     private Long id;
     private TypeEnums type;
     private Long itemId;
-    @NotNull
     private StatusEnum status;
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl;
 
 
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
@@ -17,16 +16,22 @@ import com.tencent.supersonic.headless.server.persistence.repository.DictReposit
 import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.CollectionUtils;
 
+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 @Slf4j
 @Repository
 public class DictRepositoryImpl implements DictRepository {
 
+    @Value("${dict.task.num:10}")
+    private Integer dictTaskNum;
+
     private final DictTaskMapper dictTaskMapper;
     private final DictConfMapper dictConfMapper;
     private final DictUtils dictConverter;
@@ -83,7 +88,9 @@ public class DictRepositoryImpl implements DictRepository {
         QueryWrapper<DictTaskDO> wrapper = new QueryWrapper<>();
         wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId());
         wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType());
-        List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper);
+        List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper).stream()
+                .sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
+                .limit(dictTaskNum).collect(Collectors.toList());
         if (CollectionUtils.isEmpty(dictTaskDOList)) {
             return taskResp;
         }
@@ -102,9 +109,9 @@ public class DictRepositoryImpl implements DictRepository {
 
     @Override
     public Long editDictConf(DictConfDO dictConfDO) {
-        DictItemFilter filter = DictItemFilter.builder().type(TypeEnums.valueOf(dictConfDO.getType()))
+        DictItemFilter filter = DictItemFilter.builder()
+                .type(TypeEnums.valueOf(dictConfDO.getType()))
                 .itemId(dictConfDO.getItemId())
-                .status(StatusEnum.ONLINE)
                 .build();
 
         List<DictConfDO> dictConfDOList = getDictConfDOList(filter);

headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java

@@ -42,7 +42,7 @@ public class KnowledgeController {
      * @param dictItemReq
      */
     @PostMapping("/conf")
-    public Long addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
+    public DictItemResp addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
                             HttpServletRequest request,
                             HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);
@@ -56,7 +56,7 @@ public class KnowledgeController {
      * @param dictItemReq
      */
     @PutMapping("/conf")
-    public Long editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
+    public DictItemResp editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
                              HttpServletRequest request,
                              HttpServletResponse response) {
         User user = UserHolder.findUser(request, response);

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java

@@ -12,9 +12,9 @@ import java.util.List;
  */
 public interface DictConfService {
 
-    Long addDictConf(DictItemReq itemValueReq, User user);
+    DictItemResp addDictConf(DictItemReq itemValueReq, User user);
 
-    Long editDictConf(DictItemReq itemValueReq, User user);
+    DictItemResp editDictConf(DictItemReq itemValueReq, User user);
 
     List<DictItemResp> queryDictConf(DictItemFilter dictItemFilter, User user);
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java

@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.service.impl;
 
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
@@ -10,11 +9,15 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
 import com.tencent.supersonic.headless.server.service.DictConfService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Optional;
 
 @Service
+@Slf4j
 public class DictConfServiceImpl implements DictConfService {
 
     private final DictRepository dictRepository;
@@ -27,20 +30,49 @@ public class DictConfServiceImpl implements DictConfService {
     }
 
     @Override
-    public Long addDictConf(DictItemReq itemValueReq, User user) {
+    public DictItemResp addDictConf(DictItemReq itemValueReq, User user) {
         DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user);
-        return dictRepository.addDictConf(dictConfDO);
+        Boolean exist = checkConfExist(itemValueReq, user);
+        if (exist) {
+            throw new RuntimeException("dictConf is existed");
+        }
+        Long id = dictRepository.addDictConf(dictConfDO);
+        log.debug("dictConfDO:{}", dictConfDO);
+
+        DictItemFilter filter = DictItemFilter.builder()
+                .id(id)
+                .status(itemValueReq.getStatus())
+                .build();
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream().findFirst();
+        if (dictItemResp.isPresent()) {
+            return dictItemResp.get();
+        }
+        return null;
+    }
+
+    private Boolean checkConfExist(DictItemReq itemValueReq, User user) {
+        DictItemFilter filter = DictItemFilter.builder().build();
+        BeanUtils.copyProperties(itemValueReq, filter);
+        filter.setStatus(null);
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream()
+                .findFirst();
+        if (dictItemResp.isPresent()) {
+            return true;
+        }
+        return false;
     }
 
     @Override
-    public Long editDictConf(DictItemReq itemValueReq, User user) {
+    public DictItemResp editDictConf(DictItemReq itemValueReq, User user) {
         DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user);
         dictRepository.editDictConf(dictConfDO);
-        if (StatusEnum.DELETED.equals(itemValueReq.getStatus())) {
+        DictItemFilter filter = DictItemFilter.builder().build();
-            // todo delete dict file and refresh
+        BeanUtils.copyProperties(itemValueReq, filter);
-
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream().findFirst();
+        if (dictItemResp.isPresent()) {
+            return dictItemResp.get();
         }
-        return itemValueReq.getItemId();
+        return null;
     }
 
     @Override

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java

@@ -65,7 +65,7 @@ public class DictTaskServiceImpl implements DictTaskService {
     }
 
     private Long handleDictTaskByItemResp(DictItemResp dictItemResp, User user) {
-        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user);
+        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.PENDING);
         log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
         dictRepository.addDictTask(dictTaskDO);
         Long idInDb = dictTaskDO.getId();
@@ -95,14 +95,14 @@ public class DictTaskServiceImpl implements DictTaskService {
         dictTaskDO.setStatus(TaskStatusEnum.RUNNING.getStatus());
         dictRepository.editDictTask(dictTaskDO);
 
-        // 1.生成item字典数据
+        // 1.Generate item dictionary data
         List<String> data = dictUtils.fetchItemValue(dictItemResp);
 
-        // 2.变更字典文件
+        // 2.Change dictionary file
         String fileName = dictItemResp.fetchDictFileName() + Constants.DOT + dictFileType;
         fileHandler.writeFile(data, fileName, false);
 
-        // 3.实时变更内存中字典数据
+        // 3.Change in-memory dictionary data in real time
         try {
             HanlpHelper.reloadCustomDictionary();
             dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus());
@@ -124,7 +124,10 @@ public class DictTaskServiceImpl implements DictTaskService {
         } catch (Exception e) {
             log.error("reloadCustomDictionary error", e);
         }
-
+        // Add a clear dictionary file record
+        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.INITIAL);
+        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
+        dictRepository.addDictTask(dictTaskDO);
         return 0L;
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils;
 import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
 import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
 import static com.tencent.supersonic.common.pojo.Constants.COMMA;
+import static com.tencent.supersonic.common.pojo.Constants.POUND;
 import static com.tencent.supersonic.common.pojo.Constants.SPACE;
 
 import com.google.common.base.Strings;
@@ -36,6 +37,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.QueryService;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -48,6 +50,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
+
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
@@ -78,9 +81,9 @@ public class DictUtils {
     private final ModelService modelService;
 
     public DictUtils(DimensionService dimensionService,
-            MetricService metricService,
+                     MetricService metricService,
-            QueryService queryService,
+                     QueryService queryService,
-            ModelService modelService) {
+                     ModelService modelService) {
         this.dimensionService = dimensionService;
         this.metricService = metricService;
         this.queryService = queryService;
@@ -92,7 +95,7 @@ public class DictUtils {
                 dictItemResp.getItemId());
     }
 
-    public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user) {
+    public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user, TaskStatusEnum status) {
         DictTaskDO taskDO = new DictTaskDO();
         Date createAt = new Date();
         String name = dictItemResp.fetchDictFileName();
@@ -100,7 +103,7 @@ public class DictUtils {
         taskDO.setType(dictItemResp.getType().name());
         taskDO.setItemId(dictItemResp.getItemId());
         taskDO.setConfig(JsonUtil.toString(dictItemResp.getConfig()));
-        taskDO.setStatus(TaskStatusEnum.PENDING.getStatus());
+        taskDO.setStatus(status.getStatus());
         taskDO.setCreatedAt(createAt);
         String creator = (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) ? "" : user.getName();
         taskDO.setCreatedBy(creator);
@@ -185,7 +188,12 @@ public class DictUtils {
             return;
         }
         List<String> whiteList = dictItemResp.getConfig().getWhiteList();
-        whiteList.forEach(white -> lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency)));
+        whiteList.forEach(white -> {
+            if (!Strings.isNullOrEmpty(white)) {
+                white = white.replace(SPACE, POUND);
+            }
+            lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency));
+        });
     }
 
     private void constructDictLines(Map<String, Long> valueAndFrequencyPair, List<String> lines, String nature) {
@@ -194,6 +202,9 @@ public class DictUtils {
         }
 
         valueAndFrequencyPair.forEach((value, frequency) -> {
+            if (!Strings.isNullOrEmpty(value)) {
+                value = value.replace(SPACE, POUND);
+            }
             lines.add(String.format("%s %s %s", value, nature, frequency));
         });
     }