Dict headless opt (#764)

* [improvement](headless) Enhance dictionary execution function,support clear dict data recording * [fix](headless) fix dict value support space --------- Co-authored-by: kanedai <kanedai@tencent.com>
2025-12-10 11:07:06 +00:00 · 2024-02-27 10:40:31 +08:00
parent 0541614dad
commit b72e280990
9 changed files with 84 additions and 31 deletions
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java
@@ -11,6 +11,7 @@ public class Constants {
    public static final String AT_SYMBOL = "@";
    public static final String DOT = ".";
    public static String SPACE = " ";
+    public static String POUND = "#";
    public static final String COLON = ":";
    public static final String MINUS = "-";
    public static final String UNDERLINE = "_";
--- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java
+++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java
@@ -2,6 +2,8 @@ package com.tencent.supersonic.common.pojo.enums;

 public enum TaskStatusEnum {

+    INITIAL("initial", -2),
+
    ERROR("error", -1),

    PENDING("pending", 0),
@@ -10,7 +12,7 @@ public enum TaskStatusEnum {

    SUCCESS("success", 2),

-    UNKNOWN("UNKNOWN", 3);
+    UNKNOWN("unknown", 3);

    private String status;
    private Integer code;
--- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java
+++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java
@@ -5,14 +5,11 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import lombok.Builder;
 import lombok.Data;

-import javax.validation.constraints.NotNull;
-
@Data
@Builder
 public class DictItemFilter {
    private Long id;
    private TypeEnums type;
    private Long itemId;
-    @NotNull
    private StatusEnum status;
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java
@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl;


 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
-import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.common.pojo.enums.TypeEnums;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
@@ -17,16 +16,22 @@ import com.tencent.supersonic.headless.server.persistence.repository.DictReposit
 import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Repository;
 import org.springframework.util.CollectionUtils;

+import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;

@Slf4j
@Repository
 public class DictRepositoryImpl implements DictRepository {

+    @Value("${dict.task.num:10}")
+    private Integer dictTaskNum;
+
    private final DictTaskMapper dictTaskMapper;
    private final DictConfMapper dictConfMapper;
    private final DictUtils dictConverter;
@@ -83,7 +88,9 @@ public class DictRepositoryImpl implements DictRepository {
        QueryWrapper<DictTaskDO> wrapper = new QueryWrapper<>();
        wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId());
        wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType());
-        List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper);
+        List<DictTaskDO> dictTaskDOList = dictTaskMapper.selectList(wrapper).stream()
+                .sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed())
+                .limit(dictTaskNum).collect(Collectors.toList());
        if (CollectionUtils.isEmpty(dictTaskDOList)) {
            return taskResp;
        }
@@ -102,9 +109,9 @@ public class DictRepositoryImpl implements DictRepository {

    @Override
    public Long editDictConf(DictConfDO dictConfDO) {
-        DictItemFilter filter = DictItemFilter.builder().type(TypeEnums.valueOf(dictConfDO.getType()))
+        DictItemFilter filter = DictItemFilter.builder()
+                .type(TypeEnums.valueOf(dictConfDO.getType()))
                .itemId(dictConfDO.getItemId())
-                .status(StatusEnum.ONLINE)
                .build();

        List<DictConfDO> dictConfDOList = getDictConfDOList(filter);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java
@@ -42,7 +42,7 @@ public class KnowledgeController {
     * @param dictItemReq
     */
    @PostMapping("/conf")
-    public Long addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
+    public DictItemResp addDictConf(@RequestBody @Valid DictItemReq dictItemReq,
                            HttpServletRequest request,
                            HttpServletResponse response) {
        User user = UserHolder.findUser(request, response);
@@ -56,7 +56,7 @@ public class KnowledgeController {
     * @param dictItemReq
     */
    @PutMapping("/conf")
-    public Long editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
+    public DictItemResp editDictConf(@RequestBody @Valid DictItemReq dictItemReq,
                             HttpServletRequest request,
                             HttpServletResponse response) {
        User user = UserHolder.findUser(request, response);
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java
@@ -12,9 +12,9 @@ import java.util.List;
 */
 public interface DictConfService {

-    Long addDictConf(DictItemReq itemValueReq, User user);
+    DictItemResp addDictConf(DictItemReq itemValueReq, User user);

-    Long editDictConf(DictItemReq itemValueReq, User user);
+    DictItemResp editDictConf(DictItemReq itemValueReq, User user);

    List<DictItemResp> queryDictConf(DictItemFilter dictItemFilter, User user);
 }
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java
@@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.service.impl;


 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.common.pojo.enums.StatusEnum;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
 import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
 import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
@@ -10,11 +9,15 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
 import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
 import com.tencent.supersonic.headless.server.service.DictConfService;
 import com.tencent.supersonic.headless.server.utils.DictUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;

 import java.util.List;
+import java.util.Optional;

@Service
+@Slf4j
 public class DictConfServiceImpl implements DictConfService {

    private final DictRepository dictRepository;
@@ -27,20 +30,49 @@ public class DictConfServiceImpl implements DictConfService {
    }

    @Override
-    public Long addDictConf(DictItemReq itemValueReq, User user) {
+    public DictItemResp addDictConf(DictItemReq itemValueReq, User user) {
        DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user);
-        return dictRepository.addDictConf(dictConfDO);
+        Boolean exist = checkConfExist(itemValueReq, user);
+        if (exist) {
+            throw new RuntimeException("dictConf is existed");
+        }
+        Long id = dictRepository.addDictConf(dictConfDO);
+        log.debug("dictConfDO:{}", dictConfDO);
+
+        DictItemFilter filter = DictItemFilter.builder()
+                .id(id)
+                .status(itemValueReq.getStatus())
+                .build();
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream().findFirst();
+        if (dictItemResp.isPresent()) {
+            return dictItemResp.get();
+        }
+        return null;
+    }
+
+    private Boolean checkConfExist(DictItemReq itemValueReq, User user) {
+        DictItemFilter filter = DictItemFilter.builder().build();
+        BeanUtils.copyProperties(itemValueReq, filter);
+        filter.setStatus(null);
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream()
+                .findFirst();
+        if (dictItemResp.isPresent()) {
+            return true;
+        }
+        return false;
    }

    @Override
-    public Long editDictConf(DictItemReq itemValueReq, User user) {
+    public DictItemResp editDictConf(DictItemReq itemValueReq, User user) {
        DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user);
        dictRepository.editDictConf(dictConfDO);
-        if (StatusEnum.DELETED.equals(itemValueReq.getStatus())) {
-            // todo delete dict file and refresh
-
+        DictItemFilter filter = DictItemFilter.builder().build();
+        BeanUtils.copyProperties(itemValueReq, filter);
+        Optional<DictItemResp> dictItemResp = queryDictConf(filter, user).stream().findFirst();
+        if (dictItemResp.isPresent()) {
+            return dictItemResp.get();
        }
-        return itemValueReq.getItemId();
+        return null;
    }

    @Override
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
@@ -65,7 +65,7 @@ public class DictTaskServiceImpl implements DictTaskService {
    }

    private Long handleDictTaskByItemResp(DictItemResp dictItemResp, User user) {
-        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user);
+        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.PENDING);
        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
        dictRepository.addDictTask(dictTaskDO);
        Long idInDb = dictTaskDO.getId();
@@ -95,14 +95,14 @@ public class DictTaskServiceImpl implements DictTaskService {
        dictTaskDO.setStatus(TaskStatusEnum.RUNNING.getStatus());
        dictRepository.editDictTask(dictTaskDO);

-        // 1.生成item字典数据
+        // 1.Generate item dictionary data
        List<String> data = dictUtils.fetchItemValue(dictItemResp);

-        // 2.变更字典文件
+        // 2.Change dictionary file
        String fileName = dictItemResp.fetchDictFileName() + Constants.DOT + dictFileType;
        fileHandler.writeFile(data, fileName, false);

-        // 3.实时变更内存中字典数据
+        // 3.Change in-memory dictionary data in real time
        try {
            HanlpHelper.reloadCustomDictionary();
            dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus());
@@ -124,7 +124,10 @@ public class DictTaskServiceImpl implements DictTaskService {
        } catch (Exception e) {
            log.error("reloadCustomDictionary error", e);
        }
-
+        // Add a clear dictionary file record
+        DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.INITIAL);
+        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
+        dictRepository.addDictTask(dictTaskDO);
        return 0L;
    }

--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java
@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils;
 import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
 import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
 import static com.tencent.supersonic.common.pojo.Constants.COMMA;
+import static com.tencent.supersonic.common.pojo.Constants.POUND;
 import static com.tencent.supersonic.common.pojo.Constants.SPACE;

 import com.google.common.base.Strings;
@@ -36,6 +37,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService;
 import com.tencent.supersonic.headless.server.service.MetricService;
 import com.tencent.supersonic.headless.server.service.ModelService;
 import com.tencent.supersonic.headless.server.service.QueryService;
+
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -48,6 +50,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.StringJoiner;
+
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
@@ -92,7 +95,7 @@ public class DictUtils {
                dictItemResp.getItemId());
    }

-    public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user) {
+    public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user, TaskStatusEnum status) {
        DictTaskDO taskDO = new DictTaskDO();
        Date createAt = new Date();
        String name = dictItemResp.fetchDictFileName();
@@ -100,7 +103,7 @@ public class DictUtils {
        taskDO.setType(dictItemResp.getType().name());
        taskDO.setItemId(dictItemResp.getItemId());
        taskDO.setConfig(JsonUtil.toString(dictItemResp.getConfig()));
-        taskDO.setStatus(TaskStatusEnum.PENDING.getStatus());
+        taskDO.setStatus(status.getStatus());
        taskDO.setCreatedAt(createAt);
        String creator = (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) ? "" : user.getName();
        taskDO.setCreatedBy(creator);
@@ -185,7 +188,12 @@ public class DictUtils {
            return;
        }
        List<String> whiteList = dictItemResp.getConfig().getWhiteList();
-        whiteList.forEach(white -> lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency)));
+        whiteList.forEach(white -> {
+            if (!Strings.isNullOrEmpty(white)) {
+                white = white.replace(SPACE, POUND);
+            }
+            lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency));
+        });
    }

    private void constructDictLines(Map<String, Long> valueAndFrequencyPair, List<String> lines, String nature) {
@@ -194,6 +202,9 @@ public class DictUtils {
        }

        valueAndFrequencyPair.forEach((value, frequency) -> {
+            if (!Strings.isNullOrEmpty(value)) {
+                value = value.replace(SPACE, POUND);
+            }
            lines.add(String.format("%s %s %s", value, nature, frequency));
        });
    }