diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java index 31185ce1c..ca53f33f0 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Constants.java @@ -11,6 +11,7 @@ public class Constants { public static final String AT_SYMBOL = "@"; public static final String DOT = "."; public static String SPACE = " "; + public static String POUND = "#"; public static final String COLON = ":"; public static final String MINUS = "-"; public static final String UNDERLINE = "_"; diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java index 03f3349ff..6bea6a6cd 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/TaskStatusEnum.java @@ -2,6 +2,8 @@ package com.tencent.supersonic.common.pojo.enums; public enum TaskStatusEnum { + INITIAL("initial", -2), + ERROR("error", -1), PENDING("pending", 0), @@ -10,7 +12,7 @@ public enum TaskStatusEnum { SUCCESS("success", 2), - UNKNOWN("UNKNOWN", 3); + UNKNOWN("unknown", 3); private String status; private Integer code; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java index 089bbba1a..d2e9fd167 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/DictItemFilter.java @@ -5,14 +5,11 @@ import com.tencent.supersonic.common.pojo.enums.TypeEnums; import lombok.Builder; import lombok.Data; -import javax.validation.constraints.NotNull; - @Data @Builder public class DictItemFilter { private Long id; private TypeEnums type; private Long itemId; - @NotNull private StatusEnum status; } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java index 020992ef8..6229d2c36 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/DictRepositoryImpl.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.persistence.repository.impl; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq; @@ -17,16 +16,22 @@ import com.tencent.supersonic.headless.server.persistence.repository.DictReposit import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.utils.DictUtils; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; import org.springframework.util.CollectionUtils; +import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; @Slf4j @Repository public class DictRepositoryImpl implements DictRepository { + @Value("${dict.task.num:10}") + private Integer dictTaskNum; + private final DictTaskMapper dictTaskMapper; private final DictConfMapper dictConfMapper; private final DictUtils dictConverter; @@ -83,7 +88,9 @@ public class DictRepositoryImpl implements DictRepository { QueryWrapper wrapper = new QueryWrapper<>(); wrapper.lambda().eq(DictTaskDO::getItemId, taskReq.getItemId()); wrapper.lambda().eq(DictTaskDO::getType, taskReq.getType()); - List dictTaskDOList = dictTaskMapper.selectList(wrapper); + List dictTaskDOList = dictTaskMapper.selectList(wrapper).stream() + .sorted(Comparator.comparing(DictTaskDO::getCreatedAt).reversed()) + .limit(dictTaskNum).collect(Collectors.toList()); if (CollectionUtils.isEmpty(dictTaskDOList)) { return taskResp; } @@ -102,9 +109,9 @@ public class DictRepositoryImpl implements DictRepository { @Override public Long editDictConf(DictConfDO dictConfDO) { - DictItemFilter filter = DictItemFilter.builder().type(TypeEnums.valueOf(dictConfDO.getType())) + DictItemFilter filter = DictItemFilter.builder() + .type(TypeEnums.valueOf(dictConfDO.getType())) .itemId(dictConfDO.getItemId()) - .status(StatusEnum.ONLINE) .build(); List dictConfDOList = getDictConfDOList(filter); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java index b6e2d65e6..fdf5b4f1b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java @@ -42,7 +42,7 @@ public class KnowledgeController { * @param dictItemReq */ @PostMapping("/conf") - public Long addDictConf(@RequestBody @Valid DictItemReq dictItemReq, + public DictItemResp addDictConf(@RequestBody @Valid DictItemReq dictItemReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); @@ -56,7 +56,7 @@ public class KnowledgeController { * @param dictItemReq */ @PutMapping("/conf") - public Long editDictConf(@RequestBody @Valid DictItemReq dictItemReq, + public DictItemResp editDictConf(@RequestBody @Valid DictItemReq dictItemReq, HttpServletRequest request, HttpServletResponse response) { User user = UserHolder.findUser(request, response); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java index 1dff03d54..21347f87c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/DictConfService.java @@ -12,9 +12,9 @@ import java.util.List; */ public interface DictConfService { - Long addDictConf(DictItemReq itemValueReq, User user); + DictItemResp addDictConf(DictItemReq itemValueReq, User user); - Long editDictConf(DictItemReq itemValueReq, User user); + DictItemResp editDictConf(DictItemReq itemValueReq, User user); List queryDictConf(DictItemFilter dictItemFilter, User user); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java index 14750879a..8f583508b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictConfServiceImpl.java @@ -2,7 +2,6 @@ package com.tencent.supersonic.headless.server.service.impl; import com.tencent.supersonic.auth.api.authentication.pojo.User; -import com.tencent.supersonic.common.pojo.enums.StatusEnum; import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter; import com.tencent.supersonic.headless.api.pojo.request.DictItemReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; @@ -10,11 +9,15 @@ import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO; import com.tencent.supersonic.headless.server.persistence.repository.DictRepository; import com.tencent.supersonic.headless.server.service.DictConfService; import com.tencent.supersonic.headless.server.utils.DictUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; import java.util.List; +import java.util.Optional; @Service +@Slf4j public class DictConfServiceImpl implements DictConfService { private final DictRepository dictRepository; @@ -27,20 +30,49 @@ public class DictConfServiceImpl implements DictConfService { } @Override - public Long addDictConf(DictItemReq itemValueReq, User user) { + public DictItemResp addDictConf(DictItemReq itemValueReq, User user) { DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user); - return dictRepository.addDictConf(dictConfDO); + Boolean exist = checkConfExist(itemValueReq, user); + if (exist) { + throw new RuntimeException("dictConf is existed"); + } + Long id = dictRepository.addDictConf(dictConfDO); + log.debug("dictConfDO:{}", dictConfDO); + + DictItemFilter filter = DictItemFilter.builder() + .id(id) + .status(itemValueReq.getStatus()) + .build(); + Optional dictItemResp = queryDictConf(filter, user).stream().findFirst(); + if (dictItemResp.isPresent()) { + return dictItemResp.get(); + } + return null; + } + + private Boolean checkConfExist(DictItemReq itemValueReq, User user) { + DictItemFilter filter = DictItemFilter.builder().build(); + BeanUtils.copyProperties(itemValueReq, filter); + filter.setStatus(null); + Optional dictItemResp = queryDictConf(filter, user).stream() + .findFirst(); + if (dictItemResp.isPresent()) { + return true; + } + return false; } @Override - public Long editDictConf(DictItemReq itemValueReq, User user) { + public DictItemResp editDictConf(DictItemReq itemValueReq, User user) { DictConfDO dictConfDO = dictConverter.generateDictConfDO(itemValueReq, user); dictRepository.editDictConf(dictConfDO); - if (StatusEnum.DELETED.equals(itemValueReq.getStatus())) { - // todo delete dict file and refresh - + DictItemFilter filter = DictItemFilter.builder().build(); + BeanUtils.copyProperties(itemValueReq, filter); + Optional dictItemResp = queryDictConf(filter, user).stream().findFirst(); + if (dictItemResp.isPresent()) { + return dictItemResp.get(); } - return itemValueReq.getItemId(); + return null; } @Override diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java index 79a8755d9..78f112834 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java @@ -65,7 +65,7 @@ public class DictTaskServiceImpl implements DictTaskService { } private Long handleDictTaskByItemResp(DictItemResp dictItemResp, User user) { - DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user); + DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.PENDING); log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); dictRepository.addDictTask(dictTaskDO); Long idInDb = dictTaskDO.getId(); @@ -95,14 +95,14 @@ public class DictTaskServiceImpl implements DictTaskService { dictTaskDO.setStatus(TaskStatusEnum.RUNNING.getStatus()); dictRepository.editDictTask(dictTaskDO); - // 1.生成item字典数据 + // 1.Generate item dictionary data List data = dictUtils.fetchItemValue(dictItemResp); - // 2.变更字典文件 + // 2.Change dictionary file String fileName = dictItemResp.fetchDictFileName() + Constants.DOT + dictFileType; fileHandler.writeFile(data, fileName, false); - // 3.实时变更内存中字典数据 + // 3.Change in-memory dictionary data in real time try { HanlpHelper.reloadCustomDictionary(); dictTaskDO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); @@ -124,7 +124,10 @@ public class DictTaskServiceImpl implements DictTaskService { } catch (Exception e) { log.error("reloadCustomDictionary error", e); } - + // Add a clear dictionary file record + DictTaskDO dictTaskDO = dictConverter.generateDictTaskDO(dictItemResp, user, TaskStatusEnum.INITIAL); + log.info("[addDictTask] dictTaskDO:{}", dictTaskDO); + dictRepository.addDictTask(dictTaskDO); return 0L; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java index 8c69a0cf4..c12b3a364 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DictUtils.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.utils; import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER; import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE; import static com.tencent.supersonic.common.pojo.Constants.COMMA; +import static com.tencent.supersonic.common.pojo.Constants.POUND; import static com.tencent.supersonic.common.pojo.Constants.SPACE; import com.google.common.base.Strings; @@ -36,6 +37,7 @@ import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.QueryService; + import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -48,6 +50,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.StringJoiner; + import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -78,9 +81,9 @@ public class DictUtils { private final ModelService modelService; public DictUtils(DimensionService dimensionService, - MetricService metricService, - QueryService queryService, - ModelService modelService) { + MetricService metricService, + QueryService queryService, + ModelService modelService) { this.dimensionService = dimensionService; this.metricService = metricService; this.queryService = queryService; @@ -92,7 +95,7 @@ public class DictUtils { dictItemResp.getItemId()); } - public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user) { + public DictTaskDO generateDictTaskDO(DictItemResp dictItemResp, User user, TaskStatusEnum status) { DictTaskDO taskDO = new DictTaskDO(); Date createAt = new Date(); String name = dictItemResp.fetchDictFileName(); @@ -100,7 +103,7 @@ public class DictUtils { taskDO.setType(dictItemResp.getType().name()); taskDO.setItemId(dictItemResp.getItemId()); taskDO.setConfig(JsonUtil.toString(dictItemResp.getConfig())); - taskDO.setStatus(TaskStatusEnum.PENDING.getStatus()); + taskDO.setStatus(status.getStatus()); taskDO.setCreatedAt(createAt); String creator = (Objects.isNull(user) || Strings.isNullOrEmpty(user.getName())) ? "" : user.getName(); taskDO.setCreatedBy(creator); @@ -185,7 +188,12 @@ public class DictUtils { return; } List whiteList = dictItemResp.getConfig().getWhiteList(); - whiteList.forEach(white -> lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency))); + whiteList.forEach(white -> { + if (!Strings.isNullOrEmpty(white)) { + white = white.replace(SPACE, POUND); + } + lines.add(String.format("%s %s %s", white, nature, itemValueWhiteFrequency)); + }); } private void constructDictLines(Map valueAndFrequencyPair, List lines, String nature) { @@ -194,6 +202,9 @@ public class DictUtils { } valueAndFrequencyPair.forEach((value, frequency) -> { + if (!Strings.isNullOrEmpty(value)) { + value = value.replace(SPACE, POUND); + } lines.add(String.format("%s %s %s", value, nature, frequency)); }); }