headless integrates knowledge (#722)

chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java

@@ -1,11 +1,13 @@
 package com.tencent.supersonic.chat.server.listener;
 
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.server.service.KnowledgeService;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
 import com.tencent.supersonic.chat.server.service.impl.SchemaService;
 import com.tencent.supersonic.chat.server.service.impl.WordService;
+
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+
+import com.tencent.supersonic.headless.server.service.KnowledgeService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;

chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.server.listener;
 
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
 import com.tencent.supersonic.chat.server.service.impl.SchemaService;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DataEvent;

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictConfMapper.java

@@ -1,17 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.mapper;
-
-
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO;
-import org.apache.ibatis.annotations.Mapper;
-
-@Mapper
-public interface DictConfMapper {
-
-    Boolean createDictConf(DictConfDO dictConfDO);
-
-    Boolean editDictConf(DictConfDO dictConfDO);
-
-    Boolean upsertDictInfo(DictConfDO dictConfDO);
-
-    DictConfDO getDictInfoByModelId(Long modelId);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictTaskMapper.java

@@ -1,18 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.mapper;
-
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-
-import java.util.List;
-
-import org.apache.ibatis.annotations.Mapper;
-
-@Mapper
-public interface DictTaskMapper {
-
-    Long createDimValueTask(DictTaskDO dictTaskDO);
-
-    Boolean updateTaskStatus(DictTaskDO dictTaskDO);
-
-    List<DictTaskDO> searchDictTaskList(DictTaskFilterReq filter);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/DictRepository.java

@@ -1,19 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.repository;
-
-
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import java.util.List;
-
-public interface DictRepository {
-
-    Long createDimValueDictTask(DictTaskDO dictTaskDO);
-
-    Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO);
-
-    List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter);
-
-    DictConfig getDictInfoByModelId(Long modelId);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/DictRepositoryImpl.java

@@ -1,83 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.repository.impl;
-
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.chat.server.persistence.mapper.DictConfMapper;
-import com.tencent.supersonic.chat.server.persistence.mapper.DictTaskMapper;
-import com.tencent.supersonic.chat.server.persistence.repository.DictRepository;
-import com.tencent.supersonic.chat.server.util.DictTaskConverter;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.common.util.JsonUtil;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.BeanUtils;
-import org.springframework.stereotype.Repository;
-import org.springframework.util.CollectionUtils;
-
-
-@Repository
-@Slf4j
-public class DictRepositoryImpl implements DictRepository {
-
-    private final DictTaskMapper dictTaskMapper;
-    private final DictConfMapper dictConfMapper;
-
-    public DictRepositoryImpl(DictTaskMapper dictTaskMapper,
-                              DictConfMapper dictConfMapper) {
-        this.dictTaskMapper = dictTaskMapper;
-        this.dictConfMapper = dictConfMapper;
-    }
-
-    @Override
-    public Long createDimValueDictTask(DictTaskDO dictTaskDO) {
-        dictTaskMapper.createDimValueTask(dictTaskDO);
-        return dictTaskDO.getId();
-    }
-
-    @Override
-    public Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO) {
-        dictTaskDO.setStatus(status);
-        Date createdAt = dictTaskDO.getCreatedAt();
-        long elapsedMs = System.currentTimeMillis() - createdAt.getTime();
-        dictTaskDO.setElapsedMs(elapsedMs);
-        dictTaskMapper.updateTaskStatus(dictTaskDO);
-        return true;
-    }
-
-    @Override
-    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter) {
-        List<DimValueDictInfo> dimValueDictDescList = new ArrayList<>();
-        log.info("filter:{}", filter);
-        List<DictTaskDO> dictTaskDOList = dictTaskMapper.searchDictTaskList(filter);
-        if (!CollectionUtils.isEmpty(dictTaskDOList)) {
-            dictTaskDOList.stream().forEach(dictTaskDO -> {
-                DimValueDictInfo dimValueDictDesc = new DimValueDictInfo();
-                BeanUtils.copyProperties(dictTaskDO, dimValueDictDesc);
-                dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskDO.getStatus()));
-                if (StringUtils.isNotEmpty(dictTaskDO.getDimIds())) {
-                    Set<Long> dimIds = JsonUtil.toSet(dictTaskDO.getDimIds(), Long.class);
-                    dimValueDictDesc.setDimIds(dimIds);
-                }
-                dimValueDictDescList.add(dimValueDictDesc);
-            });
-        }
-        return dimValueDictDescList;
-    }
-
-    @Override
-    public DictConfig getDictInfoByModelId(Long modelId) {
-        DictConfDO dictConfDO = dictConfMapper.getDictInfoByModelId(modelId);
-        if (Objects.isNull(dictConfDO)) {
-            return null;
-        }
-        return DictTaskConverter.dictConfPO2Config(dictConfDO);
-    }
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.chat.server.processor.execute;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java

@@ -11,14 +11,13 @@ import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT;
 import static com.tencent.supersonic.common.pojo.Constants.WEEK;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
 import com.tencent.supersonic.chat.api.pojo.response.MetricInfo;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.core.config.AggregatorConfig;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
 import com.tencent.supersonic.common.pojo.DateConf;
@@ -29,6 +28,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.pojo.enums.RatioOverType;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateUtils;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import java.text.DecimalFormat;

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.chat.server.processor.execute;
 
 import com.alibaba.fastjson.JSONObject;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.server.processor.parse;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java

@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
 import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
 import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.chat.server.service.ConfigService;
 import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.rest;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
-import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/KnowledgeController.java

@@ -1,114 +0,0 @@
-package com.tencent.supersonic.chat.server.rest;
-
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
-import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.server.listener.ApplicationStartedListener;
-import com.tencent.supersonic.chat.server.service.KnowledgeTaskService;
-import java.util.List;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.validation.Valid;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.PutMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-
-@RestController
-@RequestMapping("/api/chat/dict")
-public class KnowledgeController {
-
-    @Autowired
-    private KnowledgeTaskService knowledgeTaskService;
-
-    @Autowired
-    private ApplicationStartedListener applicationStartedListener;
-
-    /**
-     * addDictInfo
-     * write specific dimension values to the knowledge base
-     *
-     * @param dimValue2DictCommend
-     */
-    @PostMapping("/task")
-    public Long addDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend,
-                            HttpServletRequest request,
-                            HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.addDictTask(dimValue2DictCommend, user);
-    }
-
-    /**
-     * deleteDictInfo
-     * remove specific dimension values from the knowledge base
-     *
-     * @param dimValue2DictCommend
-     */
-    @PostMapping("/task/delete")
-    public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend,
-                               HttpServletRequest request,
-                               HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.deleteDictTask(dimValue2DictCommend, user);
-    }
-
-    /**
-     * searchDictTaskList
-     *
-     * @param filter
-     */
-    @PostMapping("/task/search")
-    public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilterReq filter,
-                                                     HttpServletRequest request,
-                                                     HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.searchDictTaskList(filter, user);
-    }
-
-    /**
-     * searchDictLatestTaskList
-     */
-    @PostMapping("/task/search/latest")
-    public List<DictLatestTaskResp> searchDictLatestTaskList(@RequestBody @Valid DictLatestTaskReq filter,
-                                                             HttpServletRequest request,
-                                                             HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.searchDictLatestTaskList(filter, user);
-    }
-
-    /**
-     * getDictRootPath
-     * get knowledge base file directory
-     *
-     * @return
-     */
-    @GetMapping("/rootPath")
-    public String getDictRootPath(HttpServletRequest request,
-                                  HttpServletResponse response) {
-        return knowledgeTaskService.getDictRootPath();
-    }
-
-    /**
-     * updateDimValue
-     * update in-memory dictionary files in real time
-     *
-     * @param request
-     * @param response
-     * @return
-     */
-    @PutMapping("/knowledge/dimValue")
-    public Boolean updateDimValue(HttpServletRequest request,
-                                  HttpServletResponse response) {
-        return applicationStartedListener.updateKnowledgeDimValue();
-    }
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeService.java

@@ -1,14 +0,0 @@
-package com.tencent.supersonic.chat.server.service;
-
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import java.util.List;
-
-public interface KnowledgeService {
-
-    void updateSemanticKnowledge(List<DictWord> natures);
-
-    void reloadAllData(List<DictWord> natures);
-
-    void updateOnlineKnowledge(List<DictWord> natures);
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeTaskService.java

@@ -1,24 +0,0 @@
-package com.tencent.supersonic.chat.server.service;
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import java.util.List;
-
-public interface KnowledgeTaskService {
-    Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
-
-    Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
-
-    List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user);
-
-    DictConfig getDictInfoByModelId(Long modelId);
-
-    String getDictRootPath();
-
-    List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq filter, User user);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.server.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
-import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.service;
 
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
@@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.chat.api.pojo.response.DataInfo;
 import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
 import com.tencent.supersonic.chat.api.pojo.response.ViewInfo;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
 import com.tencent.supersonic.chat.server.service.impl.SchemaService;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatKnowledgeTaskServiceImpl.java

@@ -1,264 +0,0 @@
-package com.tencent.supersonic.chat.server.service.impl;
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
-import com.tencent.supersonic.chat.core.config.DefaultMetric;
-import com.tencent.supersonic.chat.core.config.Dim4Dict;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DictUpdateMode;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.core.knowledge.FileHandler;
-import com.tencent.supersonic.chat.core.utils.DictQueryHelper;
-import com.tencent.supersonic.chat.server.listener.ApplicationStartedListener;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO;
-import com.tencent.supersonic.chat.server.persistence.repository.DictRepository;
-import com.tencent.supersonic.chat.server.service.KnowledgeTaskService;
-import com.tencent.supersonic.chat.server.util.DictMetaHelper;
-import com.tencent.supersonic.chat.server.util.DictTaskConverter;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.common.util.JsonUtil;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
-
-@Slf4j
-@Service
-public class ChatKnowledgeTaskServiceImpl implements KnowledgeTaskService {
-
-    private final DictMetaHelper metaUtils;
-    private final DictQueryHelper dictQueryHelper;
-    private final FileHandler fileHandler;
-    private final DictRepository dictRepository;
-    private final ApplicationStartedListener applicationStartedListener;
-
-    @Value("${dict.flush.enable:true}")
-    private Boolean dictFlushEnable;
-    @Value("${dict.flush.daily.enable:true}")
-    private Boolean dictFlushDailyEnable;
-    @Value("${dict.file.type:txt}")
-    private String dictFileType;
-    private String dimValue = "DimValue_%d_%d";
-
-    public ChatKnowledgeTaskServiceImpl(DictMetaHelper metaUtils,
-                                    DictQueryHelper dictQueryHelper,
-                                    FileHandler fileHandler,
-                                    DictRepository dictRepository,
-                                    ApplicationStartedListener applicationStartedListener) {
-        this.metaUtils = metaUtils;
-        this.dictQueryHelper = dictQueryHelper;
-        this.fileHandler = fileHandler;
-        this.dictRepository = dictRepository;
-        this.applicationStartedListener = applicationStartedListener;
-    }
-
-    @Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}")
-    public Boolean dailyDictTask() {
-        log.info("[dailyDictTask] start");
-        if (!dictFlushDailyEnable) {
-            log.info("dictFlushDailyEnable is false, now finish dailyDictTask");
-        }
-        DimValue2DictCommand dimValue2DictCommend = new DimValue2DictCommand();
-        dimValue2DictCommend.setUpdateMode(DictUpdateMode.OFFLINE_FULL);
-
-        User user = User.getFakeUser();
-        addDictTask(dimValue2DictCommend, user);
-        log.info("[dailyDictTask] finish");
-        return true;
-    }
-
-    @Override
-    public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
-        if (!dictFlushEnable) {
-            return 0L;
-        }
-
-        if (DictUpdateMode.REALTIME_DELETE.equals(dimValue2DictCommend.getUpdateMode())) {
-            return deleteDictTask(dimValue2DictCommend, user);
-        }
-
-        DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommend, user);
-        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
-        // todo check dimension can not be searched
-
-        dictRepository.createDimValueDictTask(dictTaskDO);
-        runDictTask(dictTaskDO, user);
-
-        return dictTaskDO.getId();
-    }
-
-    public Long runDictTask(DictTaskDO dictTaskDO, User user) {
-        if (Objects.isNull(dictTaskDO)) {
-            return -1L;
-        }
-        DimValue2DictCommand command = JsonUtil.toObject(dictTaskDO.getCommand(), DimValue2DictCommand.class);
-        try {
-            //1. construct internal dictionary requirements
-            List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(command);
-            Set<Long> dimIds = generateDimSet(dimValueDOList);
-            dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
-            dictRepository.updateDictTaskStatus(TaskStatusEnum.RUNNING.getCode(), dictTaskDO);
-            log.debug("dimValueDOList:{}", dimValueDOList);
-            //2. query dimension value information
-            for (DimValueDO dimValueDO : dimValueDOList) {
-                Long modelId = dimValueDO.getModelId();
-                DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
-                for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
-                    List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
-                    //3. local file changes
-                    String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
-                            dim4Dict.getDimId());
-                    fileHandler.writeFile(data, fileName, false);
-                }
-            }
-            applicationStartedListener.updateKnowledgeDimValue();
-            log.debug("updateDictTaskStatus to SUCCESS");
-            dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
-        } catch (Exception e) {
-            log.warn("addDictInfo exception:", e);
-            dictRepository.updateDictTaskStatus(TaskStatusEnum.ERROR.getCode(), dictTaskDO);
-        }
-        return 1L;
-    }
-
-    private Set<Long> generateDimSet(List<DimValueDO> dimValueDOList) {
-        Set<Long> dimIds = new HashSet<>();
-        if (!CollectionUtils.isEmpty(dimValueDOList)) {
-            dimValueDOList.stream().forEach(dimValueDO -> {
-                if (!CollectionUtils.isEmpty(dimValueDO.getDimensions())) {
-                    dimValueDO.getDimensions().stream().forEach(dim4Dict -> dimIds.add(dim4Dict.getDimId()));
-                }
-            });
-        }
-        return dimIds;
-    }
-
-    @Override
-    public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommand, User user) {
-        if (!dictFlushEnable) {
-            return 0L;
-        }
-        if (Objects.isNull(dimValue2DictCommand) || !DictUpdateMode.REALTIME_DELETE.equals(
-                dimValue2DictCommand.getUpdateMode())) {
-            throw new RuntimeException("illegal parameter");
-        }
-
-        DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommand, user);
-        log.info("[deleteDictTask] dictTaskDO:{}", dictTaskDO);
-        Set<Long> dimIds = generateDimSetFromCommand(dimValue2DictCommand.getModelAndDimPair());
-        dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
-        dictRepository.createDimValueDictTask(dictTaskDO);
-
-        Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommand.getModelAndDimPair();
-        if (CollectionUtils.isEmpty(modelAndDimPair)) {
-            return 0L;
-        }
-        for (Long modelId : modelAndDimPair.keySet()) {
-            if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
-                continue;
-            }
-            for (Long dimId : modelAndDimPair.get(modelId)) {
-                String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
-                fileHandler.deleteDictFile(fileName);
-            }
-        }
-        applicationStartedListener.updateKnowledgeDimValue();
-        dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
-        applicationStartedListener.updateKnowledgeDimValue();
-
-        return 1L;
-    }
-
-    private Set<Long> generateDimSetFromCommand(Map<Long, List<Long>> modelAndDimPair) {
-        Set<Long> dimIds = new HashSet<>();
-        if (!CollectionUtils.isEmpty(modelAndDimPair)) {
-            modelAndDimPair.forEach((k, v) -> dimIds.addAll(v));
-        }
-        return dimIds;
-    }
-
-    @Override
-    public String getDictRootPath() {
-        return fileHandler.getDictRootPath();
-    }
-
-    @Override
-    public List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq latestFilter, User user) {
-        DictTaskFilterReq filter = new DictTaskFilterReq();
-        BeanUtils.copyProperties(latestFilter, filter);
-        List<DimValueDictInfo> dimValueDictInfoList = searchDictTaskList(filter, user);
-        return extractLatestTask(dimValueDictInfoList, latestFilter.getDimIds());
-    }
-
-    private List<DictLatestTaskResp> extractLatestTask(List<DimValueDictInfo> dimValueDictInfoList, List<Long> dimIds) {
-        List<DictLatestTaskResp> dictLatestTaskRespList = new ArrayList<>();
-        Map<Long, DictLatestTaskResp> dimAndTaskPair = new HashMap<>(50);
-        for (DimValueDictInfo dimValueDictInfo : dimValueDictInfoList) {
-            //1. filter
-            if (Objects.isNull(dimValueDictInfo) || CollectionUtils.isEmpty(dimValueDictInfo.getDimIds())) {
-                continue;
-            }
-            if (!CollectionUtils.isEmpty(dimIds)) {
-                Set<Long> tmp = dimValueDictInfo.getDimIds();
-                tmp.retainAll(dimIds);
-                dimValueDictInfo.setDimIds(tmp);
-                if (tmp.size() <= 0) {
-                    continue;
-                }
-            }
-
-            // 2. extract
-            Set<Long> dimIdList = dimValueDictInfo.getDimIds();
-            for (Long dimId : dimIdList) {
-                DictLatestTaskResp dictLatestTaskResp = new DictLatestTaskResp();
-                if (!dimAndTaskPair.containsKey(dimId)) {
-                    BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
-                    dictLatestTaskResp.setDimId(dimId);
-                } else {
-                    DictLatestTaskResp dictLatestTaskExist = dimAndTaskPair.get(dimId);
-                    if (dictLatestTaskExist.getCreatedAt().before(dimValueDictInfo.getCreatedAt())) {
-                        BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
-                        dictLatestTaskResp.setDimId(dimId);
-                    } else {
-                        dictLatestTaskResp = dictLatestTaskExist;
-                    }
-                }
-                dimAndTaskPair.put(dimId, dictLatestTaskResp);
-            }
-
-        }
-
-        if (dimAndTaskPair.size() >= 0 && !CollectionUtils.isEmpty(dimAndTaskPair.values())) {
-            dimAndTaskPair.values().stream()
-                    .filter(v -> !v.getCommand().contains(DictUpdateMode.REALTIME_DELETE.name()))
-                    .forEach(v -> dictLatestTaskRespList.add(v));
-        }
-
-
-        return dictLatestTaskRespList;
-    }
-
-    @Override
-    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user) {
-        return dictRepository.searchDictTaskList(filter);
-    }
-
-    @Override
-    public DictConfig getDictInfoByModelId(Long modelId) {
-        return dictRepository.getDictInfoByModelId(modelId);
-    }
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java

@@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/KnowledgeServiceImpl.java

@@ -1,54 +0,0 @@
-package com.tencent.supersonic.chat.server.service.impl;
-
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.knowledge.SearchService;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
-import com.tencent.supersonic.chat.server.service.KnowledgeService;
-import com.tencent.supersonic.common.pojo.enums.DictWordType;
-import java.util.List;
-import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-@Service
-@Slf4j
-public class KnowledgeServiceImpl implements KnowledgeService {
-
-    public void updateSemanticKnowledge(List<DictWord> natures) {
-
-        List<DictWord> prefixes = natures.stream()
-                .filter(entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
-                .collect(Collectors.toList());
-
-        for (DictWord nature : prefixes) {
-            HanlpHelper.addToCustomDictionary(nature);
-        }
-
-        List<DictWord> suffixes = natures.stream()
-                .filter(entry -> entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
-                .collect(Collectors.toList());
-
-        SearchService.loadSuffix(suffixes);
-    }
-
-    public void reloadAllData(List<DictWord> natures) {
-        // 1. reload custom knowledge
-        try {
-            HanlpHelper.reloadCustomDictionary();
-        } catch (Exception e) {
-            log.error("reloadCustomDictionary error", e);
-        }
-
-        // 2. update online knowledge
-        updateOnlineKnowledge(natures);
-    }
-
-    public void updateOnlineKnowledge(List<DictWord> natures) {
-        try {
-            updateSemanticKnowledge(natures);
-        } catch (Exception e) {
-            log.error("updateSemanticKnowledge error", e);
-        }
-    }
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java

@@ -2,12 +2,12 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -21,9 +21,9 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.api.pojo.response.QueryState;
 import com.tencent.supersonic.chat.core.agent.Agent;
 import com.tencent.supersonic.chat.core.corrector.SemanticCorrector;
-import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
-import com.tencent.supersonic.chat.core.knowledge.SearchService;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.SearchService;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.mapper.SchemaMapper;
 import com.tencent.supersonic.chat.core.parser.SemanticParser;
 import com.tencent.supersonic.chat.core.plugin.Plugin;
@@ -33,8 +33,8 @@ import com.tencent.supersonic.chat.core.query.QueryManager;
 import com.tencent.supersonic.chat.core.query.SemanticQuery;
 import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery;
 import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
-import com.tencent.supersonic.chat.core.utils.NatureHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.chat.core.utils.SimilarQueryManager;
 import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO;
 import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
@@ -671,7 +671,7 @@ public class QueryServiceImpl implements QueryService {
         }
         //search from prefixSearch
         List<HanlpMapResult> hanlpMapResultList = SearchService.prefixSearch(dimensionValueReq.getValue(),
-                2000, dimensionValueReq.getAgentId(), detectModelIds);
+                2000, detectModelIds);
         HanlpHelper.transLetterOriginal(hanlpMapResultList);
         return hanlpMapResultList.stream()
                 .filter(o -> {

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
 import com.tencent.supersonic.chat.api.pojo.request.RecommendReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java

@@ -5,7 +5,7 @@ import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java

@@ -2,9 +2,8 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.google.common.collect.Lists;
-import com.hankcs.hanlp.seg.common.Term;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -12,22 +11,24 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.SearchResult;
 import com.tencent.supersonic.chat.core.agent.Agent;
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
-import com.tencent.supersonic.chat.core.knowledge.ViewInfoStat;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat;
 import com.tencent.supersonic.chat.core.mapper.MapperHelper;
 import com.tencent.supersonic.chat.core.mapper.MatchText;
 import com.tencent.supersonic.chat.core.mapper.ModelWithSemanticType;
 import com.tencent.supersonic.chat.core.mapper.SearchMatchStrategy;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
-import com.tencent.supersonic.chat.core.utils.NatureHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.chat.server.service.AgentService;
 import com.tencent.supersonic.chat.server.service.ChatService;
 import com.tencent.supersonic.chat.server.service.ConfigService;
 import com.tencent.supersonic.chat.server.service.SearchService;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.headless.server.service.KnowledgeService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.beans.BeanUtils;
@@ -72,6 +73,9 @@ public class SearchServiceImpl implements SearchService {
     @Autowired
     private ConfigService configService;
 
+    @Autowired
+    private KnowledgeService knowledgeService;
+
     @Override
     public List<SearchResult> search(QueryReq queryReq) {
         // 1. check search enable
@@ -90,7 +94,7 @@ public class SearchServiceImpl implements SearchService {
         final Map<Long, String> modelToName = semanticSchemaDb.getViewIdToName();
 
         // 3.detect by segment
-        List<Term> originals = HanlpHelper.getTerms(queryText);
+        List<S2Term> originals = knowledgeService.getTerms(queryText);
         log.info("hanlp parse result: {}", originals);
         MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
         Set<Long> detectModelIds = mapperHelper.getViewIds(queryReq.getModelId(), agentService.getAgent(agentId));
@@ -141,7 +145,7 @@ public class SearchServiceImpl implements SearchService {
         return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
     }
 
-    private List<Long> getPossibleModels(QueryReq queryCtx, List<Term> originals,
+    private List<Long> getPossibleModels(QueryReq queryCtx, List<S2Term> originals,
                                          ViewInfoStat modelStat, Long webModelId) {
 
         if (Objects.nonNull(webModelId) && webModelId > 0) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java

@@ -1,10 +1,10 @@
 package com.tencent.supersonic.chat.server.service.impl;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.knowledge.builder.WordBuilderFactory;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import lombok.extern.slf4j.Slf4j;

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java

@@ -4,7 +4,7 @@ import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.server.util;
 
 import com.tencent.supersonic.chat.core.corrector.SemanticCorrector;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.mapper.SchemaMapper;
 import com.tencent.supersonic.chat.core.parser.SemanticParser;
 import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java

@@ -1,247 +0,0 @@
-package com.tencent.supersonic.chat.server.util;
-
-import com.github.pagehelper.PageInfo;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
-import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
-import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
-import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
-import com.tencent.supersonic.chat.core.config.DefaultMetric;
-import com.tencent.supersonic.chat.core.config.Dim4Dict;
-import com.tencent.supersonic.chat.core.knowledge.DictUpdateMode;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO;
-import com.tencent.supersonic.chat.server.service.ConfigService;
-import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-
-@Component
-public class DictMetaHelper {
-
-    @Autowired
-    private ConfigService configService;
-    @Value("${model.internal.metric.suffix:internal_cnt}")
-    private String internalMetricNameSuffix;
-    @Value("${model.internal.day.number:2}")
-    private Integer internalMetricDays;
-    private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
-
-    public List<DimValueDO> generateDimValueInfo(DimValue2DictCommand dimValue2DictCommend) {
-        List<DimValueDO> dimValueDOList = new ArrayList<>();
-        DictUpdateMode updateMode = dimValue2DictCommend.getUpdateMode();
-        Set<Long> modelIds = new HashSet<>();
-        switch (updateMode) {
-            case OFFLINE_MODEL:
-                modelIds.addAll(dimValue2DictCommend.getModelIds());
-                dimValueDOList = generateDimValueInfoByModel(modelIds);
-                break;
-            case OFFLINE_FULL:
-                List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema();
-                if (CollectionUtils.isEmpty(modelSchemaDescList)) {
-                    break;
-                }
-
-                Map<Long, ViewSchema> modelIdAndDescPair = modelSchemaDescList.stream()
-                        .collect(Collectors.toMap(a -> a.getView().getId(), schema -> schema, (k1, k2) -> k1));
-                if (!CollectionUtils.isEmpty(modelIdAndDescPair)) {
-                    modelIds.addAll(modelIdAndDescPair.keySet());
-                    dimValueDOList = generateDimValueInfoByModel(modelIds);
-                    break;
-                }
-                break;
-            case REALTIME_ADD:
-                dimValueDOList = generateDimValueInfoByModelAndDim(dimValue2DictCommend.getModelAndDimPair());
-                break;
-            case NOT_SUPPORT:
-                throw new RuntimeException("illegal parameter for updateMode");
-            default:
-                break;
-        }
-
-        return dimValueDOList;
-    }
-
-    private List<DimValueDO> generateDimValueInfoByModelAndDim(Map<Long, List<Long>> modelAndDimMap) {
-        List<DimValueDO> dimValueDOList = new ArrayList<>();
-        if (CollectionUtils.isEmpty(modelAndDimMap)) {
-            return dimValueDOList;
-        }
-
-        List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema();
-        if (CollectionUtils.isEmpty(modelSchemaDescList)) {
-            return dimValueDOList;
-        }
-        Map<Long, ViewSchema> modelIdAndDescPair = modelSchemaDescList.stream()
-                .collect(Collectors.toMap(a -> a.getView().getId(), a -> a, (k1, k2) -> k1));
-
-        for (Long modelId : modelAndDimMap.keySet()) {
-            if (!modelIdAndDescPair.containsKey(modelId)) {
-                continue;
-            }
-            Map<Long, SchemaElement> dimIdAndDescPairAll;
-            dimIdAndDescPairAll = modelIdAndDescPair.get(modelId).getDimensions().stream()
-                    .collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1));
-            List<Long> dimIdReq = modelAndDimMap.get(modelId);
-            Map<Long, SchemaElement> dimIdAndDescPairReq = new HashMap<>();
-            for (Long dimId : dimIdReq) {
-                if (dimIdAndDescPairAll.containsKey(dimId)) {
-                    dimIdAndDescPairReq.put(dimId, dimIdAndDescPairAll.get(dimId));
-                }
-            }
-            fillDimValueDOList(dimValueDOList, modelId, dimIdAndDescPairReq);
-        }
-
-        return dimValueDOList;
-    }
-
-    private List<DimValueDO> generateDimValueInfoByModel(Set<Long> modelIds) {
-        List<DimValueDO> dimValueDOList = new ArrayList<>();
-        List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema(new ArrayList<>(modelIds));
-        if (CollectionUtils.isEmpty(modelSchemaDescList)) {
-            return dimValueDOList;
-        }
-
-        modelSchemaDescList.forEach(modelSchemaDesc -> {
-            Map<Long, SchemaElement> dimIdAndDescPair = modelSchemaDesc.getDimensions().stream()
-                    .collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1));
-            fillDimValueDOList(dimValueDOList, modelSchemaDesc.getView().getId(), dimIdAndDescPair);
-
-        });
-
-        return dimValueDOList;
-    }
-
-    private void fillDimValueDOList(List<DimValueDO> dimValueDOList, Long modelId,
-                                    Map<Long, SchemaElement> dimIdAndDescPair) {
-        ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(modelId);
-        if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatAggRichConfig())) {
-
-            ChatDefaultRichConfigResp chatDefaultConfig =
-                    chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
-
-            List<KnowledgeInfoReq> knowledgeAggInfo =
-                    chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos();
-
-            List<KnowledgeInfoReq> knowledgeDetailInfo =
-                    chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos();
-
-            fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
-            fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
-
-
-        }
-    }
-
-    private void fillKnowledgeDimValue(List<KnowledgeInfoReq> knowledgeInfos,
-                                       ChatDefaultRichConfigResp chatDefaultConfig,
-                                       List<DimValueDO> dimValueDOList,
-                                       Map<Long, SchemaElement> dimIdAndDescPair, Long modelId) {
-        Map<Long, DimensionResp> dimIdAndRespPair = queryDimensionRespByModelId(
-                new ArrayList<>(Arrays.asList(modelId)));
-        if (!CollectionUtils.isEmpty(knowledgeInfos)) {
-            List<Dim4Dict> dimensions = new ArrayList<>();
-            List<DefaultMetric> defaultMetricDescList = new ArrayList<>();
-            knowledgeInfos.stream()
-                    .filter(knowledgeInfo -> knowledgeInfo.getSearchEnable()
-                            && !CollectionUtils.isEmpty(dimIdAndDescPair)
-                            && dimIdAndDescPair.containsKey(knowledgeInfo.getItemId()))
-                    .forEach(knowledgeInfo -> {
-                        SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId());
-
-                        Long dimId = dimensionDesc.getId();
-                        String internalMetricName = "";
-                        if (Objects.nonNull(dimId)) {
-                            String datasourceBizName = queryDataSourceByDimId(dimId);
-                            internalMetricName = datasourceBizName + UNDERLINE + internalMetricNameSuffix;
-                        }
-
-                        if (Objects.isNull(chatDefaultConfig)) {
-                            defaultMetricDescList.add(new DefaultMetric(internalMetricName,
-                                    internalMetricDays, DAY));
-                        } else {
-                            String metric = internalMetricName;
-                            if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
-                                metric = chatDefaultConfig.getMetrics().get(0).getBizName();
-                            }
-                            defaultMetricDescList.add(new DefaultMetric(metric,
-                                    chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod()));
-
-                        }
-
-                        String bizName = dimensionDesc.getBizName();
-                        Dim4Dict dim4Dict = new Dim4Dict();
-                        dim4Dict.setDimId(knowledgeInfo.getItemId());
-                        dim4Dict.setBizName(bizName);
-                        if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) {
-                            KnowledgeAdvancedConfig knowledgeAdvancedConfig
-                                    = knowledgeInfo.getKnowledgeAdvancedConfig();
-                            BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict);
-
-                            if (Objects.nonNull(dimIdAndRespPair)
-                                    && dimIdAndRespPair.containsKey(dim4Dict.getDimId())) {
-                                String datasourceFilterSql = dimIdAndRespPair.get(
-                                        dim4Dict.getDimId()).getModelFilterSql();
-                                if (StringUtils.isNotEmpty(datasourceFilterSql)) {
-                                    dim4Dict.getRuleList().add(datasourceFilterSql);
-                                }
-
-                            }
-                        }
-                        dimensions.add(dim4Dict);
-
-                    });
-
-            if (!CollectionUtils.isEmpty(dimensions)) {
-                DimValueDO dimValueDO = new DimValueDO()
-                        .setModelId(modelId)
-                        .setDefaultMetricIds(defaultMetricDescList)
-                        .setDimensions(dimensions);
-                dimValueDOList.add(dimValueDO);
-            }
-        }
-    }
-
-    private Map<Long, DimensionResp> queryDimensionRespByModelId(List<Long> modelIds) {
-        Map<Long, DimensionResp> dimIdAndRespPair = new HashMap<>();
-        PageDimensionReq pageDimensionCmd = new PageDimensionReq();
-        pageDimensionCmd.setModelIds(modelIds);
-        PageInfo<DimensionResp> dimensionPage = semanticInterpreter.getDimensionPage(pageDimensionCmd);
-        if (Objects.nonNull(dimensionPage) && !CollectionUtils.isEmpty(dimensionPage.getList())) {
-            List<DimensionResp> dimList = dimensionPage.getList();
-            dimIdAndRespPair = dimList.stream().collect(Collectors.toMap(DimensionResp::getId, v -> v, (v1, v2) -> v2));
-        }
-        return dimIdAndRespPair;
-    }
-
-    private String queryDataSourceByDimId(Long id) {
-        PageDimensionReq pageDimensionCmd = new PageDimensionReq();
-        pageDimensionCmd.setId(id.toString());
-        PageInfo<DimensionResp> dimensionPage = semanticInterpreter.getDimensionPage(pageDimensionCmd);
-        if (Objects.nonNull(dimensionPage) && !CollectionUtils.isEmpty(dimensionPage.getList())) {
-            List<DimensionResp> list = dimensionPage.getList();
-            return list.get(0).getModelBizName();
-        }
-        return "";
-    }
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictTaskConverter.java

@@ -1,46 +0,0 @@
-package com.tencent.supersonic.chat.server.util;
-
-import com.google.common.base.Strings;
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueInfo;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.common.util.JsonUtil;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.util.Date;
-import java.util.List;
-
-public class DictTaskConverter {
-
-    private static String dateTimeFormatter = "yyyyMMddHHmmss";
-
-    public static DictTaskDO generateDimValueDictTaskDO(DimValue2DictCommand dimValue2DictCommend, User user) {
-        DictTaskDO taskPO = new DictTaskDO();
-        Date createAt = new Date();
-        String date = DateTimeFormatter.ofPattern(dateTimeFormatter)
-                .format(createAt.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime());
-        String creator = Strings.isNullOrEmpty(user.getName()) ? "" : user.getName();
-        String updateMode = dimValue2DictCommend.getUpdateMode().getValue();
-        String name = String.format("DimValue_dic_%s_%s_%s", updateMode, creator, date);
-        taskPO.setName(name);
-
-        taskPO.setCreatedAt(createAt);
-        taskPO.setCommand(JsonUtil.toString(dimValue2DictCommend));
-        taskPO.setStatus(TaskStatusEnum.PENDING.getCode());
-        taskPO.setCreatedBy(creator);
-
-        return taskPO;
-    }
-
-    public static DictConfig dictConfPO2Config(DictConfDO dictConfDO) {
-        DictConfig dictConfig = new DictConfig();
-        dictConfig.setModelId(dictConfDO.getModelId());
-        List<DimValueInfo> dimValueInfos = JsonUtil.toList(dictConfDO.getDimValueInfos(), DimValueInfo.class);
-        dictConfig.setDimValueInfoList(dimValueInfos);
-        return dictConfig;
-    }
-}

chat/server/src/main/resources/mapper/DictConfMapper.xml

@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
-        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-
-<mapper namespace="com.tencent.supersonic.chat.server.persistence.mapper.DictConfMapper">
-
-    <resultMap id="DictConfPO"
-               type="com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO">
-        <id column="id" property="id"/>
-        <result column="model_id" property="modelId"/>
-        <result column="dim_value_infos" property="dimValueInfos"/>
-        <result column="created_at" property="createdAt"/>
-        <result column="updated_at" property="updatedAt"/>
-        <result column="created_by" property="createdBy"/>
-        <result column="updated_by" property="updatedBy"/>
-    </resultMap>
-
-    <insert id="createDictConf">
-        insert into s2_dictionary
-        (`domain_id`, dim_value_infos, created_at, updated_at, created_by, updated_by)
-        values
-        (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy})
-    </insert>
-
-    <insert id="upsertDictInfo">
-         insert into s2_dictionary
-        (`model_id`, dim_value_infos, created_at, updated_at, created_by, updated_by)
-        values
-        (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy})
-        on duplicate key update
-        dim_value_infos = #{dimValueInfos},
-        updated_at = #{updatedAt},
-        updated_by = #{updatedBy}
-    </insert>
-
-    <update id="editDictConf">
-        update s2_dictionary
-        set dim_value_infos = #{dimValueInfos},
-        updated_at = #{updatedAt},
-        updated_by = #{updatedBy}
-        where model_id = #{modelId}
-        and status = 0
-    </update>
-
-    <select id="getDictInfoByModelId" resultMap="DictConfPO">
-        select *
-        from s2_dictionary
-        where model_id = #{modelId}
-        and status = 0
-    </select>
-
-
-</mapper>

chat/server/src/main/resources/mapper/DictTaskMapper.xml

@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
-        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-
-<mapper namespace="com.tencent.supersonic.chat.server.persistence.mapper.DictTaskMapper">
-
-    <resultMap id="DimValueDictTaskPO"
-               type="com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO">
-        <id column="id" property="id"/>
-        <result column="name" property="name"/>
-        <result column="description" property="description"/>
-        <result column="command" property="command"/>
-        <result column="command_md5" property="commandMd5"/>
-        <result column="dimension_ids" property="dimIds"/>
-        <result column="status" property="status"/>
-        <result column="created_by" property="createdBy"/>
-        <result column="created_at" property="createdAt"/>
-        <result column="progress" property="progress"/>
-        <result column="elapsed_ms" property="elapsedMs"/>
-    </resultMap>
-
-    <insert id="createDimValueTask" useGeneratedKeys="true" keyProperty="id">
-        insert into s2_dictionary_task
-        (`name`, description, command, command_md5, dimension_ids, status, created_by, progress, elapsed_ms)
-        values
-        (#{name}, #{description}, #{command}, #{commandMd5}, #{dimIds}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
-    </insert>
-
-    <update id="updateTaskStatus">
-        update s2_dictionary_task
-        <set>
-            <if test="description != null and description !=''">
-                description = #{description},
-            </if>
-            <if test="status != null">
-                status = #{status},
-            </if>
-            <if test="dimIds != null">
-                dimension_ids = #{dimIds},
-            </if>
-            <if test="progress != null">
-                progress = #{progress},
-            </if>
-            <if test="elapsedMs != null">
-                elapsed_ms = #{elapsedMs},
-            </if>
-
-        </set>
-        where id = #{id}
-    </update>
-
-    <select id="searchDictTaskList" resultMap="DimValueDictTaskPO">
-        select *
-        from s2_dictionary_task
-        <where>
-            <if test="id != null and id != ''">
-                and id = #{id}
-            </if>
-            <if test="name != null and name !=''">
-                and `name` like  CONCAT('%', #{name}, '%')
-            </if>
-            <if test="createdBy != null and createdBy !=''">
-                and created_by = #{createdBy}
-            </if>
-            <if test="createdAt != null and createdAt !=''">
-                and created_at &gt;= #{createdAt}
-            </if>
-            <if test="status != null">
-                and status= #{status.code}
-            </if>
-        </where>
-        order by created_at desc
-    </select>
-
-</mapper>

chat/server/src/test/java/com/tencent/supersonic/chat/server/test/context/SemanticParseObjectHelper.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.server.test.context;
 
 import com.google.gson.Gson;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;