headless integrates knowledge (#722)

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SchemaElementMatch.java

@@ -1,5 +1,6 @@
 package com.tencent.supersonic.chat.api.pojo;
 
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java

@@ -9,6 +9,7 @@ import com.tencent.supersonic.common.pojo.Order;
 import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
 import com.tencent.supersonic.common.pojo.enums.FilterType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import lombok.Data;
 
 import java.util.ArrayList;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticSchema.java

@@ -1,5 +1,7 @@
 package com.tencent.supersonic.chat.api.pojo;
 
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import org.springframework.util.CollectionUtils;
 
 import java.io.Serializable;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/ViewSchema.java

@@ -1,6 +1,8 @@
 package com.tencent.supersonic.chat.api.pojo;
 
 import com.tencent.supersonic.headless.api.pojo.QueryConfig;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
 import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
 import lombok.Data;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictLatestTaskReq.java

@@ -1,23 +0,0 @@
-package com.tencent.supersonic.chat.api.pojo.request;
-
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.ToString;
-
-import javax.validation.constraints.NotNull;
-import java.util.List;
-
-import static java.time.LocalDate.now;
-
-@ToString
-@Data
-@NoArgsConstructor
-public class DictLatestTaskReq {
-
-    @NotNull
-    private Long modelId;
-
-    private List<Long> dimIds;
-
-    private String createdAt = now().plusDays(-4).toString();
-}

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/DictTaskFilterReq.java

@@ -1,20 +0,0 @@
-package com.tencent.supersonic.chat.api.pojo.request;
-
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import lombok.Data;
-import lombok.ToString;
-
-@ToString
-@Data
-public class DictTaskFilterReq {
-
-    private Long id;
-
-    private String name;
-
-    private String createdBy;
-
-    private String createdAt;
-
-    private TaskStatusEnum status;
-}

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/request/QueryDataReq.java

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.api.pojo.request;
 
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.common.pojo.DateConf;
 import java.util.HashSet;
 import java.util.Set;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/ChatDefaultRichConfigResp.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.api.pojo.response;
 
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.enums.TimeMode;
 import lombok.Data;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/EntityRichInfoResp.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.api.pojo.response;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 
 import java.util.List;
 import lombok.Data;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/QueryResult.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.api.pojo.response;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.common.pojo.QueryAuthorization;
 import com.tencent.supersonic.common.pojo.QueryColumn;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/RecommendResp.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.api.pojo.response;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import lombok.Data;
 
 import java.util.List;

chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/response/SearchResult.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.api.pojo.response;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import java.util.Objects;
 import lombok.Builder;
 import lombok.Data;

chat/core/pom.xml

@@ -21,70 +21,6 @@
             <groupId>org.springframework</groupId>
             <artifactId>spring-context</artifactId>
         </dependency>
-
-        <dependency>
-            <groupId>com.hankcs</groupId>
-            <artifactId>hanlp</artifactId>
-            <version>${hanlp.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-client</artifactId>
-            <version>${hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.zookeeper</groupId>
-                    <artifactId>zookeeper</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>${hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.zookeeper</groupId>
-                    <artifactId>zookeeper</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <version>${hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.zookeeper</groupId>
-                    <artifactId>zookeeper</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.curator</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>

chat/core/src/main/java/com/tencent/supersonic/chat/core/config/ChatLocalFileConfig.java

@@ -1,7 +1,9 @@
 package com.tencent.supersonic.chat.core.config;
 
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
+
 import java.io.FileNotFoundException;
+
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
@@ -11,7 +13,7 @@ import org.springframework.context.annotation.Configuration;
 @Data
 @Configuration
 @Slf4j
-public class LocalFileConfig {
+public class ChatLocalFileConfig {
 
 
     @Value("${dict.directory.latest:/data/dictionary/custom}")

chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/BaseSemanticCorrector.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.core.corrector;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;

chat/core/src/main/java/com/tencent/supersonic/chat/core/corrector/WhereCorrector.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.corrector;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
+import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;

chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DictConfig.java

@@ -1,13 +0,0 @@
-package com.tencent.supersonic.chat.core.knowledge;
-
-import java.util.List;
-import lombok.Data;
-
-
-@Data
-public class DictConfig {
-
-    private Long modelId;
-
-    private List<DimValueInfo> dimValueInfoList;
-}

chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValue2DictCommand.java

@@ -1,18 +0,0 @@
-package com.tencent.supersonic.chat.core.knowledge;
-
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import lombok.Data;
-
-@Data
-public class DimValue2DictCommand {
-
-    private DictUpdateMode updateMode;
-
-    private List<Long> modelIds;
-
-    private Map<Long, List<Long>> modelAndDimPair = new HashMap<>();
-}

chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueDictInfo.java

@@ -1,31 +0,0 @@
-package com.tencent.supersonic.chat.core.knowledge;
-
-
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-
-import java.util.Date;
-import java.util.Set;
-
-import lombok.Data;
-
-@Data
-public class DimValueDictInfo {
-
-    private Long id;
-
-    private String name;
-
-    private String description;
-
-    private String command;
-
-    private TaskStatusEnum status;
-
-    private String createdBy;
-
-    private Date createdAt;
-
-    private Long elapsedMs;
-
-    private Set<Long> dimIds;
-}

chat/core/src/main/java/com/tencent/supersonic/chat/core/knowledge/DimValueInfo.java

@@ -1,26 +0,0 @@
-package com.tencent.supersonic.chat.core.knowledge;
-
-
-import com.tencent.supersonic.common.pojo.enums.TypeEnums;
-import java.util.List;
-import javax.validation.constraints.NotNull;
-
-public class DimValueInfo {
-
-    /**
-     * metricId、DimensionId、domainId
-     */
-    private Long itemId;
-
-    /**
-     * type: IntentionTypeEnum
-     * temporarily only supports dimension-related information
-     */
-    @NotNull
-    private TypeEnums type = TypeEnums.DIMENSION;
-
-    private List<String> blackList;
-    private List<String> whiteList;
-    private List<String> ruleList;
-    private Boolean isDictInfo;
-}

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMapper.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.chat.core.mapper;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/BaseMatchStrategy.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.chat.core.mapper;
 
-import com.hankcs.hanlp.seg.common.Term;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
-import com.tencent.supersonic.chat.core.utils.NatureHelper;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
+import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -28,7 +28,8 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
     private MapperHelper mapperHelper;
 
     @Override
-    public Map<MatchText, List<T>> match(QueryContext queryContext, List<Term> terms, Set<Long> detectViewIds) {
+    public Map<MatchText, List<T>> match(QueryContext queryContext, List<S2Term> terms,
+                                         Set<Long> detectViewIds) {
         String text = queryContext.getQueryText();
         if (Objects.isNull(terms) || StringUtils.isEmpty(text)) {
             return null;
@@ -43,7 +44,7 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
         return result;
     }
 
-    public List<T> detect(QueryContext queryContext, List<Term> terms, Set<Long> detectModelIds) {
+    public List<T> detect(QueryContext queryContext, List<S2Term> terms, Set<Long> detectModelIds) {
         Map<Integer, Integer> regOffsetToLength = getRegOffsetToLength(terms);
         String text = queryContext.getQueryText();
         Set<T> results = new HashSet<>();
@@ -72,9 +73,10 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
         return;
     }
 
-    public Map<Integer, Integer> getRegOffsetToLength(List<Term> terms) {
+    public Map<Integer, Integer> getRegOffsetToLength(List<S2Term> terms) {
-        return terms.stream().sorted(Comparator.comparing(Term::length))
+        return terms.stream().sorted(Comparator.comparing(S2Term::length))
-                .collect(Collectors.toMap(Term::getOffset, term -> term.word.length(), (value1, value2) -> value2));
+                .collect(Collectors.toMap(S2Term::getOffset, term -> term.word.length(),
+                        (value1, value2) -> value2));
     }
 
     public void selectResultInOneRound(Set<T> existResults, List<T> oneRoundResults) {
@@ -102,7 +104,7 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
         }
     }
 
-    public List<T> getMatches(QueryContext queryContext, List<Term> terms) {
+    public List<T> getMatches(QueryContext queryContext, List<S2Term> terms) {
         Set<Long> viewIds = mapperHelper.getViewIds(queryContext.getViewId(), queryContext.getAgent());
         terms = filterByViewId(terms, viewIds);
         Map<MatchText, List<T>> matchResult = match(queryContext, terms, viewIds);
@@ -120,7 +122,7 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
         return matches;
     }
 
-    public List<Term> filterByViewId(List<Term> terms, Set<Long> viewIds) {
+    public List<S2Term> filterByViewId(List<S2Term> terms, Set<Long> viewIds) {
         logTerms(terms);
         if (CollectionUtils.isNotEmpty(viewIds)) {
             terms = terms.stream().filter(term -> {
@@ -136,11 +138,11 @@ public abstract class BaseMatchStrategy<T> implements MatchStrategy<T> {
         return terms;
     }
 
-    public void logTerms(List<Term> terms) {
+    public void logTerms(List<S2Term> terms) {
         if (CollectionUtils.isEmpty(terms)) {
             return;
         }
-        for (Term term : terms) {
+        for (S2Term term : terms) {
             log.debug("word:{},nature:{},frequency:{}", term.word, term.nature.toString(), term.getFrequency());
         }
     }

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/DatabaseMatchStrategy.java

@@ -1,10 +1,10 @@
 package com.tencent.supersonic.chat.core.mapper;
 
-import com.hankcs.hanlp.seg.common.Term;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
+import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.common.pojo.Constants;
 import lombok.extern.slf4j.Slf4j;
@@ -36,7 +36,7 @@ public class DatabaseMatchStrategy extends BaseMatchStrategy<DatabaseMapResult>
     private List<SchemaElement> allElements;
 
     @Override
-    public Map<MatchText, List<DatabaseMapResult>> match(QueryContext queryContext, List<Term> terms,
+    public Map<MatchText, List<DatabaseMapResult>> match(QueryContext queryContext, List<S2Term> terms,
             Set<Long> detectModelIds) {
         this.allElements = getSchemaElements(queryContext);
         return super.match(queryContext, terms, detectModelIds);

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMapper.java

@@ -1,13 +1,13 @@
 package com.tencent.supersonic.chat.core.mapper;
 
 import com.alibaba.fastjson.JSONObject;
-import com.hankcs.hanlp.seg.common.Term;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
-import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder;
+import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.embedding.Retrieval;
 import java.util.List;
@@ -26,7 +26,7 @@ public class EmbeddingMapper extends BaseMapper {
     public void doMap(QueryContext queryContext) {
         //1. query from embedding by queryText
         String queryText = queryContext.getQueryText();
-        List<Term> terms = HanlpHelper.getTerms(queryText);
+        List<S2Term> terms = HanlpHelper.getTerms(queryText);
 
         EmbeddingMatchStrategy matchStrategy = ContextUtils.getBean(EmbeddingMatchStrategy.class);
         List<EmbeddingResult> matchResults = matchStrategy.getMatches(queryContext, terms);

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EmbeddingMatchStrategy.java

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.mapper;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.knowledge.EmbeddingResult;
+import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.common.config.EmbeddingConfig;
 import com.tencent.supersonic.common.pojo.Constants;

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/EntityMapper.java

@@ -1,9 +1,9 @@
 package com.tencent.supersonic.chat.core.mapper;
 
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/HanlpDictMatchStrategy.java

@@ -1,9 +1,9 @@
 package com.tencent.supersonic.chat.core.mapper;
 
-import com.hankcs.hanlp.seg.common.Term;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
-import com.tencent.supersonic.chat.core.knowledge.SearchService;
+import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.SearchService;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.common.pojo.Constants;
 import java.util.HashMap;
@@ -35,7 +35,7 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
     private OptimizationConfig optimizationConfig;
 
     @Override
-    public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<Term> terms,
+    public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<S2Term> terms,
             Set<Long> detectModelIds) {
         String text = queryContext.getQueryText();
         if (Objects.isNull(terms) || StringUtils.isEmpty(text)) {
@@ -60,16 +60,15 @@ public class HanlpDictMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
     public void detectByStep(QueryContext queryContext, Set<HanlpMapResult> existResults, Set<Long> detectModelIds,
             Integer startIndex, Integer index, int offset) {
         String text = queryContext.getQueryText();
-        Integer agentId = queryContext.getAgentId();
         String detectSegment = text.substring(startIndex, index);
 
         // step1. pre search
         Integer oneDetectionMaxSize = optimizationConfig.getOneDetectionMaxSize();
         LinkedHashSet<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment, oneDetectionMaxSize,
-                agentId, detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
+                detectModelIds).stream().collect(Collectors.toCollection(LinkedHashSet::new));
         // step2. suffix search
         LinkedHashSet<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(detectSegment,
-                        oneDetectionMaxSize, agentId, detectModelIds).stream()
+                        oneDetectionMaxSize, detectModelIds).stream()
                 .collect(Collectors.toCollection(LinkedHashSet::new));
 
         hanlpMapResults.addAll(suffixHanlpMapResults);

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/KeywordMapper.java

@@ -1,15 +1,16 @@
 package com.tencent.supersonic.chat.core.mapper;
 
-import com.hankcs.hanlp.seg.common.Term;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
-import com.tencent.supersonic.chat.core.knowledge.DatabaseMapResult;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
-import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult;
+import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
+import com.tencent.supersonic.headless.server.service.KnowledgeService;
-import com.tencent.supersonic.chat.core.utils.NatureHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.common.util.ContextUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.util.CollectionUtils;
@@ -32,7 +33,8 @@ public class KeywordMapper extends BaseMapper {
     public void doMap(QueryContext queryContext) {
         String queryText = queryContext.getQueryText();
         //1.hanlpDict Match
-        List<Term> terms = HanlpHelper.getTerms(queryText);
+        KnowledgeService knowledgeService = ContextUtils.getBean(KnowledgeService.class);
+        List<S2Term> terms = knowledgeService.getTerms(queryText);
         HanlpDictMatchStrategy hanlpMatchStrategy = ContextUtils.getBean(HanlpDictMatchStrategy.class);
 
         List<HanlpMapResult> hanlpMapResults = hanlpMatchStrategy.getMatches(queryContext, terms);
@@ -46,7 +48,7 @@ public class KeywordMapper extends BaseMapper {
     }
 
     private void convertHanlpMapResultToMapInfo(List<HanlpMapResult> mapResults, QueryContext queryContext,
-            List<Term> terms) {
+            List<S2Term> terms) {
         if (CollectionUtils.isEmpty(mapResults)) {
             return;
         }

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MapperHelper.java

@@ -1,10 +1,10 @@
 package com.tencent.supersonic.chat.core.mapper;
 
 import com.hankcs.hanlp.algorithm.EditDistance;
-import com.hankcs.hanlp.seg.common.Term;
 import com.tencent.supersonic.chat.core.agent.Agent;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.utils.NatureHelper;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
+import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -36,8 +36,8 @@ public class MapperHelper {
         return index;
     }
 
-    public Integer getStepOffset(List<Term> termList, Integer index) {
+    public Integer getStepOffset(List<S2Term> termList, Integer index) {
-        List<Integer> offsetList = termList.stream().sorted(Comparator.comparing(Term::getOffset))
+        List<Integer> offsetList = termList.stream().sorted(Comparator.comparing(S2Term::getOffset))
                 .map(term -> term.getOffset()).collect(Collectors.toList());
 
         for (int j = 0; j < termList.size() - 1; j++) {

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/MatchStrategy.java

@@ -1,7 +1,8 @@
 package com.tencent.supersonic.chat.core.mapper;
 
-import com.hankcs.hanlp.seg.common.Term;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
+
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -12,6 +13,6 @@ import java.util.Set;
  */
 public interface MatchStrategy<T> {
 
-    Map<MatchText, List<T>> match(QueryContext queryContext, List<Term> terms, Set<Long> detectModelId);
+    Map<MatchText, List<T>> match(QueryContext queryContext, List<S2Term> terms, Set<Long> detectModelId);
 
 }

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/ModelWithSemanticType.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.core.mapper;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import java.io.Serializable;
 import lombok.Data;
 import lombok.ToString;

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/QueryFilterMapper.java

@@ -1,13 +1,13 @@
 package com.tencent.supersonic.chat.core.mapper;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
-import com.tencent.supersonic.chat.core.knowledge.builder.BaseWordBuilder;
+import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.common.pojo.Constants;
 import lombok.extern.slf4j.Slf4j;

chat/core/src/main/java/com/tencent/supersonic/chat/core/mapper/SearchMatchStrategy.java

@@ -1,9 +1,9 @@
 package com.tencent.supersonic.chat.core.mapper;
 
 import com.google.common.collect.Lists;
-import com.hankcs.hanlp.seg.common.Term;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
-import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
-import com.tencent.supersonic.chat.core.knowledge.SearchService;
+import com.tencent.supersonic.headless.core.knowledge.SearchService;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import java.util.List;
@@ -26,7 +26,7 @@ public class SearchMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
     private static final int SEARCH_SIZE = 3;
 
     @Override
-    public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<Term> originals,
+    public Map<MatchText, List<HanlpMapResult>> match(QueryContext queryContext, List<S2Term> originals,
             Set<Long> detectModelIds) {
         String text = queryContext.getQueryText();
         Map<Integer, Integer> regOffsetToLength = getRegOffsetToLength(originals);
@@ -52,9 +52,9 @@ public class SearchMatchStrategy extends BaseMatchStrategy<HanlpMapResult> {
 
                     if (StringUtils.isNotEmpty(detectSegment)) {
                         List<HanlpMapResult> hanlpMapResults = SearchService.prefixSearch(detectSegment,
-                                SearchService.SEARCH_SIZE, queryContext.getAgentId(), detectModelIds);
+                                SearchService.SEARCH_SIZE, detectModelIds);
                         List<HanlpMapResult> suffixHanlpMapResults = SearchService.suffixSearch(
-                                detectSegment, SEARCH_SIZE, queryContext.getAgentId(), detectModelIds);
+                                detectSegment, SEARCH_SIZE, detectModelIds);
                         hanlpMapResults.addAll(suffixHanlpMapResults);
                         // remove entity name where search
                         hanlpMapResults = hanlpMapResults.stream().filter(entry -> {

chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/QueryTypeParser.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.parser;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;

chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/plugin/PluginParser.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.parser.plugin;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;

chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/HeuristicViewResolver.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.parser.sql.llm;
 
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.chat.core.query.SemanticQuery;

chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/llm/LLMRequestService.java

@@ -1,16 +1,16 @@
 package com.tencent.supersonic.chat.core.parser.sql.llm;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.core.agent.Agent;
 import com.tencent.supersonic.chat.core.agent.AgentToolType;
 import com.tencent.supersonic.chat.core.agent.NL2SQLTool;
 import com.tencent.supersonic.chat.core.config.LLMParserConfig;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.parser.SatisfactionChecker;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMReq;

chat/core/src/main/java/com/tencent/supersonic/chat/core/parser/sql/rule/ContextInheritParser.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.parser.sql.rule;
 
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.core.parser.SemanticParser;
 import com.tencent.supersonic.chat.core.pojo.ChatContext;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;

chat/core/src/main/java/com/tencent/supersonic/chat/core/plugin/PluginManager.java

@@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.core.plugin;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.core.agent.Agent;
 import com.tencent.supersonic.chat.core.agent.AgentToolType;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/BaseSemanticQuery.java

@@ -6,7 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
 import com.tencent.supersonic.common.pojo.Aggregator;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/analytics/MetricAnalyzeQuery.java

@@ -2,14 +2,14 @@ package com.tencent.supersonic.chat.core.query.llm.analytics;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.api.pojo.response.QueryState;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.query.QueryManager;
 import com.tencent.supersonic.chat.core.query.llm.LLMSemanticQuery;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/llm/s2sql/LLMSqlQuery.java

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.core.query.llm.s2sql;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.api.pojo.response.QueryState;
 import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/plugin/PluginSemanticQuery.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.core.query.plugin;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/QueryMatcher.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.query.rule;
 
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
 import lombok.Data;
 import lombok.ToString;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/RuleSemanticQuery.java

@@ -2,16 +2,16 @@
 package com.tencent.supersonic.chat.core.query.rule;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.api.pojo.response.QueryState;
 import com.tencent.supersonic.chat.core.config.OptimizationConfig;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.pojo.ChatContext;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
 import com.tencent.supersonic.chat.core.query.BaseSemanticQuery;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricFilterQuery.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.core.query.rule.metric;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricGroupByQuery.java

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.core.query.rule.metric;
 
 import org.springframework.stereotype.Component;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION;
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricModelQuery.java

@@ -4,7 +4,7 @@ import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.Optio
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_MOST;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import org.springframework.stereotype.Component;
 @Component

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricSemanticQuery.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.chat.core.query.rule.metric;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.METRIC;
 
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTagQuery.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.query.rule.metric;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY;
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/metric/MetricTopNQuery.java

@@ -1,13 +1,13 @@
 package com.tencent.supersonic.chat.core.query.rule.metric;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION;
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.OPTIONAL;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 import static com.tencent.supersonic.common.pojo.Constants.DESC_UPPER;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
 import com.tencent.supersonic.chat.core.pojo.ChatContext;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagDetailQuery.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.query.rule.tag;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.DIMENSION;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.DIMENSION;
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagFilterQuery.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.core.query.rule.tag;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.VALUE;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagIdQuery.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.core.query.rule.tag;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ID;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ID;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagListQuery.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.query.rule.tag;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
 import com.tencent.supersonic.chat.core.pojo.ChatContext;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/rule/tag/TagSemanticQuery.java

@@ -15,7 +15,7 @@ import java.time.LocalDate;
 import java.util.List;
 import java.util.Objects;
 
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
+import static com.tencent.supersonic.headless.api.pojo.SchemaElementType.ENTITY;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.OptionType.REQUIRED;
 import static com.tencent.supersonic.chat.core.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
 

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/BaseSemanticInterpreter.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.core.knowledge.semantic;
+package com.tencent.supersonic.chat.core.query.semantic;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/LocalSemanticInterpreter.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.core.knowledge.semantic;
+package com.tencent.supersonic.chat.core.query.semantic;
 
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/RemoteSemanticInterpreter.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.core.knowledge.semantic;
+package com.tencent.supersonic.chat.core.query.semantic;
 
 import com.github.pagehelper.PageInfo;
 import com.google.gson.Gson;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/SemanticInterpreter.java

@@ -1,4 +1,4 @@
-package com.tencent.supersonic.chat.core.knowledge.semantic;
+package com.tencent.supersonic.chat.core.query.semantic;
 
 import com.github.pagehelper.PageInfo;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;

chat/core/src/main/java/com/tencent/supersonic/chat/core/query/semantic/ViewSchemaBuilder.java

@@ -1,10 +1,10 @@
-package com.tencent.supersonic.chat.core.knowledge.semantic;
+package com.tencent.supersonic.chat.core.query.semantic;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
+import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
-import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
+import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
 import com.tencent.supersonic.headless.api.pojo.DimValueMap;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;

chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/ComponentFactory.java

@@ -1,6 +1,6 @@
 package com.tencent.supersonic.chat.core.utils;
 
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.parser.JavaLLMProxy;
 import com.tencent.supersonic.chat.core.parser.LLMProxy;
 import com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver;

chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/DictQueryHelper.java

@@ -1,219 +0,0 @@
-package com.tencent.supersonic.chat.core.utils;
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.core.config.DefaultMetric;
-import com.tencent.supersonic.chat.core.config.Dim4Dict;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
-import com.tencent.supersonic.common.pojo.Aggregator;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.DateConf;
-import com.tencent.supersonic.common.pojo.Filter;
-import com.tencent.supersonic.common.pojo.Order;
-import com.tencent.supersonic.common.pojo.QueryColumn;
-import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
-import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
-import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
-import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.util.Strings;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.StringJoiner;
-
-import static com.tencent.supersonic.common.pojo.Constants.AND_UPPER;
-import static com.tencent.supersonic.common.pojo.Constants.APOSTROPHE;
-import static com.tencent.supersonic.common.pojo.Constants.COMMA;
-import static com.tencent.supersonic.common.pojo.Constants.SPACE;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE_DOUBLE;
-
-@Slf4j
-@Component
-public class DictQueryHelper {
-
-    private static final Long MAX_FREQUENCY = 99999999L;
-    private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
-    @Value("${dimension.multi.value.split:#}")
-    private String dimMultiValueSplit;
-    @Value("${dimension.value.show:50}")
-    private Integer printDataShow;
-    @Value("${dimension.max.limit:3000000}")
-    private Long dimMaxLimit;
-    @Value("${dimension.white.weight:60000000}")
-    private Long dimensionWhiteWeight;
-
-    public List<String> fetchDimValueSingle(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict,
-            User user) {
-        List<String> data = new ArrayList<>();
-        QueryStructReq queryStructCmd = generateQueryStructCmd(modelId, defaultMetricDesc, dim4Dict);
-        try {
-            SemanticQueryResp semanticQueryResp = semanticInterpreter.queryByStruct(queryStructCmd, user);
-
-            log.info("fetchDimValueSingle sql:{}", semanticQueryResp.getSql());
-            String nature = String.format("_%d_%d", modelId, dim4Dict.getDimId());
-            String dimNameRewrite = rewriteDimName(semanticQueryResp.getColumns(), dim4Dict.getBizName());
-            data = generateFileData(semanticQueryResp.getResultList(), nature, dimNameRewrite,
-                    defaultMetricDesc.getBizName(), dim4Dict);
-            if (!CollectionUtils.isEmpty(data)) {
-                int size = (data.size() > printDataShow) ? printDataShow : data.size();
-                log.info("data:{}", data.subList(0, size));
-            } else {
-                log.warn("data is empty. nature:{}", nature);
-                if (Objects.nonNull(semanticQueryResp)) {
-                    log.warn("sql:{}", semanticQueryResp.getSql());
-                }
-            }
-
-        } catch (Exception e) {
-            log.warn("fetchDimValueSingle,e:", e);
-        }
-        return data;
-    }
-
-    private String rewriteDimName(List<QueryColumn> columns, String bizName) {
-        // metric parser join dimension style
-        String dimNameRewrite = bizName;
-
-        if (!CollectionUtils.isEmpty(columns)) {
-            for (QueryColumn column : columns) {
-                if (StringUtils.isNotEmpty(column.getNameEn())) {
-                    String nameEn = column.getNameEn();
-                    if (nameEn.endsWith(UNDERLINE_DOUBLE + bizName)) {
-                        dimNameRewrite = nameEn;
-                    }
-                }
-            }
-        }
-        return dimNameRewrite;
-    }
-
-    private List<String> generateFileData(List<Map<String, Object>> resultList, String nature, String dimName,
-            String metricName, Dim4Dict dim4Dict) {
-        List<String> data = new ArrayList<>();
-        if (CollectionUtils.isEmpty(resultList)) {
-            return data;
-        }
-        Map<String, Long> valueAndFrequencyPair = new HashMap<>(2000);
-        for (Map<String, Object> line : resultList) {
-
-            if (CollectionUtils.isEmpty(line) || !line.containsKey(dimName)
-                    || !line.containsKey(metricName)
-                    || line.get(dimName) == null) {
-                continue;
-            }
-
-            String dimValue = line.get(dimName).toString();
-            Object metricObject = line.get(metricName);
-            if (Strings.isNotEmpty(dimValue) && Objects.nonNull(metricObject)) {
-                Long metric = Math.round(Double.parseDouble(metricObject.toString()));
-                mergeMultivaluedValue(valueAndFrequencyPair, dimValue, metric);
-            }
-
-        }
-        constructDataLines(valueAndFrequencyPair, nature, data, dim4Dict);
-        return data;
-    }
-
-    private void constructDataLines(Map<String, Long> valueAndFrequencyPair, String nature,
-            List<String> data, Dim4Dict dim4Dict) {
-        valueAndFrequencyPair.forEach((dimValue, metric) -> {
-            if (metric > MAX_FREQUENCY) {
-                metric = MAX_FREQUENCY;
-            }
-            if (Strings.isNotEmpty(dimValue) && dimValue.contains(SPACE)) {
-                dimValue = dimValue.replace(SPACE, "#");
-            }
-            data.add(String.format("%s %s %s", dimValue, nature, metric));
-        });
-
-        if (Objects.nonNull(dim4Dict) && !CollectionUtils.isEmpty(dim4Dict.getWhiteList())) {
-            dim4Dict.getWhiteList().stream()
-                    .forEach(white -> data.add(String.format("%s %s %s", white, nature, dimensionWhiteWeight)));
-        }
-    }
-
-    private void mergeMultivaluedValue(Map<String, Long> valueAndFrequencyPair, String dimValue, Long metric) {
-        if (Strings.isEmpty(dimValue)) {
-            return;
-        }
-        Map<String, Long> tmp = new HashMap<>();
-        if (dimValue.contains(dimMultiValueSplit)) {
-            Arrays.stream(dimValue.split(dimMultiValueSplit))
-                    .forEach(dimValueSingle -> tmp.put(dimValueSingle, metric));
-        } else {
-            tmp.put(dimValue, metric);
-        }
-
-        for (String value : tmp.keySet()) {
-            long metricOld = valueAndFrequencyPair.containsKey(value) ? valueAndFrequencyPair.get(value) : 0L;
-            valueAndFrequencyPair.put(value, metric + metricOld);
-        }
-    }
-
-    private QueryStructReq generateQueryStructCmd(Long modelId, DefaultMetric defaultMetricDesc, Dim4Dict dim4Dict) {
-        QueryStructReq queryStructCmd = new QueryStructReq();
-
-        List<Filter> filters = generateFilters(dim4Dict, queryStructCmd);
-        queryStructCmd.setDimensionFilters(filters);
-
-        List<Aggregator> aggregators = new ArrayList<>();
-        aggregators.add(new Aggregator(defaultMetricDesc.getBizName(), AggOperatorEnum.SUM));
-        queryStructCmd.setAggregators(aggregators);
-
-        List<Order> orders = new ArrayList<>();
-        orders.add(new Order(defaultMetricDesc.getBizName(), Constants.DESC_UPPER));
-        queryStructCmd.setOrders(orders);
-
-        DateConf dateInfo = new DateConf();
-        dateInfo.setDateMode(DateConf.DateMode.RECENT);
-        log.debug("defaultMetric unit():{}", defaultMetricDesc.getUnit());
-        dateInfo.setUnit(defaultMetricDesc.getUnit());
-        dateInfo.setPeriod(defaultMetricDesc.getPeriod());
-        queryStructCmd.setDateInfo(dateInfo);
-
-        queryStructCmd.setLimit(dimMaxLimit);
-        queryStructCmd.setNeedAuth(false);
-        return queryStructCmd;
-
-    }
-
-    private List<Filter> generateFilters(Dim4Dict dim4Dict, QueryStructReq queryStructCmd) {
-        String whereStr = generateFilter(dim4Dict);
-        if (Strings.isEmpty(whereStr)) {
-            return new ArrayList<>();
-        }
-        Filter filter = new Filter("", FilterOperatorEnum.SQL_PART, whereStr);
-        List<Filter> filters = Objects.isNull(queryStructCmd.getOriginalFilter()) ? new ArrayList<>()
-                : queryStructCmd.getOriginalFilter();
-        filters.add(filter);
-        return filters;
-    }
-
-    private String generateFilter(Dim4Dict dim4Dict) {
-        if (Objects.isNull(dim4Dict)) {
-            return "";
-        }
-        StringJoiner joiner = new StringJoiner(SPACE + AND_UPPER + SPACE);
-
-        String dimName = dim4Dict.getBizName();
-        if (!CollectionUtils.isEmpty(dim4Dict.getBlackList())) {
-            StringJoiner joinerBlack = new StringJoiner(COMMA);
-            dim4Dict.getBlackList().stream().forEach(black -> joinerBlack.add(APOSTROPHE + black + APOSTROPHE));
-            joiner.add(String.format("(%s not in (%s))", dimName, joinerBlack.toString()));
-        }
-
-        if (!CollectionUtils.isEmpty(dim4Dict.getRuleList())) {
-            dim4Dict.getRuleList().stream().forEach(rule -> joiner.add(rule));
-        }
-
-        return joiner.toString();
-    }
-}

chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/QueryReqBuilder.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.utils;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.core.query.QueryManager;
 import com.tencent.supersonic.common.pojo.Aggregator;

chat/core/src/main/java/com/tencent/supersonic/chat/core/utils/SchemaMatchHelper.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.utils;
 
 import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

chat/core/src/main/resources/META-INF/spring.factories

@@ -1,2 +0,0 @@
-com.tencent.supersonic.chat.core.knowledge.FileHandler=\
-    com.tencent.supersonic.chat.core.knowledge.LocalFileHandler

chat/core/src/test/java/com/tencent/supersonic/chat/core/s2sql/LLMSqlParserTest.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.core.s2sql;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaValueMap;
+import com.tencent.supersonic.headless.api.pojo.SchemaValueMap;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;

chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/ApplicationStartedListener.java

@@ -1,11 +1,13 @@
 package com.tencent.supersonic.chat.server.listener;
 
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.server.service.KnowledgeService;
 import com.tencent.supersonic.chat.server.service.impl.SchemaService;
 import com.tencent.supersonic.chat.server.service.impl.WordService;
+
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+
+import com.tencent.supersonic.headless.server.service.KnowledgeService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;

chat/server/src/main/java/com/tencent/supersonic/chat/server/listener/SchemaDictUpdateListener.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.server.listener;
 
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
 import com.tencent.supersonic.chat.server.service.impl.SchemaService;
 import com.tencent.supersonic.common.pojo.Constants;
 import com.tencent.supersonic.common.pojo.DataEvent;

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictConfMapper.java

@@ -1,17 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.mapper;
-
-
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO;
-import org.apache.ibatis.annotations.Mapper;
-
-@Mapper
-public interface DictConfMapper {
-
-    Boolean createDictConf(DictConfDO dictConfDO);
-
-    Boolean editDictConf(DictConfDO dictConfDO);
-
-    Boolean upsertDictInfo(DictConfDO dictConfDO);
-
-    DictConfDO getDictInfoByModelId(Long modelId);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/mapper/DictTaskMapper.java

@@ -1,18 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.mapper;
-
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-
-import java.util.List;
-
-import org.apache.ibatis.annotations.Mapper;
-
-@Mapper
-public interface DictTaskMapper {
-
-    Long createDimValueTask(DictTaskDO dictTaskDO);
-
-    Boolean updateTaskStatus(DictTaskDO dictTaskDO);
-
-    List<DictTaskDO> searchDictTaskList(DictTaskFilterReq filter);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/DictRepository.java

@@ -1,19 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.repository;
-
-
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import java.util.List;
-
-public interface DictRepository {
-
-    Long createDimValueDictTask(DictTaskDO dictTaskDO);
-
-    Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO);
-
-    List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter);
-
-    DictConfig getDictInfoByModelId(Long modelId);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/repository/impl/DictRepositoryImpl.java

@@ -1,83 +0,0 @@
-package com.tencent.supersonic.chat.server.persistence.repository.impl;
-
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.chat.server.persistence.mapper.DictConfMapper;
-import com.tencent.supersonic.chat.server.persistence.mapper.DictTaskMapper;
-import com.tencent.supersonic.chat.server.persistence.repository.DictRepository;
-import com.tencent.supersonic.chat.server.util.DictTaskConverter;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.common.util.JsonUtil;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.BeanUtils;
-import org.springframework.stereotype.Repository;
-import org.springframework.util.CollectionUtils;
-
-
-@Repository
-@Slf4j
-public class DictRepositoryImpl implements DictRepository {
-
-    private final DictTaskMapper dictTaskMapper;
-    private final DictConfMapper dictConfMapper;
-
-    public DictRepositoryImpl(DictTaskMapper dictTaskMapper,
-                              DictConfMapper dictConfMapper) {
-        this.dictTaskMapper = dictTaskMapper;
-        this.dictConfMapper = dictConfMapper;
-    }
-
-    @Override
-    public Long createDimValueDictTask(DictTaskDO dictTaskDO) {
-        dictTaskMapper.createDimValueTask(dictTaskDO);
-        return dictTaskDO.getId();
-    }
-
-    @Override
-    public Boolean updateDictTaskStatus(Integer status, DictTaskDO dictTaskDO) {
-        dictTaskDO.setStatus(status);
-        Date createdAt = dictTaskDO.getCreatedAt();
-        long elapsedMs = System.currentTimeMillis() - createdAt.getTime();
-        dictTaskDO.setElapsedMs(elapsedMs);
-        dictTaskMapper.updateTaskStatus(dictTaskDO);
-        return true;
-    }
-
-    @Override
-    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter) {
-        List<DimValueDictInfo> dimValueDictDescList = new ArrayList<>();
-        log.info("filter:{}", filter);
-        List<DictTaskDO> dictTaskDOList = dictTaskMapper.searchDictTaskList(filter);
-        if (!CollectionUtils.isEmpty(dictTaskDOList)) {
-            dictTaskDOList.stream().forEach(dictTaskDO -> {
-                DimValueDictInfo dimValueDictDesc = new DimValueDictInfo();
-                BeanUtils.copyProperties(dictTaskDO, dimValueDictDesc);
-                dimValueDictDesc.setStatus(TaskStatusEnum.of(dictTaskDO.getStatus()));
-                if (StringUtils.isNotEmpty(dictTaskDO.getDimIds())) {
-                    Set<Long> dimIds = JsonUtil.toSet(dictTaskDO.getDimIds(), Long.class);
-                    dimValueDictDesc.setDimIds(dimIds);
-                }
-                dimValueDictDescList.add(dimValueDictDesc);
-            });
-        }
-        return dimValueDictDescList;
-    }
-
-    @Override
-    public DictConfig getDictInfoByModelId(Long modelId) {
-        DictConfDO dictConfDO = dictConfMapper.getDictInfoByModelId(modelId);
-        if (Objects.isNull(dictConfDO)) {
-            return null;
-        }
-        return DictTaskConverter.dictConfPO2Config(dictConfDO);
-    }
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/DimensionRecommendProcessor.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.chat.server.processor.execute;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
+import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRatioProcessor.java

@@ -11,14 +11,13 @@ import static com.tencent.supersonic.common.pojo.Constants.TIME_FORMAT;
 import static com.tencent.supersonic.common.pojo.Constants.WEEK;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.AggregateInfo;
 import com.tencent.supersonic.chat.api.pojo.response.MetricInfo;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.core.config.AggregatorConfig;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
 import com.tencent.supersonic.common.pojo.DateConf;
@@ -29,6 +28,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.pojo.enums.RatioOverType;
 import com.tencent.supersonic.common.util.ContextUtils;
 import com.tencent.supersonic.common.util.DateUtils;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import java.text.DecimalFormat;

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java

@@ -1,8 +1,8 @@
 package com.tencent.supersonic.chat.server.processor.execute;
 
 import com.alibaba.fastjson.JSONObject;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.QueryResult;

chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/parse/ParseInfoProcessor.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.server.processor.parse;
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatConfigController.java

@@ -8,7 +8,7 @@ import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
 import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
 import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.chat.server.service.ConfigService;
 import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/ChatQueryController.java

@@ -3,7 +3,7 @@ package com.tencent.supersonic.chat.server.rest;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
-import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/rest/KnowledgeController.java

@@ -1,114 +0,0 @@
-package com.tencent.supersonic.chat.server.rest;
-
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
-import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.server.listener.ApplicationStartedListener;
-import com.tencent.supersonic.chat.server.service.KnowledgeTaskService;
-import java.util.List;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.validation.Valid;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.PutMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-
-@RestController
-@RequestMapping("/api/chat/dict")
-public class KnowledgeController {
-
-    @Autowired
-    private KnowledgeTaskService knowledgeTaskService;
-
-    @Autowired
-    private ApplicationStartedListener applicationStartedListener;
-
-    /**
-     * addDictInfo
-     * write specific dimension values to the knowledge base
-     *
-     * @param dimValue2DictCommend
-     */
-    @PostMapping("/task")
-    public Long addDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend,
-                            HttpServletRequest request,
-                            HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.addDictTask(dimValue2DictCommend, user);
-    }
-
-    /**
-     * deleteDictInfo
-     * remove specific dimension values from the knowledge base
-     *
-     * @param dimValue2DictCommend
-     */
-    @PostMapping("/task/delete")
-    public Long deleteDictTask(@RequestBody DimValue2DictCommand dimValue2DictCommend,
-                               HttpServletRequest request,
-                               HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.deleteDictTask(dimValue2DictCommend, user);
-    }
-
-    /**
-     * searchDictTaskList
-     *
-     * @param filter
-     */
-    @PostMapping("/task/search")
-    public List<DimValueDictInfo> searchDictTaskList(@RequestBody DictTaskFilterReq filter,
-                                                     HttpServletRequest request,
-                                                     HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.searchDictTaskList(filter, user);
-    }
-
-    /**
-     * searchDictLatestTaskList
-     */
-    @PostMapping("/task/search/latest")
-    public List<DictLatestTaskResp> searchDictLatestTaskList(@RequestBody @Valid DictLatestTaskReq filter,
-                                                             HttpServletRequest request,
-                                                             HttpServletResponse response) {
-        User user = UserHolder.findUser(request, response);
-        return knowledgeTaskService.searchDictLatestTaskList(filter, user);
-    }
-
-    /**
-     * getDictRootPath
-     * get knowledge base file directory
-     *
-     * @return
-     */
-    @GetMapping("/rootPath")
-    public String getDictRootPath(HttpServletRequest request,
-                                  HttpServletResponse response) {
-        return knowledgeTaskService.getDictRootPath();
-    }
-
-    /**
-     * updateDimValue
-     * update in-memory dictionary files in real time
-     *
-     * @param request
-     * @param response
-     * @return
-     */
-    @PutMapping("/knowledge/dimValue")
-    public Boolean updateDimValue(HttpServletRequest request,
-                                  HttpServletResponse response) {
-        return applicationStartedListener.updateKnowledgeDimValue();
-    }
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeService.java

@@ -1,14 +0,0 @@
-package com.tencent.supersonic.chat.server.service;
-
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import java.util.List;
-
-public interface KnowledgeService {
-
-    void updateSemanticKnowledge(List<DictWord> natures);
-
-    void reloadAllData(List<DictWord> natures);
-
-    void updateOnlineKnowledge(List<DictWord> natures);
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/KnowledgeTaskService.java

@@ -1,24 +0,0 @@
-package com.tencent.supersonic.chat.server.service;
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import java.util.List;
-
-public interface KnowledgeTaskService {
-    Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
-
-    Long deleteDictTask(DimValue2DictCommand dimValue2DictCommend, User user);
-
-    List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user);
-
-    DictConfig getDictInfoByModelId(Long modelId);
-
-    String getDictRootPath();
-
-    List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq filter, User user);
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/QueryService.java

@@ -2,7 +2,7 @@ package com.tencent.supersonic.chat.server.service;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
-import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/SemanticService.java

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.service;
 
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
@@ -11,7 +11,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.chat.api.pojo.response.DataInfo;
 import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
 import com.tencent.supersonic.chat.api.pojo.response.ViewInfo;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
 import com.tencent.supersonic.chat.server.service.impl.SchemaService;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ChatKnowledgeTaskServiceImpl.java

@@ -1,264 +0,0 @@
-package com.tencent.supersonic.chat.server.service.impl;
-
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.request.DictLatestTaskReq;
-import com.tencent.supersonic.chat.api.pojo.request.DictTaskFilterReq;
-import com.tencent.supersonic.chat.api.pojo.response.DictLatestTaskResp;
-import com.tencent.supersonic.chat.core.config.DefaultMetric;
-import com.tencent.supersonic.chat.core.config.Dim4Dict;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DictUpdateMode;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueDictInfo;
-import com.tencent.supersonic.chat.core.knowledge.FileHandler;
-import com.tencent.supersonic.chat.core.utils.DictQueryHelper;
-import com.tencent.supersonic.chat.server.listener.ApplicationStartedListener;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO;
-import com.tencent.supersonic.chat.server.persistence.repository.DictRepository;
-import com.tencent.supersonic.chat.server.service.KnowledgeTaskService;
-import com.tencent.supersonic.chat.server.util.DictMetaHelper;
-import com.tencent.supersonic.chat.server.util.DictTaskConverter;
-import com.tencent.supersonic.common.pojo.Constants;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.common.util.JsonUtil;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
-
-@Slf4j
-@Service
-public class ChatKnowledgeTaskServiceImpl implements KnowledgeTaskService {
-
-    private final DictMetaHelper metaUtils;
-    private final DictQueryHelper dictQueryHelper;
-    private final FileHandler fileHandler;
-    private final DictRepository dictRepository;
-    private final ApplicationStartedListener applicationStartedListener;
-
-    @Value("${dict.flush.enable:true}")
-    private Boolean dictFlushEnable;
-    @Value("${dict.flush.daily.enable:true}")
-    private Boolean dictFlushDailyEnable;
-    @Value("${dict.file.type:txt}")
-    private String dictFileType;
-    private String dimValue = "DimValue_%d_%d";
-
-    public ChatKnowledgeTaskServiceImpl(DictMetaHelper metaUtils,
-                                    DictQueryHelper dictQueryHelper,
-                                    FileHandler fileHandler,
-                                    DictRepository dictRepository,
-                                    ApplicationStartedListener applicationStartedListener) {
-        this.metaUtils = metaUtils;
-        this.dictQueryHelper = dictQueryHelper;
-        this.fileHandler = fileHandler;
-        this.dictRepository = dictRepository;
-        this.applicationStartedListener = applicationStartedListener;
-    }
-
-    @Scheduled(cron = "${knowledge.dimension.value.cron:0 0 0 * * ?}")
-    public Boolean dailyDictTask() {
-        log.info("[dailyDictTask] start");
-        if (!dictFlushDailyEnable) {
-            log.info("dictFlushDailyEnable is false, now finish dailyDictTask");
-        }
-        DimValue2DictCommand dimValue2DictCommend = new DimValue2DictCommand();
-        dimValue2DictCommend.setUpdateMode(DictUpdateMode.OFFLINE_FULL);
-
-        User user = User.getFakeUser();
-        addDictTask(dimValue2DictCommend, user);
-        log.info("[dailyDictTask] finish");
-        return true;
-    }
-
-    @Override
-    public Long addDictTask(DimValue2DictCommand dimValue2DictCommend, User user) {
-        if (!dictFlushEnable) {
-            return 0L;
-        }
-
-        if (DictUpdateMode.REALTIME_DELETE.equals(dimValue2DictCommend.getUpdateMode())) {
-            return deleteDictTask(dimValue2DictCommend, user);
-        }
-
-        DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommend, user);
-        log.info("[addDictTask] dictTaskDO:{}", dictTaskDO);
-        // todo check dimension can not be searched
-
-        dictRepository.createDimValueDictTask(dictTaskDO);
-        runDictTask(dictTaskDO, user);
-
-        return dictTaskDO.getId();
-    }
-
-    public Long runDictTask(DictTaskDO dictTaskDO, User user) {
-        if (Objects.isNull(dictTaskDO)) {
-            return -1L;
-        }
-        DimValue2DictCommand command = JsonUtil.toObject(dictTaskDO.getCommand(), DimValue2DictCommand.class);
-        try {
-            //1. construct internal dictionary requirements
-            List<DimValueDO> dimValueDOList = metaUtils.generateDimValueInfo(command);
-            Set<Long> dimIds = generateDimSet(dimValueDOList);
-            dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
-            dictRepository.updateDictTaskStatus(TaskStatusEnum.RUNNING.getCode(), dictTaskDO);
-            log.debug("dimValueDOList:{}", dimValueDOList);
-            //2. query dimension value information
-            for (DimValueDO dimValueDO : dimValueDOList) {
-                Long modelId = dimValueDO.getModelId();
-                DefaultMetric defaultMetricDesc = dimValueDO.getDefaultMetricDescList().get(0);
-                for (Dim4Dict dim4Dict : dimValueDO.getDimensions()) {
-                    List<String> data = dictQueryHelper.fetchDimValueSingle(modelId, defaultMetricDesc, dim4Dict, user);
-                    //3. local file changes
-                    String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId,
-                            dim4Dict.getDimId());
-                    fileHandler.writeFile(data, fileName, false);
-                }
-            }
-            applicationStartedListener.updateKnowledgeDimValue();
-            log.debug("updateDictTaskStatus to SUCCESS");
-            dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
-        } catch (Exception e) {
-            log.warn("addDictInfo exception:", e);
-            dictRepository.updateDictTaskStatus(TaskStatusEnum.ERROR.getCode(), dictTaskDO);
-        }
-        return 1L;
-    }
-
-    private Set<Long> generateDimSet(List<DimValueDO> dimValueDOList) {
-        Set<Long> dimIds = new HashSet<>();
-        if (!CollectionUtils.isEmpty(dimValueDOList)) {
-            dimValueDOList.stream().forEach(dimValueDO -> {
-                if (!CollectionUtils.isEmpty(dimValueDO.getDimensions())) {
-                    dimValueDO.getDimensions().stream().forEach(dim4Dict -> dimIds.add(dim4Dict.getDimId()));
-                }
-            });
-        }
-        return dimIds;
-    }
-
-    @Override
-    public Long deleteDictTask(DimValue2DictCommand dimValue2DictCommand, User user) {
-        if (!dictFlushEnable) {
-            return 0L;
-        }
-        if (Objects.isNull(dimValue2DictCommand) || !DictUpdateMode.REALTIME_DELETE.equals(
-                dimValue2DictCommand.getUpdateMode())) {
-            throw new RuntimeException("illegal parameter");
-        }
-
-        DictTaskDO dictTaskDO = DictTaskConverter.generateDimValueDictTaskDO(dimValue2DictCommand, user);
-        log.info("[deleteDictTask] dictTaskDO:{}", dictTaskDO);
-        Set<Long> dimIds = generateDimSetFromCommand(dimValue2DictCommand.getModelAndDimPair());
-        dictTaskDO.setDimIds(JsonUtil.toString(dimIds));
-        dictRepository.createDimValueDictTask(dictTaskDO);
-
-        Map<Long, List<Long>> modelAndDimPair = dimValue2DictCommand.getModelAndDimPair();
-        if (CollectionUtils.isEmpty(modelAndDimPair)) {
-            return 0L;
-        }
-        for (Long modelId : modelAndDimPair.keySet()) {
-            if (CollectionUtils.isEmpty(modelAndDimPair.get(modelId))) {
-                continue;
-            }
-            for (Long dimId : modelAndDimPair.get(modelId)) {
-                String fileName = String.format(dimValue + Constants.DOT + dictFileType, modelId, dimId);
-                fileHandler.deleteDictFile(fileName);
-            }
-        }
-        applicationStartedListener.updateKnowledgeDimValue();
-        dictRepository.updateDictTaskStatus(TaskStatusEnum.SUCCESS.getCode(), dictTaskDO);
-        applicationStartedListener.updateKnowledgeDimValue();
-
-        return 1L;
-    }
-
-    private Set<Long> generateDimSetFromCommand(Map<Long, List<Long>> modelAndDimPair) {
-        Set<Long> dimIds = new HashSet<>();
-        if (!CollectionUtils.isEmpty(modelAndDimPair)) {
-            modelAndDimPair.forEach((k, v) -> dimIds.addAll(v));
-        }
-        return dimIds;
-    }
-
-    @Override
-    public String getDictRootPath() {
-        return fileHandler.getDictRootPath();
-    }
-
-    @Override
-    public List<DictLatestTaskResp> searchDictLatestTaskList(DictLatestTaskReq latestFilter, User user) {
-        DictTaskFilterReq filter = new DictTaskFilterReq();
-        BeanUtils.copyProperties(latestFilter, filter);
-        List<DimValueDictInfo> dimValueDictInfoList = searchDictTaskList(filter, user);
-        return extractLatestTask(dimValueDictInfoList, latestFilter.getDimIds());
-    }
-
-    private List<DictLatestTaskResp> extractLatestTask(List<DimValueDictInfo> dimValueDictInfoList, List<Long> dimIds) {
-        List<DictLatestTaskResp> dictLatestTaskRespList = new ArrayList<>();
-        Map<Long, DictLatestTaskResp> dimAndTaskPair = new HashMap<>(50);
-        for (DimValueDictInfo dimValueDictInfo : dimValueDictInfoList) {
-            //1. filter
-            if (Objects.isNull(dimValueDictInfo) || CollectionUtils.isEmpty(dimValueDictInfo.getDimIds())) {
-                continue;
-            }
-            if (!CollectionUtils.isEmpty(dimIds)) {
-                Set<Long> tmp = dimValueDictInfo.getDimIds();
-                tmp.retainAll(dimIds);
-                dimValueDictInfo.setDimIds(tmp);
-                if (tmp.size() <= 0) {
-                    continue;
-                }
-            }
-
-            // 2. extract
-            Set<Long> dimIdList = dimValueDictInfo.getDimIds();
-            for (Long dimId : dimIdList) {
-                DictLatestTaskResp dictLatestTaskResp = new DictLatestTaskResp();
-                if (!dimAndTaskPair.containsKey(dimId)) {
-                    BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
-                    dictLatestTaskResp.setDimId(dimId);
-                } else {
-                    DictLatestTaskResp dictLatestTaskExist = dimAndTaskPair.get(dimId);
-                    if (dictLatestTaskExist.getCreatedAt().before(dimValueDictInfo.getCreatedAt())) {
-                        BeanUtils.copyProperties(dimValueDictInfo, dictLatestTaskResp);
-                        dictLatestTaskResp.setDimId(dimId);
-                    } else {
-                        dictLatestTaskResp = dictLatestTaskExist;
-                    }
-                }
-                dimAndTaskPair.put(dimId, dictLatestTaskResp);
-            }
-
-        }
-
-        if (dimAndTaskPair.size() >= 0 && !CollectionUtils.isEmpty(dimAndTaskPair.values())) {
-            dimAndTaskPair.values().stream()
-                    .filter(v -> !v.getCommand().contains(DictUpdateMode.REALTIME_DELETE.name()))
-                    .forEach(v -> dictLatestTaskRespList.add(v));
-        }
-
-
-        return dictLatestTaskRespList;
-    }
-
-    @Override
-    public List<DimValueDictInfo> searchDictTaskList(DictTaskFilterReq filter, User user) {
-        return dictRepository.searchDictTaskList(filter);
-    }
-
-    @Override
-    public DictConfig getDictInfoByModelId(Long modelId) {
-        return dictRepository.getDictInfoByModelId(modelId);
-    }
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/ConfigServiceImpl.java

@@ -3,9 +3,9 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/KnowledgeServiceImpl.java

@@ -1,54 +0,0 @@
-package com.tencent.supersonic.chat.server.service.impl;
-
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.knowledge.SearchService;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
-import com.tencent.supersonic.chat.server.service.KnowledgeService;
-import com.tencent.supersonic.common.pojo.enums.DictWordType;
-import java.util.List;
-import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-@Service
-@Slf4j
-public class KnowledgeServiceImpl implements KnowledgeService {
-
-    public void updateSemanticKnowledge(List<DictWord> natures) {
-
-        List<DictWord> prefixes = natures.stream()
-                .filter(entry -> !entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
-                .collect(Collectors.toList());
-
-        for (DictWord nature : prefixes) {
-            HanlpHelper.addToCustomDictionary(nature);
-        }
-
-        List<DictWord> suffixes = natures.stream()
-                .filter(entry -> entry.getNatureWithFrequency().contains(DictWordType.SUFFIX.getType()))
-                .collect(Collectors.toList());
-
-        SearchService.loadSuffix(suffixes);
-    }
-
-    public void reloadAllData(List<DictWord> natures) {
-        // 1. reload custom knowledge
-        try {
-            HanlpHelper.reloadCustomDictionary();
-        } catch (Exception e) {
-            log.error("reloadCustomDictionary error", e);
-        }
-
-        // 2. update online knowledge
-        updateOnlineKnowledge(natures);
-    }
-
-    public void updateOnlineKnowledge(List<DictWord> natures) {
-        try {
-            updateSemanticKnowledge(natures);
-        } catch (Exception e) {
-            log.error("updateSemanticKnowledge error", e);
-        }
-    }
-
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/QueryServiceImpl.java

@@ -2,12 +2,12 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq;
+import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
 import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -21,9 +21,9 @@ import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
 import com.tencent.supersonic.chat.api.pojo.response.QueryState;
 import com.tencent.supersonic.chat.core.agent.Agent;
 import com.tencent.supersonic.chat.core.corrector.SemanticCorrector;
-import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
-import com.tencent.supersonic.chat.core.knowledge.SearchService;
+import com.tencent.supersonic.headless.core.knowledge.SearchService;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.mapper.SchemaMapper;
 import com.tencent.supersonic.chat.core.parser.SemanticParser;
 import com.tencent.supersonic.chat.core.plugin.Plugin;
@@ -33,8 +33,8 @@ import com.tencent.supersonic.chat.core.query.QueryManager;
 import com.tencent.supersonic.chat.core.query.SemanticQuery;
 import com.tencent.supersonic.chat.core.query.llm.s2sql.LLMSqlQuery;
 import com.tencent.supersonic.chat.core.query.rule.RuleSemanticQuery;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
-import com.tencent.supersonic.chat.core.utils.NatureHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.chat.core.utils.SimilarQueryManager;
 import com.tencent.supersonic.chat.server.persistence.dataobject.ChatParseDO;
 import com.tencent.supersonic.chat.server.persistence.dataobject.ChatQueryDO;
@@ -671,7 +671,7 @@ public class QueryServiceImpl implements QueryService {
         }
         //search from prefixSearch
         List<HanlpMapResult> hanlpMapResultList = SearchService.prefixSearch(dimensionValueReq.getValue(),
-                2000, dimensionValueReq.getAgentId(), detectModelIds);
+                2000, detectModelIds);
         HanlpHelper.transLetterOriginal(hanlpMapResultList);
         return hanlpMapResultList.stream()
                 .filter(o -> {

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/RecommendServiceImpl.java

@@ -2,8 +2,8 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 
 import com.google.common.collect.Lists;
-import com.tencent.supersonic.chat.api.pojo.RelatedSchemaElement;
+import com.tencent.supersonic.headless.api.pojo.RelatedSchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigFilter;
 import com.tencent.supersonic.chat.api.pojo.request.RecommendReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SchemaService.java

@@ -5,7 +5,7 @@ import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/SearchServiceImpl.java

@@ -2,9 +2,8 @@ package com.tencent.supersonic.chat.server.service.impl;
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.google.common.collect.Lists;
-import com.hankcs.hanlp.seg.common.Term;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
-import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
 import com.tencent.supersonic.chat.api.pojo.request.ItemNameVisibilityInfo;
 import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -12,22 +11,24 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryFilters;
 import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
 import com.tencent.supersonic.chat.api.pojo.response.SearchResult;
 import com.tencent.supersonic.chat.core.agent.Agent;
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.api.pojo.response.S2Term;
-import com.tencent.supersonic.chat.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.knowledge.ViewInfoStat;
+import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
+import com.tencent.supersonic.headless.core.knowledge.ViewInfoStat;
 import com.tencent.supersonic.chat.core.mapper.MapperHelper;
 import com.tencent.supersonic.chat.core.mapper.MatchText;
 import com.tencent.supersonic.chat.core.mapper.ModelWithSemanticType;
 import com.tencent.supersonic.chat.core.mapper.SearchMatchStrategy;
 import com.tencent.supersonic.chat.core.pojo.QueryContext;
-import com.tencent.supersonic.chat.core.utils.HanlpHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
-import com.tencent.supersonic.chat.core.utils.NatureHelper;
+import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
 import com.tencent.supersonic.chat.server.service.AgentService;
 import com.tencent.supersonic.chat.server.service.ChatService;
 import com.tencent.supersonic.chat.server.service.ConfigService;
 import com.tencent.supersonic.chat.server.service.SearchService;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import com.tencent.supersonic.common.util.ContextUtils;
+import com.tencent.supersonic.headless.server.service.KnowledgeService;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.beans.BeanUtils;
@@ -72,6 +73,9 @@ public class SearchServiceImpl implements SearchService {
     @Autowired
     private ConfigService configService;
 
+    @Autowired
+    private KnowledgeService knowledgeService;
+
     @Override
     public List<SearchResult> search(QueryReq queryReq) {
         // 1. check search enable
@@ -90,7 +94,7 @@ public class SearchServiceImpl implements SearchService {
         final Map<Long, String> modelToName = semanticSchemaDb.getViewIdToName();
 
         // 3.detect by segment
-        List<Term> originals = HanlpHelper.getTerms(queryText);
+        List<S2Term> originals = knowledgeService.getTerms(queryText);
         log.info("hanlp parse result: {}", originals);
         MapperHelper mapperHelper = ContextUtils.getBean(MapperHelper.class);
         Set<Long> detectModelIds = mapperHelper.getViewIds(queryReq.getModelId(), agentService.getAgent(agentId));
@@ -141,7 +145,7 @@ public class SearchServiceImpl implements SearchService {
         return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
     }
 
-    private List<Long> getPossibleModels(QueryReq queryCtx, List<Term> originals,
+    private List<Long> getPossibleModels(QueryReq queryCtx, List<S2Term> originals,
                                          ViewInfoStat modelStat, Long webModelId) {
 
         if (Objects.nonNull(webModelId) && webModelId > 0) {

chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/WordService.java

@@ -1,10 +1,10 @@
 package com.tencent.supersonic.chat.server.service.impl;
 
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
-import com.tencent.supersonic.chat.core.knowledge.DictWord;
+import com.tencent.supersonic.headless.core.knowledge.DictWord;
-import com.tencent.supersonic.chat.core.knowledge.builder.WordBuilderFactory;
+import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.utils.ComponentFactory;
 import com.tencent.supersonic.common.pojo.enums.DictWordType;
 import lombok.extern.slf4j.Slf4j;

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ChatConfigHelper.java

@@ -4,7 +4,7 @@ import static com.tencent.supersonic.common.pojo.Constants.ADMIN_LOWER;
 
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
+import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/ComponentFactory.java

@@ -1,7 +1,7 @@
 package com.tencent.supersonic.chat.server.util;
 
 import com.tencent.supersonic.chat.core.corrector.SemanticCorrector;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
+import com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter;
 import com.tencent.supersonic.chat.core.mapper.SchemaMapper;
 import com.tencent.supersonic.chat.core.parser.SemanticParser;
 import com.tencent.supersonic.chat.server.processor.execute.ExecuteResultProcessor;

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictMetaHelper.java

@@ -1,247 +0,0 @@
-package com.tencent.supersonic.chat.server.util;
-
-import com.github.pagehelper.PageInfo;
-import com.tencent.supersonic.chat.api.pojo.SchemaElement;
-import com.tencent.supersonic.chat.api.pojo.ViewSchema;
-import com.tencent.supersonic.chat.api.pojo.request.KnowledgeAdvancedConfig;
-import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
-import com.tencent.supersonic.chat.api.pojo.response.ChatConfigRichResp;
-import com.tencent.supersonic.chat.api.pojo.response.ChatDefaultRichConfigResp;
-import com.tencent.supersonic.chat.core.config.DefaultMetric;
-import com.tencent.supersonic.chat.core.config.Dim4Dict;
-import com.tencent.supersonic.chat.core.knowledge.DictUpdateMode;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DimValueDO;
-import com.tencent.supersonic.chat.server.service.ConfigService;
-import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
-import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.BeanUtils;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-
-@Component
-public class DictMetaHelper {
-
-    @Autowired
-    private ConfigService configService;
-    @Value("${model.internal.metric.suffix:internal_cnt}")
-    private String internalMetricNameSuffix;
-    @Value("${model.internal.day.number:2}")
-    private Integer internalMetricDays;
-    private SemanticInterpreter semanticInterpreter = ComponentFactory.getSemanticLayer();
-
-    public List<DimValueDO> generateDimValueInfo(DimValue2DictCommand dimValue2DictCommend) {
-        List<DimValueDO> dimValueDOList = new ArrayList<>();
-        DictUpdateMode updateMode = dimValue2DictCommend.getUpdateMode();
-        Set<Long> modelIds = new HashSet<>();
-        switch (updateMode) {
-            case OFFLINE_MODEL:
-                modelIds.addAll(dimValue2DictCommend.getModelIds());
-                dimValueDOList = generateDimValueInfoByModel(modelIds);
-                break;
-            case OFFLINE_FULL:
-                List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema();
-                if (CollectionUtils.isEmpty(modelSchemaDescList)) {
-                    break;
-                }
-
-                Map<Long, ViewSchema> modelIdAndDescPair = modelSchemaDescList.stream()
-                        .collect(Collectors.toMap(a -> a.getView().getId(), schema -> schema, (k1, k2) -> k1));
-                if (!CollectionUtils.isEmpty(modelIdAndDescPair)) {
-                    modelIds.addAll(modelIdAndDescPair.keySet());
-                    dimValueDOList = generateDimValueInfoByModel(modelIds);
-                    break;
-                }
-                break;
-            case REALTIME_ADD:
-                dimValueDOList = generateDimValueInfoByModelAndDim(dimValue2DictCommend.getModelAndDimPair());
-                break;
-            case NOT_SUPPORT:
-                throw new RuntimeException("illegal parameter for updateMode");
-            default:
-                break;
-        }
-
-        return dimValueDOList;
-    }
-
-    private List<DimValueDO> generateDimValueInfoByModelAndDim(Map<Long, List<Long>> modelAndDimMap) {
-        List<DimValueDO> dimValueDOList = new ArrayList<>();
-        if (CollectionUtils.isEmpty(modelAndDimMap)) {
-            return dimValueDOList;
-        }
-
-        List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema();
-        if (CollectionUtils.isEmpty(modelSchemaDescList)) {
-            return dimValueDOList;
-        }
-        Map<Long, ViewSchema> modelIdAndDescPair = modelSchemaDescList.stream()
-                .collect(Collectors.toMap(a -> a.getView().getId(), a -> a, (k1, k2) -> k1));
-
-        for (Long modelId : modelAndDimMap.keySet()) {
-            if (!modelIdAndDescPair.containsKey(modelId)) {
-                continue;
-            }
-            Map<Long, SchemaElement> dimIdAndDescPairAll;
-            dimIdAndDescPairAll = modelIdAndDescPair.get(modelId).getDimensions().stream()
-                    .collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1));
-            List<Long> dimIdReq = modelAndDimMap.get(modelId);
-            Map<Long, SchemaElement> dimIdAndDescPairReq = new HashMap<>();
-            for (Long dimId : dimIdReq) {
-                if (dimIdAndDescPairAll.containsKey(dimId)) {
-                    dimIdAndDescPairReq.put(dimId, dimIdAndDescPairAll.get(dimId));
-                }
-            }
-            fillDimValueDOList(dimValueDOList, modelId, dimIdAndDescPairReq);
-        }
-
-        return dimValueDOList;
-    }
-
-    private List<DimValueDO> generateDimValueInfoByModel(Set<Long> modelIds) {
-        List<DimValueDO> dimValueDOList = new ArrayList<>();
-        List<ViewSchema> modelSchemaDescList = semanticInterpreter.getViewSchema(new ArrayList<>(modelIds));
-        if (CollectionUtils.isEmpty(modelSchemaDescList)) {
-            return dimValueDOList;
-        }
-
-        modelSchemaDescList.forEach(modelSchemaDesc -> {
-            Map<Long, SchemaElement> dimIdAndDescPair = modelSchemaDesc.getDimensions().stream()
-                    .collect(Collectors.toMap(SchemaElement::getId, dimSchemaDesc -> dimSchemaDesc, (k1, k2) -> k1));
-            fillDimValueDOList(dimValueDOList, modelSchemaDesc.getView().getId(), dimIdAndDescPair);
-
-        });
-
-        return dimValueDOList;
-    }
-
-    private void fillDimValueDOList(List<DimValueDO> dimValueDOList, Long modelId,
-                                    Map<Long, SchemaElement> dimIdAndDescPair) {
-        ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(modelId);
-        if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatAggRichConfig())) {
-
-            ChatDefaultRichConfigResp chatDefaultConfig =
-                    chaConfigRichDesc.getChatAggRichConfig().getChatDefaultConfig();
-
-            List<KnowledgeInfoReq> knowledgeAggInfo =
-                    chaConfigRichDesc.getChatAggRichConfig().getKnowledgeInfos();
-
-            List<KnowledgeInfoReq> knowledgeDetailInfo =
-                    chaConfigRichDesc.getChatDetailRichConfig().getKnowledgeInfos();
-
-            fillKnowledgeDimValue(knowledgeDetailInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
-            fillKnowledgeDimValue(knowledgeAggInfo, chatDefaultConfig, dimValueDOList, dimIdAndDescPair, modelId);
-
-
-        }
-    }
-
-    private void fillKnowledgeDimValue(List<KnowledgeInfoReq> knowledgeInfos,
-                                       ChatDefaultRichConfigResp chatDefaultConfig,
-                                       List<DimValueDO> dimValueDOList,
-                                       Map<Long, SchemaElement> dimIdAndDescPair, Long modelId) {
-        Map<Long, DimensionResp> dimIdAndRespPair = queryDimensionRespByModelId(
-                new ArrayList<>(Arrays.asList(modelId)));
-        if (!CollectionUtils.isEmpty(knowledgeInfos)) {
-            List<Dim4Dict> dimensions = new ArrayList<>();
-            List<DefaultMetric> defaultMetricDescList = new ArrayList<>();
-            knowledgeInfos.stream()
-                    .filter(knowledgeInfo -> knowledgeInfo.getSearchEnable()
-                            && !CollectionUtils.isEmpty(dimIdAndDescPair)
-                            && dimIdAndDescPair.containsKey(knowledgeInfo.getItemId()))
-                    .forEach(knowledgeInfo -> {
-                        SchemaElement dimensionDesc = dimIdAndDescPair.get(knowledgeInfo.getItemId());
-
-                        Long dimId = dimensionDesc.getId();
-                        String internalMetricName = "";
-                        if (Objects.nonNull(dimId)) {
-                            String datasourceBizName = queryDataSourceByDimId(dimId);
-                            internalMetricName = datasourceBizName + UNDERLINE + internalMetricNameSuffix;
-                        }
-
-                        if (Objects.isNull(chatDefaultConfig)) {
-                            defaultMetricDescList.add(new DefaultMetric(internalMetricName,
-                                    internalMetricDays, DAY));
-                        } else {
-                            String metric = internalMetricName;
-                            if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())) {
-                                metric = chatDefaultConfig.getMetrics().get(0).getBizName();
-                            }
-                            defaultMetricDescList.add(new DefaultMetric(metric,
-                                    chatDefaultConfig.getUnit(), chatDefaultConfig.getPeriod()));
-
-                        }
-
-                        String bizName = dimensionDesc.getBizName();
-                        Dim4Dict dim4Dict = new Dim4Dict();
-                        dim4Dict.setDimId(knowledgeInfo.getItemId());
-                        dim4Dict.setBizName(bizName);
-                        if (Objects.nonNull(knowledgeInfo.getKnowledgeAdvancedConfig())) {
-                            KnowledgeAdvancedConfig knowledgeAdvancedConfig
-                                    = knowledgeInfo.getKnowledgeAdvancedConfig();
-                            BeanUtils.copyProperties(knowledgeAdvancedConfig, dim4Dict);
-
-                            if (Objects.nonNull(dimIdAndRespPair)
-                                    && dimIdAndRespPair.containsKey(dim4Dict.getDimId())) {
-                                String datasourceFilterSql = dimIdAndRespPair.get(
-                                        dim4Dict.getDimId()).getModelFilterSql();
-                                if (StringUtils.isNotEmpty(datasourceFilterSql)) {
-                                    dim4Dict.getRuleList().add(datasourceFilterSql);
-                                }
-
-                            }
-                        }
-                        dimensions.add(dim4Dict);
-
-                    });
-
-            if (!CollectionUtils.isEmpty(dimensions)) {
-                DimValueDO dimValueDO = new DimValueDO()
-                        .setModelId(modelId)
-                        .setDefaultMetricIds(defaultMetricDescList)
-                        .setDimensions(dimensions);
-                dimValueDOList.add(dimValueDO);
-            }
-        }
-    }
-
-    private Map<Long, DimensionResp> queryDimensionRespByModelId(List<Long> modelIds) {
-        Map<Long, DimensionResp> dimIdAndRespPair = new HashMap<>();
-        PageDimensionReq pageDimensionCmd = new PageDimensionReq();
-        pageDimensionCmd.setModelIds(modelIds);
-        PageInfo<DimensionResp> dimensionPage = semanticInterpreter.getDimensionPage(pageDimensionCmd);
-        if (Objects.nonNull(dimensionPage) && !CollectionUtils.isEmpty(dimensionPage.getList())) {
-            List<DimensionResp> dimList = dimensionPage.getList();
-            dimIdAndRespPair = dimList.stream().collect(Collectors.toMap(DimensionResp::getId, v -> v, (v1, v2) -> v2));
-        }
-        return dimIdAndRespPair;
-    }
-
-    private String queryDataSourceByDimId(Long id) {
-        PageDimensionReq pageDimensionCmd = new PageDimensionReq();
-        pageDimensionCmd.setId(id.toString());
-        PageInfo<DimensionResp> dimensionPage = semanticInterpreter.getDimensionPage(pageDimensionCmd);
-        if (Objects.nonNull(dimensionPage) && !CollectionUtils.isEmpty(dimensionPage.getList())) {
-            List<DimensionResp> list = dimensionPage.getList();
-            return list.get(0).getModelBizName();
-        }
-        return "";
-    }
-}

chat/server/src/main/java/com/tencent/supersonic/chat/server/util/DictTaskConverter.java

@@ -1,46 +0,0 @@
-package com.tencent.supersonic.chat.server.util;
-
-import com.google.common.base.Strings;
-import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.chat.core.knowledge.DictConfig;
-import com.tencent.supersonic.chat.core.knowledge.DimValue2DictCommand;
-import com.tencent.supersonic.chat.core.knowledge.DimValueInfo;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO;
-import com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO;
-import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
-import com.tencent.supersonic.common.util.JsonUtil;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.util.Date;
-import java.util.List;
-
-public class DictTaskConverter {
-
-    private static String dateTimeFormatter = "yyyyMMddHHmmss";
-
-    public static DictTaskDO generateDimValueDictTaskDO(DimValue2DictCommand dimValue2DictCommend, User user) {
-        DictTaskDO taskPO = new DictTaskDO();
-        Date createAt = new Date();
-        String date = DateTimeFormatter.ofPattern(dateTimeFormatter)
-                .format(createAt.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime());
-        String creator = Strings.isNullOrEmpty(user.getName()) ? "" : user.getName();
-        String updateMode = dimValue2DictCommend.getUpdateMode().getValue();
-        String name = String.format("DimValue_dic_%s_%s_%s", updateMode, creator, date);
-        taskPO.setName(name);
-
-        taskPO.setCreatedAt(createAt);
-        taskPO.setCommand(JsonUtil.toString(dimValue2DictCommend));
-        taskPO.setStatus(TaskStatusEnum.PENDING.getCode());
-        taskPO.setCreatedBy(creator);
-
-        return taskPO;
-    }
-
-    public static DictConfig dictConfPO2Config(DictConfDO dictConfDO) {
-        DictConfig dictConfig = new DictConfig();
-        dictConfig.setModelId(dictConfDO.getModelId());
-        List<DimValueInfo> dimValueInfos = JsonUtil.toList(dictConfDO.getDimValueInfos(), DimValueInfo.class);
-        dictConfig.setDimValueInfoList(dimValueInfos);
-        return dictConfig;
-    }
-}

chat/server/src/main/resources/mapper/DictConfMapper.xml

@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
-        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-
-<mapper namespace="com.tencent.supersonic.chat.server.persistence.mapper.DictConfMapper">
-
-    <resultMap id="DictConfPO"
-               type="com.tencent.supersonic.chat.server.persistence.dataobject.DictConfDO">
-        <id column="id" property="id"/>
-        <result column="model_id" property="modelId"/>
-        <result column="dim_value_infos" property="dimValueInfos"/>
-        <result column="created_at" property="createdAt"/>
-        <result column="updated_at" property="updatedAt"/>
-        <result column="created_by" property="createdBy"/>
-        <result column="updated_by" property="updatedBy"/>
-    </resultMap>
-
-    <insert id="createDictConf">
-        insert into s2_dictionary
-        (`domain_id`, dim_value_infos, created_at, updated_at, created_by, updated_by)
-        values
-        (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy})
-    </insert>
-
-    <insert id="upsertDictInfo">
-         insert into s2_dictionary
-        (`model_id`, dim_value_infos, created_at, updated_at, created_by, updated_by)
-        values
-        (#{modelId}, #{dimValueInfos}, #{createdAt}, #{updatedAt}, #{createdBy}, #{updatedBy})
-        on duplicate key update
-        dim_value_infos = #{dimValueInfos},
-        updated_at = #{updatedAt},
-        updated_by = #{updatedBy}
-    </insert>
-
-    <update id="editDictConf">
-        update s2_dictionary
-        set dim_value_infos = #{dimValueInfos},
-        updated_at = #{updatedAt},
-        updated_by = #{updatedBy}
-        where model_id = #{modelId}
-        and status = 0
-    </update>
-
-    <select id="getDictInfoByModelId" resultMap="DictConfPO">
-        select *
-        from s2_dictionary
-        where model_id = #{modelId}
-        and status = 0
-    </select>
-
-
-</mapper>

chat/server/src/main/resources/mapper/DictTaskMapper.xml

@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
-        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
-
-<mapper namespace="com.tencent.supersonic.chat.server.persistence.mapper.DictTaskMapper">
-
-    <resultMap id="DimValueDictTaskPO"
-               type="com.tencent.supersonic.chat.server.persistence.dataobject.DictTaskDO">
-        <id column="id" property="id"/>
-        <result column="name" property="name"/>
-        <result column="description" property="description"/>
-        <result column="command" property="command"/>
-        <result column="command_md5" property="commandMd5"/>
-        <result column="dimension_ids" property="dimIds"/>
-        <result column="status" property="status"/>
-        <result column="created_by" property="createdBy"/>
-        <result column="created_at" property="createdAt"/>
-        <result column="progress" property="progress"/>
-        <result column="elapsed_ms" property="elapsedMs"/>
-    </resultMap>
-
-    <insert id="createDimValueTask" useGeneratedKeys="true" keyProperty="id">
-        insert into s2_dictionary_task
-        (`name`, description, command, command_md5, dimension_ids, status, created_by, progress, elapsed_ms)
-        values
-        (#{name}, #{description}, #{command}, #{commandMd5}, #{dimIds}, #{status}, #{createdBy}, #{progress}, #{elapsedMs})
-    </insert>
-
-    <update id="updateTaskStatus">
-        update s2_dictionary_task
-        <set>
-            <if test="description != null and description !=''">
-                description = #{description},
-            </if>
-            <if test="status != null">
-                status = #{status},
-            </if>
-            <if test="dimIds != null">
-                dimension_ids = #{dimIds},
-            </if>
-            <if test="progress != null">
-                progress = #{progress},
-            </if>
-            <if test="elapsedMs != null">
-                elapsed_ms = #{elapsedMs},
-            </if>
-
-        </set>
-        where id = #{id}
-    </update>
-
-    <select id="searchDictTaskList" resultMap="DimValueDictTaskPO">
-        select *
-        from s2_dictionary_task
-        <where>
-            <if test="id != null and id != ''">
-                and id = #{id}
-            </if>
-            <if test="name != null and name !=''">
-                and `name` like  CONCAT('%', #{name}, '%')
-            </if>
-            <if test="createdBy != null and createdBy !=''">
-                and created_by = #{createdBy}
-            </if>
-            <if test="createdAt != null and createdAt !=''">
-                and created_at &gt;= #{createdAt}
-            </if>
-            <if test="status != null">
-                and status= #{status.code}
-            </if>
-        </where>
-        order by created_at desc
-    </select>
-
-</mapper>