topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 13:07:32 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) move Knowledge package to chat package (#816)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-03-13 20:15:29 +08:00
committed by GitHub
parent be1ebb8d8c
commit 9dcf8c23f2
47 changed files with 109 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
chat/server/src/main/java/com/tencent/supersonic/chat/server/processor/execute/MetricRecommendProcessor.java
View File

@@ -12,7 +12,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.headless.api.pojo.response.QueryResult;
import com.tencent.supersonic.headless.core.knowledge.MetaEmbeddingService;
import com.tencent.supersonic.headless.core.chat.knowledge.MetaEmbeddingService;
import org.springframework.util.CollectionUtils;
import java.util.Collections;

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaElement.java
View File

@@ -27,9 +27,8 @@ public class SchemaElement implements Serializable {
private List<String> alias;
private List<SchemaValueMap> schemaValueMaps;
private List<RelatedSchemaElement> relatedSchemaElements;
private String defaultAgg;
private String dataFormatType;
private double order;
@Override

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DataSetInfoStat.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/DataSetInfoStat.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import lombok.Builder;
import lombok.Data;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DatabaseMapResult.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/DatabaseMapResult.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.google.common.base.Objects;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictUpdateMode.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/DictUpdateMode.java
View File

@@ -1,5 +1,5 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
public enum DictUpdateMode {

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictWord.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/DictWord.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import java.util.Objects;
import lombok.Data;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/DictionaryAttributeUtil.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/DictionaryAttributeUtil.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.hankcs.hanlp.dictionary.CoreDictionary;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/EmbeddingResult.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/EmbeddingResult.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.google.common.base.Objects;
import java.util.Map;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HadoopFileIOAdapter.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/HadoopFileIOAdapter.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.hankcs.hanlp.corpus.io.IIOAdapter;
import java.io.IOException;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/HanlpMapResult.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/HanlpMapResult.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.google.common.base.Objects;
import java.util.List;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/KnowledgeService.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/KnowledgeService.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MapResult.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/MapResult.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import java.io.Serializable;
import lombok.Data;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MetaEmbeddingService.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/MetaEmbeddingService.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.common.pojo.Constants;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/MultiCustomDictionary.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/MultiCustomDictionary.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import static com.hankcs.hanlp.utility.Predefine.logger;
@@ -15,7 +15,7 @@ import com.hankcs.hanlp.seg.common.Term;
import com.hankcs.hanlp.utility.LexiconUtility;
import com.hankcs.hanlp.utility.Predefine;
import com.hankcs.hanlp.utility.TextUtility;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/SearchService.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/SearchService.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge;
package com.tencent.supersonic.headless.core.chat.knowledge;
import com.hankcs.hanlp.collection.trie.bintrie.BaseNode;
import com.hankcs.hanlp.collection.trie.bintrie.BinTrie;
@@ -7,7 +7,7 @@ import com.hankcs.hanlp.dictionary.CoreDictionary;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq;
import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/BaseWordBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/BaseWordBuilder.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/BaseWordWithAliasBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/BaseWordWithAliasBuilder.java
View File

@@ -1,7 +1,8 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.ArrayList;
import java.util.List;
import org.springframework.util.CollectionUtils;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/DimensionWordBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/DimensionWordBuilder.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/EntityWordBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/EntityWordBuilder.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/MetricWordBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/MetricWordBuilder.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ModelWordBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/ModelWordBuilder.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/TagWordBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/TagWordBuilder.java
View File

@@ -1,9 +1,10 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/ValueWordBuilder.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/ValueWordBuilder.java
View File

@@ -1,10 +1,10 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import java.util.List;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/builder/WordBuilderFactory.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/builder/WordBuilderFactory.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge.builder;
package com.tencent.supersonic.headless.core.chat.knowledge.builder;
import com.tencent.supersonic.common.pojo.enums.DictWordType;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/FileHelper.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/helper/FileHelper.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge.helper;
package com.tencent.supersonic.headless.core.chat.knowledge.helper;
import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HanlpHelper.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/helper/HanlpHelper.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge.helper;
package com.tencent.supersonic.headless.core.chat.knowledge.helper;
import com.google.common.collect.Lists;
import com.hankcs.hanlp.HanLP;
@@ -9,11 +9,11 @@ import com.hankcs.hanlp.seg.Segment;
import com.hankcs.hanlp.seg.common.Term;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.HadoopFileIOAdapter;
import com.tencent.supersonic.headless.core.knowledge.MapResult;
import com.tencent.supersonic.headless.core.knowledge.MultiCustomDictionary;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.HadoopFileIOAdapter;
import com.tencent.supersonic.headless.core.chat.knowledge.MapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.MultiCustomDictionary;
import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/HdfsFileHelper.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/helper/HdfsFileHelper.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.knowledge.helper;
package com.tencent.supersonic.headless.core.chat.knowledge.helper;
import static com.hankcs.hanlp.HanLP.Config.CustomDictionaryPath;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/knowledge/helper/NatureHelper.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/knowledge/helper/NatureHelper.java
View File

@@ -1,11 +1,11 @@
package com.tencent.supersonic.headless.core.knowledge.helper;
package com.tencent.supersonic.headless.core.chat.knowledge.helper;
import com.google.common.collect.Lists;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat;
import com.tencent.supersonic.headless.core.chat.knowledge.DataSetInfoStat;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/BaseMatchStrategy.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.chat.mapper;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/DatabaseMatchStrategy.java
View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.config.OptimizationConfig;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.DatabaseMapResult;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMapper.java
View File

@@ -8,9 +8,9 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult;
import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.EmbeddingResult;
import com.tencent.supersonic.headless.core.chat.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import java.util.List;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/EmbeddingMatchStrategy.java
View File

@@ -6,8 +6,8 @@ import com.tencent.supersonic.common.util.embedding.Retrieval;
import com.tencent.supersonic.common.util.embedding.RetrieveQuery;
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
import com.tencent.supersonic.headless.core.config.OptimizationConfig;
import com.tencent.supersonic.headless.core.knowledge.EmbeddingResult;
import com.tencent.supersonic.headless.core.knowledge.MetaEmbeddingService;
import com.tencent.supersonic.headless.core.chat.knowledge.EmbeddingResult;
import com.tencent.supersonic.headless.core.chat.knowledge.MetaEmbeddingService;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/HanlpDictMatchStrategy.java
View File

@@ -4,8 +4,8 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.config.OptimizationConfig;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/KeywordMapper.java
View File

@@ -7,10 +7,10 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.DatabaseMapResult;
import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.DatabaseMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/MapperHelper.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.headless.core.chat.mapper;
import com.hankcs.hanlp.algorithm.EditDistance;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.config.OptimizationConfig;
import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/QueryFilterMapper.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaMapInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilters;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.builder.BaseWordBuilder;
import com.tencent.supersonic.headless.core.chat.knowledge.builder.BaseWordBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/mapper/SearchMatchStrategy.java
View File

@@ -4,9 +4,9 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.response.S2Term;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;

17
headless/core/src/main/java/com/tencent/supersonic/headless/core/chat/parser/llm/LLMRequestService.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.headless.core.chat.parser.llm;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.enums.DataFormatTypeEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
@@ -8,9 +7,7 @@ import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.response.DataSetSchemaResp;
import com.tencent.supersonic.headless.core.chat.parser.SatisfactionChecker;
import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq;
import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMReq.ElementValue;
@@ -80,7 +77,7 @@ public class LLMRequestService {
List<String> fieldNameList = getFieldNameList(queryCtx, dataSetId, llmParserConfig);
String priorExts = getPriorExts(dataSetId, fieldNameList);
String priorExts = getPriorExts(queryCtx, fieldNameList);
llmReq.setPriorExts(priorExts);
fieldNameList.add(TimeDimensionEnum.DAY.getChName());
@@ -117,19 +114,16 @@ public class LLMRequestService {
return new ArrayList<>(results);
}
private String getPriorExts(Long dataSetId, List<String> fieldNameList) {
private String getPriorExts(QueryContext queryContext, List<String> fieldNameList) {
StringBuilder extraInfoSb = new StringBuilder();
//todo
List<DataSetSchemaResp> dataSetSchemaResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(dataSetSchemaResps)) {
DataSetSchemaResp dataSetSchemaResp = dataSetSchemaResps.get(0);
Map<String, String> fieldNameToDataFormatType = dataSetSchemaResp.getMetrics()
SemanticSchema semanticSchema = queryContext.getSemanticSchema();
Map<String, String> fieldNameToDataFormatType = semanticSchema.getMetrics()
.stream().filter(metricSchemaResp -> Objects.nonNull(metricSchemaResp.getDataFormatType()))
.flatMap(metricSchemaResp -> {
Set<Pair<String, String>> result = new HashSet<>();
String dataFormatType = metricSchemaResp.getDataFormatType();
result.add(Pair.of(metricSchemaResp.getName(), dataFormatType));
List<String> aliasList = SchemaItem.getAliasList(metricSchemaResp.getAlias());
List<String> aliasList = metricSchemaResp.getAlias();
if (!CollectionUtils.isEmpty(aliasList)) {
for (String alias : aliasList) {
result.add(Pair.of(alias, dataFormatType));
@@ -146,7 +140,6 @@ public class LLMRequestService {
extraInfoSb.append(format);
}
}
}
return extraInfoSb.toString();
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/config/ChatLocalFileConfig.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.config;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/file/LocalFileConfig.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.core.file;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/ApplicationStartedListener.java
View File

@@ -1,8 +1,8 @@
package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.server.service.impl.WordService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SchemaDictUpdateListener.java
View File

@@ -4,8 +4,8 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DataEvent;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.EventType;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationListener;
import org.springframework.scheduling.annotation.Async;

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
View File

@@ -43,11 +43,11 @@ import com.tencent.supersonic.headless.core.chat.query.QueryManager;
import com.tencent.supersonic.headless.core.chat.query.SemanticQuery;
import com.tencent.supersonic.headless.core.chat.query.llm.s2sql.LLMSqlQuery;
import com.tencent.supersonic.headless.core.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.knowledge.SearchService;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.pojo.ChatContext;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.server.persistence.dataobject.StatisticsDO;

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
View File

@@ -9,8 +9,8 @@ import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.core.file.FileHandler;
import com.tencent.supersonic.headless.core.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
import com.tencent.supersonic.headless.server.service.DictTaskService;

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java
View File

@@ -14,12 +14,12 @@ import com.tencent.supersonic.headless.core.chat.mapper.MatchText;
import com.tencent.supersonic.headless.core.chat.mapper.ModelWithSemanticType;
import com.tencent.supersonic.headless.core.chat.mapper.SearchMatchStrategy;
import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.knowledge.DataSetInfoStat;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.DataSetInfoStat;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.server.service.ChatContextService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.SearchService;
@@ -147,10 +147,6 @@ public class SearchServiceImpl implements SearchService {
&& modelStat.getDimensionValueDataSetCount() <= 0 && modelStat.getDataSetCount() <= 0;
}
private boolean effectiveModel(Long contextModel) {
return Objects.nonNull(contextModel) && contextModel > 0;
}
private Set<SearchResult> searchDimensionValue(List<SchemaElement> metricsDb,
Map<Long, String> modelToName,
long metricModelCount,

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/WordService.java
View File

@@ -3,8 +3,8 @@ package com.tencent.supersonic.headless.server.service.impl;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.core.knowledge.DictWord;
import com.tencent.supersonic.headless.core.knowledge.builder.WordBuilderFactory;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.builder.WordBuilderFactory;
import com.tencent.supersonic.headless.server.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/DataSetSchemaBuilder.java
View File

@@ -187,6 +187,7 @@ public class DataSetSchemaBuilder {
.alias(alias)
.relatedSchemaElements(getRelateSchemaElement(metric))
.defaultAgg(metric.getDefaultAgg())
.dataFormatType(metric.getDataFormatType())
.build();
metrics.add(metricToAdd);