topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-13 13:07:32 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)Refactor LLMParser impl naming and structure.

Browse Source
This commit is contained in:
jerryjzhang
2024-05-24 15:08:18 +08:00
parent 987154c4a3
commit 67b69768df
33 changed files with 158 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/ApplicationStartedListener.java
View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.server.service.impl.WordService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@@ -21,7 +21,7 @@ import java.util.concurrent.CompletableFuture;
public class ApplicationStartedListener implements CommandLineRunner {
@Autowired
private KnowledgeService knowledgeService;
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private WordService wordService;
@@ -37,7 +37,7 @@ public class ApplicationStartedListener implements CommandLineRunner {
List<DictWord> dictWords = wordService.getAllDictWords();
wordService.setPreDictWords(dictWords);
knowledgeService.reloadAllData(dictWords);
knowledgeBaseService.reloadAllData(dictWords);
log.debug("ApplicationStartedInit end");
isOk = true;
@@ -72,7 +72,7 @@ public class ApplicationStartedListener implements CommandLineRunner {
}
log.info("dictWords has changed");
wordService.setPreDictWords(dictWords);
knowledgeService.updateOnlineKnowledge(wordService.getAllDictWords());
knowledgeBaseService.updateOnlineKnowledge(wordService.getAllDictWords());
} catch (Exception e) {
log.error("reloadKnowledge error", e);
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FullMetaEmbeddingListener.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.headless.core.chat.parser.JavaLLMProxy;
import com.tencent.supersonic.headless.core.chat.parser.llm.JavaLLMProxy;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.schedule.EmbeddingTask;
import lombok.extern.slf4j.Slf4j;

12
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/SqlEmbeddingListener.java
View File

@@ -1,9 +1,9 @@
package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.headless.core.chat.parser.JavaLLMProxy;
import com.tencent.supersonic.headless.core.chat.parser.llm.SqlExamplarLoader;
import com.tencent.supersonic.headless.core.chat.parser.llm.SqlExample;
import com.tencent.supersonic.headless.core.chat.parser.llm.JavaLLMProxy;
import com.tencent.supersonic.headless.core.chat.parser.llm.ExemplarManager;
import com.tencent.supersonic.headless.core.chat.parser.llm.Exemplar;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -19,7 +19,7 @@ import java.util.List;
public class SqlEmbeddingListener implements CommandLineRunner {
@Autowired
private SqlExamplarLoader sqlExamplarLoader;
private ExemplarManager exemplarManager;
@Autowired
private EmbeddingConfig embeddingConfig;
@@ -31,9 +31,9 @@ public class SqlEmbeddingListener implements CommandLineRunner {
public void initSqlExamples() {
try {
if (ComponentFactory.getLLMProxy() instanceof JavaLLMProxy) {
List<SqlExample> sqlExamples = sqlExamplarLoader.getSqlExamples();
List<Exemplar> exemplars = exemplarManager.getExemplars();
String collectionName = embeddingConfig.getText2sqlCollectionName();
sqlExamplarLoader.addEmbeddingStore(sqlExamples, collectionName);
exemplarManager.addExemplars(exemplars, collectionName);
}
} catch (Exception e) {
log.error("initSqlExamples error", e);

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ChatQueryServiceImpl.java
View File

@@ -42,7 +42,7 @@ import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.core.chat.corrector.GrammarCorrector;
import com.tencent.supersonic.headless.core.chat.corrector.SchemaCorrector;
import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.core.chat.knowledge.SearchService;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
@@ -96,7 +96,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
@Autowired
private ChatContextService chatContextService;
@Autowired
private KnowledgeService knowledgeService;
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private QueryService queryService;
@Autowired
@@ -557,7 +557,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
Map<Long, List<Long>> modelIdToDataSetIds = new HashMap<>();
modelIdToDataSetIds.put(dimensionValueReq.getModelId(), new ArrayList<>(dataSetIds));
//search from prefixSearch
List<HanlpMapResult> hanlpMapResultList = knowledgeService.prefixSearch(dimensionValueReq.getValue(),
List<HanlpMapResult> hanlpMapResultList = knowledgeBaseService.prefixSearch(dimensionValueReq.getValue(),
2000, modelIdToDataSetIds, dataSetIds);
HanlpHelper.transLetterOriginal(hanlpMapResultList);
return hanlpMapResultList.stream()

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DictTaskServiceImpl.java
View File

@@ -9,7 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.core.file.FileHandler;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictTaskDO;
import com.tencent.supersonic.headless.server.persistence.repository.DictRepository;
@@ -46,7 +46,7 @@ public class DictTaskServiceImpl implements DictTaskService {
DictUtils dictConverter,
DictUtils dictUtils,
FileHandler fileHandler,
KnowledgeService knowledgeService) {
KnowledgeBaseService knowledgeBaseService) {
this.dictRepository = dictRepository;
this.dictConverter = dictConverter;
this.dictUtils = dictUtils;

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SearchServiceImpl.java
View File

@@ -18,7 +18,7 @@ import com.tencent.supersonic.headless.core.pojo.QueryContext;
import com.tencent.supersonic.headless.core.chat.knowledge.DataSetInfoStat;
import com.tencent.supersonic.headless.core.chat.knowledge.DictWord;
import com.tencent.supersonic.headless.core.chat.knowledge.HanlpMapResult;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeService;
import com.tencent.supersonic.headless.core.chat.knowledge.KnowledgeBaseService;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.HanlpHelper;
import com.tencent.supersonic.headless.core.chat.knowledge.helper.NatureHelper;
import com.tencent.supersonic.headless.server.service.ChatContextService;
@@ -58,7 +58,7 @@ public class SearchServiceImpl implements SearchService {
@Autowired
private ChatContextService chatContextService;
@Autowired
private KnowledgeService knowledgeService;
private KnowledgeBaseService knowledgeBaseService;
@Autowired
private DataSetService dataSetService;
@@ -73,7 +73,7 @@ public class SearchServiceImpl implements SearchService {
Map<Long, List<Long>> modelIdToDataSetIds =
dataSetService.getModelIdToDataSetIds(new ArrayList<>(dataSetIdToName.keySet()), User.getFakeUser());
// 2.detect by segment
List<S2Term> originals = knowledgeService.getTerms(queryText, modelIdToDataSetIds);
List<S2Term> originals = knowledgeBaseService.getTerms(queryText, modelIdToDataSetIds);
log.info("hanlp parse result: {}", originals);
Set<Long> dataSetIds = queryReq.getDataSetIds();