topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 20:51:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(project) Remove Python module and related Python code. (#1263)

Browse Source
This commit is contained in:
lexluo09
2024-06-28 12:01:13 +08:00
committed by GitHub
parent 26cd73518d
commit a3d4f18df4
47 changed files with 17 additions and 3565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
chat/server/src/main/java/com/tencent/supersonic/chat/server/plugin/recognize/embedding/EmbeddingRecallRecognizer.java
View File

@@ -7,14 +7,10 @@ import com.tencent.supersonic.chat.server.plugin.PluginManager;
import com.tencent.supersonic.chat.server.plugin.PluginRecallResult;
import com.tencent.supersonic.chat.server.plugin.recognize.PluginRecognizer;
import com.tencent.supersonic.chat.server.pojo.ChatParseContext;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.common.util.ContextUtils;
import dev.langchain4j.store.embedding.Retrieval;
import dev.langchain4j.store.embedding.RetrieveQueryResult;
import com.tencent.supersonic.headless.chat.utils.ComponentFactory;
import com.tencent.supersonic.headless.chat.parser.llm.PythonLLMProxy;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.util.CollectionUtils;
@@ -31,10 +27,6 @@ import java.util.stream.Collectors;
public class EmbeddingRecallRecognizer extends PluginRecognizer {
public boolean checkPreCondition(ChatParseContext chatParseContext) {
EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class);
if (StringUtils.isBlank(embeddingConfig.getUrl()) && ComponentFactory.getLLMProxy() instanceof PythonLLMProxy) {
return false;
}
List<Plugin> plugins = getPluginList(chatParseContext);
return !CollectionUtils.isEmpty(plugins);
}

62
chat/server/src/main/java/com/tencent/supersonic/chat/server/util/SimilarQueryManager.java
View File

@@ -1,36 +1,21 @@
package com.tencent.supersonic.chat.server.util;
import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.pojo.request.SimilarQueryReq;
import com.tencent.supersonic.chat.api.pojo.response.SimilarQueryRecallResp;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.common.service.EmbeddingService;
import dev.langchain4j.data.document.Metadata;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.store.embedding.Retrieval;
import dev.langchain4j.store.embedding.RetrieveQuery;
import dev.langchain4j.store.embedding.RetrieveQueryResult;
import dev.langchain4j.store.embedding.TextSegmentConvert;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.net.URI;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -48,28 +33,7 @@ public class SimilarQueryManager {
this.embeddingConfig = embeddingConfig;
}
public void saveSimilarQuery(SimilarQueryReq similarQueryReq) {
if (StringUtils.isBlank(embeddingConfig.getUrl())) {
return;
}
String queryText = similarQueryReq.getQueryText();
try {
Map<String, Object> metaData = new HashMap<>();
metaData.put("agentId", String.valueOf(similarQueryReq.getAgentId()));
TextSegment textSegment = TextSegment.from(queryText, new Metadata(metaData));
TextSegmentConvert.addQueryId(textSegment, String.valueOf(similarQueryReq.getQueryId()));
String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection();
embeddingService.addQuery(solvedQueryCollection, Lists.newArrayList(textSegment));
} catch (Exception e) {
log.warn("save history question to embedding failed, queryText:{}", queryText, e);
}
}
public List<SimilarQueryRecallResp> recallSimilarQuery(String queryText, Integer agentId) {
if (StringUtils.isBlank(embeddingConfig.getUrl())) {
return Lists.newArrayList();
}
List<SimilarQueryRecallResp> similarQueryRecallResps = Lists.newArrayList();
try {
String solvedQueryCollection = embeddingConfig.getSolvedQueryCollection();
@@ -113,30 +77,4 @@ public class SimilarQueryManager {
return similarQueryRecallResps.stream()
.limit(embeddingConfig.getSolvedQueryResultNum()).collect(Collectors.toList());
}
private ResponseEntity<String> doRequest(String path, String jsonBody) {
if (StringUtils.isEmpty(embeddingConfig.getUrl())) {
return ResponseEntity.of(Optional.empty());
}
String url = embeddingConfig.getUrl() + path;
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setLocation(URI.create(url));
URI requestUrl = UriComponentsBuilder
.fromHttpUrl(url).build().encode().toUri();
HttpEntity<String> entity = new HttpEntity<>(jsonBody, headers);
log.info("[embedding] request body :{}, url:{}", jsonBody, url);
RestTemplate restTemplate = new RestTemplate();
ResponseEntity<String> responseEntity = restTemplate.exchange(requestUrl,
HttpMethod.POST, entity, new ParameterizedTypeReference<String>() {
});
log.info("[embedding] result body:{}", responseEntity);
return responseEntity;
} catch (Exception e) {
log.warn("connect to embedding service failed, url:{}", url);
}
return ResponseEntity.of(Optional.empty());
}
}