topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Upgrade to the latest version of langchain4j and add support for embedding deletion operation and reset. (#1660)

Browse Source
This commit is contained in:
lexluo09
2024-09-12 18:16:16 +08:00
committed by GitHub
parent 693356e46a
commit 4b1dab8e4a
16 changed files with 13307 additions and 16497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
common/pom.xml
View File

@@ -186,32 +186,6 @@
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-embeddings</artifactId>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-anthropic-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-ollama-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-open-ai-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-azure-ai-search-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-azure-open-ai-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-embeddings-all-minilm-l6-v2-q</artifactId>
@@ -244,6 +218,10 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-ollama</artifactId>
</dependency>
<!--langchain4j-->
<dependency>
<groupId>com.hankcs</groupId>

2
common/src/main/java/com/tencent/supersonic/common/service/EmbeddingService.java
View File

@@ -18,4 +18,6 @@ public interface EmbeddingService {
List<RetrieveQueryResult> retrieveQuery(
String collectionName, RetrieveQuery retrieveQuery, int num);
void removeAll();
}

99
common/src/main/java/com/tencent/supersonic/common/service/impl/EmbeddingServiceImpl.java
View File

@@ -7,11 +7,11 @@ import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.provider.ModelProvider;
import dev.langchain4j.store.embedding.BaseEmbeddingStoreFactory;
import dev.langchain4j.store.embedding.EmbeddingMatch;
import dev.langchain4j.store.embedding.EmbeddingSearchRequest;
import dev.langchain4j.store.embedding.EmbeddingSearchResult;
import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.EmbeddingStoreFactory;
import dev.langchain4j.store.embedding.EmbeddingStoreFactoryProvider;
import dev.langchain4j.store.embedding.Retrieval;
import dev.langchain4j.store.embedding.RetrieveQuery;
@@ -20,7 +20,6 @@ import dev.langchain4j.store.embedding.TextSegmentConvert;
import dev.langchain4j.store.embedding.filter.Filter;
import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder;
import dev.langchain4j.store.embedding.filter.comparison.IsEqualTo;
import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.collections4.CollectionUtils;
@@ -46,9 +45,8 @@ public class EmbeddingServiceImpl implements EmbeddingService {
@Override
public void addQuery(String collectionName, List<TextSegment> queries) {
EmbeddingStoreFactory embeddingStoreFactory = EmbeddingStoreFactoryProvider.getFactory();
EmbeddingStore embeddingStore = embeddingStoreFactory.create(collectionName);
EmbeddingStore embeddingStore =
EmbeddingStoreFactoryProvider.getFactory().create(collectionName);
for (TextSegment query : queries) {
String question = query.text();
try {
@@ -101,30 +99,23 @@ public class EmbeddingServiceImpl implements EmbeddingService {
@Override
public void deleteQuery(String collectionName, List<TextSegment> queries) {
// Not supported yet in Milvus and Chroma
EmbeddingStoreFactory embeddingStoreFactory = EmbeddingStoreFactoryProvider.getFactory();
EmbeddingStore embeddingStore = embeddingStoreFactory.create(collectionName);
EmbeddingStore embeddingStore =
EmbeddingStoreFactoryProvider.getFactory().create(collectionName);
try {
if (embeddingStore instanceof InMemoryEmbeddingStore) {
InMemoryEmbeddingStore inMemoryEmbeddingStore =
(InMemoryEmbeddingStore) embeddingStore;
List<String> queryIds =
queries.stream()
.map(textSegment -> TextSegmentConvert.getQueryId(textSegment))
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(queryIds)) {
MetadataFilterBuilder filterBuilder =
new MetadataFilterBuilder(TextSegmentConvert.QUERY_ID);
Filter filter = filterBuilder.isIn(queryIds);
inMemoryEmbeddingStore.removeAll(filter);
for (String queryId : queryIds) {
cache.put(queryId, false);
}
}
} else {
throw new RuntimeException("Not supported yet.");
List<String> queryIds =
queries.stream()
.map(textSegment -> TextSegmentConvert.getQueryId(textSegment))
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(queryIds)) {
MetadataFilterBuilder filterBuilder =
new MetadataFilterBuilder(TextSegmentConvert.QUERY_ID);
Filter filter = filterBuilder.isIn(queryIds);
embeddingStore.removeAll(filter);
queryIds.stream().forEach(queryId -> cache.put(queryId, false));
}
} catch (Exception e) {
log.error("deleteQuery error,collectionName:{},queries:{}", collectionName, queries);
}
@@ -149,6 +140,18 @@ public class EmbeddingServiceImpl implements EmbeddingService {
.collect(Collectors.toList());
}
@Override
public void removeAll() {
BaseEmbeddingStoreFactory factory =
(BaseEmbeddingStoreFactory) EmbeddingStoreFactoryProvider.getFactory();
Map<String, EmbeddingStore<TextSegment>> collectionNameToStore =
factory.getCollectionNameToStore();
for (EmbeddingStore<TextSegment> embeddingStore : collectionNameToStore.values()) {
embeddingStore.removeAll();
}
cache.invalidateAll();
}
private RetrieveQueryResult retrieveSingleQuery(
String queryText,
EmbeddingModel embeddingModel,
@@ -193,28 +196,36 @@ public class EmbeddingServiceImpl implements EmbeddingService {
return retrieval;
}
public static Filter createCombinedFilter(Map<String, Object> map) {
if (MapUtils.isEmpty(map)) {
public static Filter createCombinedFilter(Map<String, Object> criteriaMap) {
if (MapUtils.isEmpty(criteriaMap)) {
return null;
}
Filter result = null;
for (Map.Entry<String, Object> entry : map.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
Filter orFilter = null;
if (value instanceof List) {
for (String val : (List<String>) value) {
IsEqualTo isEqualTo = new IsEqualTo(key, val);
orFilter = (orFilter == null) ? isEqualTo : Filter.or(orFilter, isEqualTo);
Filter combinedFilter = null;
for (Map.Entry<String, Object> entry : criteriaMap.entrySet()) {
String fieldName = entry.getKey();
Object fieldValue = entry.getValue();
Filter fieldFilter = null;
if (fieldValue instanceof List) {
// Create an OR filter for each value in the list
for (String value : (List<String>) fieldValue) {
IsEqualTo equalToFilter = new IsEqualTo(fieldName, value);
fieldFilter =
(fieldFilter == null)
? equalToFilter
: Filter.or(fieldFilter, equalToFilter);
}
} else if (value instanceof String) {
orFilter = new IsEqualTo(key, value);
} else if (fieldValue instanceof String) {
// Create a simple equality filter
fieldFilter = new IsEqualTo(fieldName, fieldValue);
}
if (orFilter != null) {
result = (result == null) ? orFilter : Filter.and(result, orFilter);
// Combine the current field filter with the overall filter using AND logic
if (fieldFilter != null) {
combinedFilter =
(combinedFilter == null)
? fieldFilter
: Filter.and(combinedFilter, fieldFilter);
}
}
return result;
return combinedFilter;
}
}

1
common/src/main/java/com/tencent/supersonic/common/service/impl/ExemplarServiceImpl.java
View File

@@ -54,6 +54,7 @@ public class ExemplarServiceImpl implements ExemplarService, CommandLineRunner {
Metadata.from(
JsonUtil.toMap(JsonUtil.toString(exemplar), String.class, Object.class));
TextSegment segment = TextSegment.from(exemplar.getQuestion(), metadata);
TextSegmentConvert.addQueryId(segment, exemplar.getQuestion());
embeddingService.deleteQuery(collection, Lists.newArrayList(segment));
}

121
common/src/main/java/dev/langchain4j/model/openai/OpenAiChatModel.java
View File

@@ -4,15 +4,23 @@ import dev.ai4j.openai4j.OpenAiClient;
import dev.ai4j.openai4j.OpenAiHttpException;
import dev.ai4j.openai4j.chat.ChatCompletionRequest;
import dev.ai4j.openai4j.chat.ChatCompletionResponse;
import dev.ai4j.openai4j.chat.ResponseFormat;
import dev.ai4j.openai4j.chat.ResponseFormatType;
import dev.langchain4j.agent.tool.ToolSpecification;
import dev.langchain4j.data.message.AiMessage;
import dev.langchain4j.data.message.ChatMessage;
import dev.langchain4j.model.Tokenizer;
import dev.langchain4j.model.chat.Capability;
import dev.langchain4j.model.chat.ChatLanguageModel;
import dev.langchain4j.model.chat.TokenCountEstimator;
import dev.langchain4j.model.chat.listener.ChatLanguageModelRequest;
import dev.langchain4j.model.chat.listener.ChatLanguageModelResponse;
import dev.langchain4j.model.listener.ModelListener;
import dev.langchain4j.model.chat.listener.ChatModelErrorContext;
import dev.langchain4j.model.chat.listener.ChatModelListener;
import dev.langchain4j.model.chat.listener.ChatModelRequest;
import dev.langchain4j.model.chat.listener.ChatModelRequestContext;
import dev.langchain4j.model.chat.listener.ChatModelResponse;
import dev.langchain4j.model.chat.listener.ChatModelResponseContext;
import dev.langchain4j.model.chat.request.ChatRequest;
import dev.langchain4j.model.chat.response.ChatResponse;
import dev.langchain4j.model.openai.spi.OpenAiChatModelBuilderFactory;
import dev.langchain4j.model.output.Response;
import lombok.Builder;
@@ -21,11 +29,17 @@ import lombok.extern.slf4j.Slf4j;
import java.net.Proxy;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import static dev.ai4j.openai4j.chat.ResponseFormatType.JSON_SCHEMA;
import static dev.langchain4j.internal.RetryUtils.withRetry;
import static dev.langchain4j.internal.Utils.getOrDefault;
import static dev.langchain4j.model.chat.Capability.RESPONSE_FORMAT_JSON_SCHEMA;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.DEFAULT_USER_AGENT;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.OPENAI_DEMO_API_KEY;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.OPENAI_DEMO_URL;
@@ -35,6 +49,7 @@ import static dev.langchain4j.model.openai.InternalOpenAiHelper.createModelListe
import static dev.langchain4j.model.openai.InternalOpenAiHelper.createModelListenerResponse;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.finishReasonFrom;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.toOpenAiMessages;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.toOpenAiResponseFormat;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.toTools;
import static dev.langchain4j.model.openai.InternalOpenAiHelper.tokenUsageFrom;
import static dev.langchain4j.model.openai.OpenAiModelName.GPT_3_5_TURBO;
@@ -62,14 +77,15 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
private final Double presencePenalty;
private final Double frequencyPenalty;
private final Map<String, Integer> logitBias;
private final String responseFormat;
private final ResponseFormat responseFormat;
private final Boolean strictJsonSchema;
private final Integer seed;
private final String user;
private final Boolean strictTools;
private final Boolean parallelToolCalls;
private final Integer maxRetries;
private final Tokenizer tokenizer;
private final List<ModelListener<ChatLanguageModelRequest, ChatLanguageModelResponse>>
listeners;
private final List<ChatModelListener> listeners;
@Builder
public OpenAiChatModel(
@@ -85,8 +101,11 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
Double frequencyPenalty,
Map<String, Integer> logitBias,
String responseFormat,
Boolean strictJsonSchema,
Integer seed,
String user,
Boolean strictTools,
Boolean parallelToolCalls,
Duration timeout,
Integer maxRetries,
Proxy proxy,
@@ -94,7 +113,7 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
Boolean logResponses,
Tokenizer tokenizer,
Map<String, String> customHeaders,
List<ModelListener<ChatLanguageModelRequest, ChatLanguageModelResponse>> listeners) {
List<ChatModelListener> listeners) {
baseUrl = getOrDefault(baseUrl, OPENAI_URL);
if (OPENAI_DEMO_API_KEY.equals(apiKey)) {
@@ -127,9 +146,19 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
this.presencePenalty = presencePenalty;
this.frequencyPenalty = frequencyPenalty;
this.logitBias = logitBias;
this.responseFormat = responseFormat;
this.responseFormat =
responseFormat == null
? null
: ResponseFormat.builder()
.type(
ResponseFormatType.valueOf(
responseFormat.toUpperCase(Locale.ROOT)))
.build();
this.strictJsonSchema = getOrDefault(strictJsonSchema, false);
this.seed = seed;
this.user = user;
this.strictTools = getOrDefault(strictTools, false);
this.parallelToolCalls = parallelToolCalls;
this.maxRetries = getOrDefault(maxRetries, 3);
this.tokenizer = getOrDefault(tokenizer, OpenAiTokenizer::new);
this.listeners = listeners == null ? emptyList() : new ArrayList<>(listeners);
@@ -139,27 +168,62 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
return modelName;
}
@Override
public Set<Capability> supportedCapabilities() {
Set<Capability> capabilities = new HashSet<>();
if (responseFormat != null && responseFormat.type() == JSON_SCHEMA) {
capabilities.add(RESPONSE_FORMAT_JSON_SCHEMA);
}
return capabilities;
}
@Override
public Response<AiMessage> generate(List<ChatMessage> messages) {
return generate(messages, null, null);
return generate(messages, null, null, this.responseFormat);
}
@Override
public Response<AiMessage> generate(
List<ChatMessage> messages, List<ToolSpecification> toolSpecifications) {
return generate(messages, toolSpecifications, null);
return generate(messages, toolSpecifications, null, this.responseFormat);
}
@Override
public Response<AiMessage> generate(
List<ChatMessage> messages, ToolSpecification toolSpecification) {
return generate(messages, singletonList(toolSpecification), toolSpecification);
return generate(
messages, singletonList(toolSpecification), toolSpecification, this.responseFormat);
}
@Override
public ChatResponse chat(ChatRequest request) {
Response<AiMessage> response =
generate(
request.messages(),
request.toolSpecifications(),
null,
getOrDefault(
toOpenAiResponseFormat(request.responseFormat(), strictJsonSchema),
this.responseFormat));
return ChatResponse.builder()
.aiMessage(response.content())
.tokenUsage(response.tokenUsage())
.finishReason(response.finishReason())
.build();
}
private Response<AiMessage> generate(
List<ChatMessage> messages,
List<ToolSpecification> toolSpecifications,
ToolSpecification toolThatMustBeExecuted) {
ToolSpecification toolThatMustBeExecuted,
ResponseFormat responseFormat) {
if (responseFormat != null
&& responseFormat.type() == JSON_SCHEMA
&& responseFormat.jsonSchema() == null) {
responseFormat = null;
}
ChatCompletionRequest.Builder requestBuilder =
ChatCompletionRequest.builder()
.model(modelName)
@@ -172,13 +236,15 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
.logitBias(logitBias)
.responseFormat(responseFormat)
.seed(seed)
.user(user);
.user(user)
.parallelToolCalls(parallelToolCalls);
if (!(baseUrl.contains(ZHIPU))) {
requestBuilder.temperature(temperature);
}
if (toolSpecifications != null && !toolSpecifications.isEmpty()) {
requestBuilder.tools(toTools(toolSpecifications));
requestBuilder.tools(toTools(toolSpecifications, strictTools));
}
if (toolThatMustBeExecuted != null) {
requestBuilder.toolChoice(toolThatMustBeExecuted.name());
@@ -186,12 +252,15 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
ChatCompletionRequest request = requestBuilder.build();
ChatLanguageModelRequest modelListenerRequest =
ChatModelRequest modelListenerRequest =
createModelListenerRequest(request, messages, toolSpecifications);
Map<Object, Object> attributes = new ConcurrentHashMap<>();
ChatModelRequestContext requestContext =
new ChatModelRequestContext(modelListenerRequest, attributes);
listeners.forEach(
listener -> {
try {
listener.onRequest(modelListenerRequest);
listener.onRequest(requestContext);
} catch (Exception e) {
log.warn("Exception while calling model listener", e);
}
@@ -208,13 +277,16 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
finishReasonFrom(
chatCompletionResponse.choices().get(0).finishReason()));
ChatLanguageModelResponse modelListenerResponse =
ChatModelResponse modelListenerResponse =
createModelListenerResponse(
chatCompletionResponse.id(), chatCompletionResponse.model(), response);
ChatModelResponseContext responseContext =
new ChatModelResponseContext(
modelListenerResponse, modelListenerRequest, attributes);
listeners.forEach(
listener -> {
try {
listener.onResponse(modelListenerResponse, modelListenerRequest);
listener.onResponse(responseContext);
} catch (Exception e) {
log.warn("Exception while calling model listener", e);
}
@@ -230,14 +302,18 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
error = e;
}
ChatModelErrorContext errorContext =
new ChatModelErrorContext(error, modelListenerRequest, null, attributes);
listeners.forEach(
listener -> {
try {
listener.onError(error, null, modelListenerRequest);
listener.onError(errorContext);
} catch (Exception e2) {
log.warn("Exception while calling model listener", e2);
}
});
throw e;
}
}
@@ -270,5 +346,10 @@ public class OpenAiChatModel implements ChatLanguageModel, TokenCountEstimator {
this.modelName = modelName;
return this;
}
public OpenAiChatModelBuilder modelName(OpenAiChatModelName modelName) {
this.modelName = modelName.toString();
return this;
}
}
}

43
common/src/main/java/dev/langchain4j/model/openai/OpenAiChatModelName.java Normal file
View File

@@ -0,0 +1,43 @@
package dev.langchain4j.model.openai;
public enum OpenAiChatModelName {
GPT_3_5_TURBO("gpt-3.5-turbo"), // alias
@Deprecated
GPT_3_5_TURBO_0613("gpt-3.5-turbo-0613"),
GPT_3_5_TURBO_1106("gpt-3.5-turbo-1106"),
GPT_3_5_TURBO_0125("gpt-3.5-turbo-0125"),
GPT_3_5_TURBO_16K("gpt-3.5-turbo-16k"), // alias
@Deprecated
GPT_3_5_TURBO_16K_0613("gpt-3.5-turbo-16k-0613"),
GPT_4("gpt-4"), // alias
@Deprecated
GPT_4_0314("gpt-4-0314"),
GPT_4_0613("gpt-4-0613"),
GPT_4_TURBO_PREVIEW("gpt-4-turbo-preview"), // alias
GPT_4_1106_PREVIEW("gpt-4-1106-preview"),
GPT_4_0125_PREVIEW("gpt-4-0125-preview"),
GPT_4_32K("gpt-4-32k"), // alias
GPT_4_32K_0314("gpt-4-32k-0314"),
GPT_4_32K_0613("gpt-4-32k-0613"),
@Deprecated
GPT_4_VISION_PREVIEW("gpt-4-vision-preview"),
GPT_4_O("gpt-4o"),
GPT_4_O_MINI("gpt-4o-mini");
private final String stringValue;
OpenAiChatModelName(String stringValue) {
this.stringValue = stringValue;
}
@Override
public String toString() {
return stringValue;
}
}

4
common/src/main/java/dev/langchain4j/store/embedding/BaseEmbeddingStoreFactory.java
View File

@@ -14,4 +14,8 @@ public abstract class BaseEmbeddingStoreFactory implements EmbeddingStoreFactory
}
public abstract EmbeddingStore<TextSegment> createEmbeddingStore(String collectionName);
public Map<String, EmbeddingStore<TextSegment>> getCollectionNameToStore() {
return collectionNameToStore;
}
}

133
common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java
View File

@@ -19,25 +19,22 @@ import io.milvus.param.dml.SearchParam;
import io.milvus.response.SearchResultsWrapper;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static dev.langchain4j.internal.Utils.getOrDefault;
import static dev.langchain4j.internal.ValidationUtils.ensureNotEmpty;
import static dev.langchain4j.internal.ValidationUtils.ensureNotNull;
import static dev.langchain4j.store.embedding.milvus.CollectionOperationsExecutor.createCollection;
import static dev.langchain4j.store.embedding.milvus.CollectionOperationsExecutor.createIndex;
import static dev.langchain4j.store.embedding.milvus.CollectionOperationsExecutor.flush;
import static dev.langchain4j.store.embedding.milvus.CollectionOperationsExecutor.hasCollection;
import static dev.langchain4j.store.embedding.milvus.CollectionOperationsExecutor.insert;
import static dev.langchain4j.store.embedding.milvus.CollectionOperationsExecutor.loadCollectionInMemory;
import static dev.langchain4j.store.embedding.milvus.CollectionOperationsExecutor.*;
import static dev.langchain4j.store.embedding.milvus.CollectionRequestBuilder.buildSearchRequest;
import static dev.langchain4j.store.embedding.milvus.Generator.generateRandomIds;
import static dev.langchain4j.store.embedding.milvus.Mapper.toEmbeddingMatches;
import static dev.langchain4j.store.embedding.milvus.Mapper.toMetadataJsons;
import static dev.langchain4j.store.embedding.milvus.Mapper.toScalars;
import static dev.langchain4j.store.embedding.milvus.Mapper.toVectors;
import static dev.langchain4j.store.embedding.milvus.Mapper.*;
import static dev.langchain4j.store.embedding.milvus.MilvusMetadataFilterMapper.formatValues;
import static dev.langchain4j.store.embedding.milvus.MilvusMetadataFilterMapper.map;
import static io.milvus.common.clientenum.ConsistencyLevelEnum.EVENTUALLY;
import static io.milvus.param.IndexType.FLAT;
import static io.milvus.param.MetricType.COSINE;
import static java.lang.String.format;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
@@ -59,7 +56,6 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
private final MetricType metricType;
private final ConsistencyLevelEnum consistencyLevel;
private final boolean retrieveEmbeddingsOnSearch;
private final boolean autoFlushOnInsert;
public MilvusEmbeddingStore(
@@ -83,7 +79,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
.withPort(getOrDefault(port, 19530))
.withUri(uri)
.withToken(token)
.withAuthorization(username, password);
.withAuthorization(getOrDefault(username, ""), getOrDefault(password, ""));
if (databaseName != null) {
connectBuilder.withDatabaseName(databaseName);
@@ -95,21 +91,26 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
this.consistencyLevel = getOrDefault(consistencyLevel, EVENTUALLY);
this.retrieveEmbeddingsOnSearch = getOrDefault(retrieveEmbeddingsOnSearch, false);
this.autoFlushOnInsert = getOrDefault(autoFlushOnInsert, false);
if (!hasCollection(milvusClient, this.collectionName)) {
if (!hasCollection(this.milvusClient, this.collectionName)) {
createCollection(
milvusClient, this.collectionName, ensureNotNull(dimension, "dimension"));
this.milvusClient, this.collectionName, ensureNotNull(dimension, "dimension"));
createIndex(
milvusClient,
this.milvusClient,
this.collectionName,
getOrDefault(indexType, FLAT),
this.metricType);
}
loadCollectionInMemory(milvusClient, collectionName);
loadCollectionInMemory(this.milvusClient, collectionName);
}
public static Builder builder() {
return new Builder();
}
public void dropCollection(String collectionName) {
CollectionOperationsExecutor.dropCollection(milvusClient, collectionName);
CollectionOperationsExecutor.dropCollection(this.milvusClient, collectionName);
}
public String add(Embedding embedding) {
@@ -189,14 +190,104 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
METADATA_FIELD_NAME, toMetadataJsons(textSegments, ids.size())));
fields.add(new InsertParam.Field(VECTOR_FIELD_NAME, toVectors(embeddings)));
insert(milvusClient, collectionName, fields);
insert(this.milvusClient, this.collectionName, fields);
if (autoFlushOnInsert) {
flush(this.milvusClient, this.collectionName);
}
}
public static Builder builder() {
return new Builder();
/**
* Removes a single embedding from the store by ID.
*
* <p>CAUTION
*
* <ul>
* <li>Deleted entities can still be retrieved immediately after the deletion if the
* consistency level is set lower than {@code Strong}
* <li>Entities deleted beyond the pre-specified span of time for Time Travel cannot be
* retrieved again.
* <li>Frequent deletion operations will impact the system performance.
* <li>Before deleting entities by comlpex boolean expressions, make sure the collection has
* been loaded.
* <li>Deleting entities by complex boolean expressions is not an atomic operation. Therefore,
* if it fails halfway through, some data may still be deleted.
* <li>Deleting entities by complex boolean expressions is supported only when the consistency
* is set to Bounded. For details, <a
* href="https://milvus.io/docs/v2.3.x/consistency.md#Consistency-levels">see
* Consistency</a>
* </ul>
*
* @param ids A collection of unique IDs of the embeddings to be removed.
* @since Milvus version 2.3.x
*/
@Override
public void removeAll(Collection<String> ids) {
ensureNotEmpty(ids, "ids");
removeForVector(
this.milvusClient,
this.collectionName,
format("%s in %s", ID_FIELD_NAME, formatValues(ids)));
}
/**
* Removes all embeddings that match the specified {@link Filter} from the store.
*
* <p>CAUTION
*
* <ul>
* <li>Deleted entities can still be retrieved immediately after the deletion if the
* consistency level is set lower than {@code Strong}
* <li>Entities deleted beyond the pre-specified span of time for Time Travel cannot be
* retrieved again.
* <li>Frequent deletion operations will impact the system performance.
* <li>Before deleting entities by comlpex boolean expressions, make sure the collection has
* been loaded.
* <li>Deleting entities by complex boolean expressions is not an atomic operation. Therefore,
* if it fails halfway through, some data may still be deleted.
* <li>Deleting entities by complex boolean expressions is supported only when the consistency
* is set to Bounded. For details, <a
* href="https://milvus.io/docs/v2.3.x/consistency.md#Consistency-levels">see
* Consistency</a>
* </ul>
*
* @param filter The filter to be applied to the {@link Metadata} of the {@link TextSegment}
* during removal. Only embeddings whose {@code TextSegment}'s {@code Metadata} match the
* {@code Filter} will be removed.
* @since Milvus version 2.3.x
*/
@Override
public void removeAll(Filter filter) {
ensureNotNull(filter, "filter");
removeForVector(this.milvusClient, this.collectionName, map(filter));
}
/**
* Removes all embeddings from the store.
*
* <p>CAUTION
*
* <ul>
* <li>Deleted entities can still be retrieved immediately after the deletion if the
* consistency level is set lower than {@code Strong}
* <li>Entities deleted beyond the pre-specified span of time for Time Travel cannot be
* retrieved again.
* <li>Frequent deletion operations will impact the system performance.
* <li>Before deleting entities by comlpex boolean expressions, make sure the collection has
* been loaded.
* <li>Deleting entities by complex boolean expressions is not an atomic operation. Therefore,
* if it fails halfway through, some data may still be deleted.
* <li>Deleting entities by complex boolean expressions is supported only when the consistency
* is set to Bounded. For details, <a
* href="https://milvus.io/docs/v2.3.x/consistency.md#Consistency-levels">see
* Consistency</a>
* </ul>
*
* @since Milvus version 2.3.x
*/
@Override
public void removeAll() {
removeForVector(
this.milvusClient, this.collectionName, format("%s != \"\"", ID_FIELD_NAME));
}
public static class Builder {
@@ -213,8 +304,8 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
private String password;
private ConsistencyLevelEnum consistencyLevel;
private Boolean retrieveEmbeddingsOnSearch;
private Boolean autoFlushOnInsert;
private String databaseName;
private Boolean autoFlushOnInsert;
/**
* @param host The host of the self-managed Milvus instance. Default value: "localhost".

22
headless/server/pom.xml
View File

@@ -114,6 +114,28 @@
<groupId>org.apache.arrow</groupId>
<artifactId>flight-sql</artifactId>
<version>${flight-sql.version}</version>
<exclusions>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java
View File

@@ -7,6 +7,7 @@ import javax.validation.Valid;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.service.EmbeddingService;
import com.tencent.supersonic.common.service.ExemplarService;
import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
@@ -43,6 +44,8 @@ public class KnowledgeController {
@Autowired private ExemplarService exemplarService;
@Autowired private EmbeddingService embeddingService;
/**
* addDictConf-新增item的字典配置 Add configuration information for dictionary entries
*
@@ -115,7 +118,7 @@ public class KnowledgeController {
/** dailyDictTask-手动离线更新所有字典 */
@PutMapping("/task/all")
public Boolean dailyDictTask(HttpServletRequest request, HttpServletResponse response) {
public Boolean dailyDictTask() {
return taskService.dailyDictTask();
}
@@ -140,6 +143,12 @@ public class KnowledgeController {
return true;
}
@GetMapping("/embedding/reset")
public Object resetEmbedding() {
embeddingService.removeAll();
return reloadEmbedding();
}
@GetMapping("/embedding/persistFile")
public Object executePersistFileTask() {
metaEmbeddingTask.executePersistFileTask();
@@ -175,10 +184,7 @@ public class KnowledgeController {
}
@PostMapping("/dict/reload")
public boolean reloadKnowledge(
@RequestBody @Valid DictValueReq dictValueReq,
HttpServletRequest request,
HttpServletResponse response) {
public boolean reloadKnowledge() {
dictionaryReloadTask.reloadKnowledge();
return true;
}

117
headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java
View File

@@ -1,117 +0,0 @@
package com.tencent.supersonic.headless.server.task;
import javax.annotation.PreDestroy;
import com.tencent.supersonic.headless.server.service.FlightService;
import lombok.extern.slf4j.Slf4j;
import org.apache.arrow.flight.FlightServer;
import org.apache.arrow.flight.Location;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/** Initialize flight jdbc server */
@Component
@Slf4j
public class FlightServerInitTask implements CommandLineRunner {
@Value("${s2.flightSql.enable:false}")
private Boolean enable = false;
@Value("${s2.flightSql.host:localhost}")
private String host = "localhost";
@Value("${s2.flightSql.port:9081}")
private Integer port = 9081;
@Value("${s2.flightSql.executor:4}")
private Integer executor = 4;
@Value("${s2.flightSql.queue:128}")
private Integer queue = 128;
@Value("${s2.flightSql.expireMinute:10}")
private Integer expireMinute = 10;
private final FlightService flightService;
private ExecutorService executorService;
private FlightServer flightServer;
private BufferAllocator allocator;
private Boolean isRunning = false;
public FlightServerInitTask(FlightService flightService) {
this.allocator = new RootAllocator();
this.flightService = flightService;
this.flightService.setLocation(host, port);
executorService = Executors.newFixedThreadPool(executor);
this.flightService.setExecutorService(executorService, queue, expireMinute);
Location listenLocation = Location.forGrpcInsecure(host, port);
flightServer = FlightServer.builder(allocator, listenLocation, this.flightService).build();
}
public String getHost() {
return host;
}
public Integer getPort() {
return port;
}
public void startServer() {
try {
log.info("Arrow Flight JDBC server started on {} {}", host, port);
flightServer.start();
isRunning = true;
} catch (Exception e) {
log.error("FlightServerInitTask start error {}", e);
}
}
public Boolean isRunning() {
return isRunning;
}
@PreDestroy
public void onShutdown() {
try {
log.info("Arrow Flight JDBC server stop on {} {}", host, port);
flightServer.close();
allocator.close();
} catch (Exception e) {
log.error("FlightServerInitTask start error {}", e);
}
}
@Override
public void run(String... args) throws Exception {
if (enable) {
new Thread() {
@Override
public void run() {
try {
startServer();
Runtime.getRuntime()
.addShutdownHook(
new Thread(
() -> {
try {
flightServer.close();
allocator.close();
} catch (Exception e) {
log.error("flightServer close error {}", e);
}
}));
// flightServer.awaitTermination();
} catch (Exception e) {
log.error("run error {}", e);
}
}
}.start();
}
}
}

8
launchers/standalone/src/main/resources/application.yaml
View File

@@ -13,14 +13,6 @@ spring:
config:
import:
- classpath:s2-config.yaml
autoconfigure:
exclude:
- spring.dev.langchain4j.spring.LangChain4jAutoConfig
- spring.dev.langchain4j.openai.spring.AutoConfig
- spring.dev.langchain4j.ollama.spring.AutoConfig
- spring.dev.langchain4j.azure.openai.spring.AutoConfig
- spring.dev.langchain4j.azure.aisearch.spring.AutoConfig
- spring.dev.langchain4j.anthropic.spring.AutoConfig
main:
allow-circular-references: true
mvc:

109
launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java
View File

@@ -1,109 +0,0 @@
package com.tencent.supersonic.headless;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.auth.authentication.strategy.FakeUserStrategy;
import com.tencent.supersonic.headless.server.task.FlightServerInitTask;
import lombok.extern.slf4j.Slf4j;
import org.apache.arrow.flight.CallHeaders;
import org.apache.arrow.flight.FlightCallHeaders;
import org.apache.arrow.flight.FlightClient;
import org.apache.arrow.flight.FlightInfo;
import org.apache.arrow.flight.FlightStream;
import org.apache.arrow.flight.HeaderCallOption;
import org.apache.arrow.flight.Location;
import org.apache.arrow.flight.sql.FlightSqlClient;
import org.apache.arrow.memory.RootAllocator;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@Slf4j
public class FlightSqlTest extends BaseTest {
@Autowired private FlightServerInitTask flightSqlListener;
@Autowired private FakeUserStrategy fakeUserStrategy;
@Test
void test01() throws Exception {
startServer();
String host = flightSqlListener.getHost();
Integer port = flightSqlListener.getPort();
FlightSqlClient sqlClient =
new FlightSqlClient(
FlightClient.builder(
new RootAllocator(Integer.MAX_VALUE),
Location.forGrpcInsecure(host, port))
.build());
CallHeaders headers = new FlightCallHeaders();
headers.insert("dataSetId", "1");
headers.insert("name", "admin");
headers.insert("password", "admin");
HeaderCallOption headerOption = new HeaderCallOption(headers);
try (final FlightSqlClient.PreparedStatement preparedStatement =
sqlClient.prepare(
"SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门", headerOption)) {
final FlightInfo info = preparedStatement.execute();
FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket());
int rowCnt = 0;
int colCnt = 0;
while (stream.next()) {
if (stream.getRoot().getRowCount() > 0) {
colCnt = stream.getRoot().getFieldVectors().size();
rowCnt += stream.getRoot().getRowCount();
}
}
assertEquals(2, colCnt);
assertTrue(rowCnt > 0);
} catch (Exception e) {
log.error("", e);
}
}
@Test
void test02() throws Exception {
startServer();
String host = flightSqlListener.getHost();
Integer port = flightSqlListener.getPort();
FlightSqlClient sqlClient =
new FlightSqlClient(
FlightClient.builder(
new RootAllocator(Integer.MAX_VALUE),
Location.forGrpcInsecure(host, port))
.build());
CallHeaders headers = new FlightCallHeaders();
headers.insert("dataSetId", "1");
headers.insert("name", "admin");
headers.insert("password", "admin");
HeaderCallOption headerOption = new HeaderCallOption(headers);
try {
FlightInfo flightInfo =
sqlClient.execute(
"SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门",
headerOption);
FlightStream stream = sqlClient.getStream(flightInfo.getEndpoints().get(0).getTicket());
int rowCnt = 0;
int colCnt = 0;
while (stream.next()) {
if (stream.getRoot().getRowCount() > 0) {
colCnt = stream.getRoot().getFieldVectors().size();
rowCnt += stream.getRoot().getRowCount();
}
}
assertEquals(2, colCnt);
assertTrue(rowCnt > 0);
} catch (Exception e) {
log.error("", e);
}
}
private void startServer() {
if (!flightSqlListener.isRunning()) {
UserHolder.setStrategy(fakeUserStrategy);
flightSqlListener.startServer();
}
}
}

8
launchers/standalone/src/test/resources/application.yaml
View File

@@ -13,14 +13,6 @@ spring:
config:
import:
- classpath:s2-config.yaml
autoconfigure:
exclude:
- spring.dev.langchain4j.spring.LangChain4jAutoConfig
- spring.dev.langchain4j.openai.spring.AutoConfig
- spring.dev.langchain4j.ollama.spring.AutoConfig
- spring.dev.langchain4j.azure.openai.spring.AutoConfig
- spring.dev.langchain4j.azure.aisearch.spring.AutoConfig
- spring.dev.langchain4j.anthropic.spring.AutoConfig
main:
allow-circular-references: true
mvc:

37
pom.xml
View File

@@ -66,7 +66,7 @@
<mockito-inline.version>4.5.1</mockito-inline.version>
<easyexcel.version>2.2.6</easyexcel.version>
<poi.version>3.17</poi.version>
<langchain4j.version>0.31.0</langchain4j.version>
<langchain4j.version>0.34.0</langchain4j.version>
<langchain4j.embedding.version>0.27.1</langchain4j.embedding.version>
<postgresql.version>42.7.1</postgresql.version>
<st.version>4.0.8</st.version>
@@ -145,36 +145,6 @@
<artifactId>langchain4j-azure-open-ai</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-anthropic-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-ollama-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-open-ai-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-azure-ai-search-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-azure-open-ai-spring-boot-starter</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-embeddings-all-minilm-l6-v2-q</artifactId>
@@ -205,6 +175,11 @@
<artifactId>langchain4j-chatglm</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>dev.langchain4j</groupId>
<artifactId>langchain4j-ollama</artifactId>
<version>${langchain4j.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>

29054
webapp/pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff