topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 19:51:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Improvement] Upgrade dependencies and fix vulnerabilities. (#2190)
Some checks failed
supersonic CentOS CI / build (21) (push) Has been cancelled
Details
supersonic mac CI / build (21) (push) Has been cancelled
Details
supersonic ubuntu CI / build (21) (push) Has been cancelled
Details
supersonic windows CI / build (21) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
beat4ocean
2025-03-28 09:07:49 +08:00
committed by GitHub
parent 791c493a6a
commit 232a202275
20 changed files with 130 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlConformance.java
View File

@@ -77,11 +77,6 @@ public class SemanticSqlConformance implements SqlConformance {
return SqlConformanceEnum.BIG_QUERY.isMinusAllowed();
}
@Override
public boolean isRegexReplaceCaptureGroupDollarIndexed() {
return SqlConformanceEnum.BIG_QUERY.isRegexReplaceCaptureGroupDollarIndexed();
}
@Override
public boolean isApplyAllowed() {
return SqlConformanceEnum.BIG_QUERY.isApplyAllowed();

4
common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java
View File

@@ -1,9 +1,9 @@
package dev.langchain4j.inmemory.spring;
import dev.langchain4j.model.embedding.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.model.embedding.BgeSmallZhEmbeddingModel;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.model.embedding.S2OnnxEmbeddingModel;
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.model.embedding.onnx.bgesmallzh.BgeSmallZhEmbeddingModel;
import dev.langchain4j.provider.EmbeddingModelConstant;
import dev.langchain4j.store.embedding.EmbeddingStoreFactory;
import org.apache.commons.lang3.StringUtils;

7
common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java
View File

@@ -1,5 +1,8 @@
package dev.langchain4j.model.embedding;
import dev.langchain4j.model.embedding.onnx.AbstractInProcessEmbeddingModel;
import dev.langchain4j.model.embedding.onnx.OnnxBertBiEncoder;
import dev.langchain4j.model.embedding.onnx.PoolingMode;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
@@ -9,6 +12,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.concurrent.Executors;
/**
* An embedding model that runs within your Java application's process. Any BERT-based model (e.g.,
@@ -25,6 +29,7 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel {
private static volatile String cachedVocabularyPath;
public S2OnnxEmbeddingModel(String pathToModel, String vocabularyPath) {
super(Executors.newSingleThreadExecutor());
if (shouldReloadModel(pathToModel, vocabularyPath)) {
synchronized (S2OnnxEmbeddingModel.class) {
if (shouldReloadModel(pathToModel, vocabularyPath)) {
@@ -61,7 +66,7 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel {
static OnnxBertBiEncoder loadFromFileSystem(Path pathToModel, URL vocabularyFile) {
try {
return new OnnxBertBiEncoder(Files.newInputStream(pathToModel), vocabularyFile,
return new OnnxBertBiEncoder(Files.newInputStream(pathToModel), vocabularyFile.openStream(),
PoolingMode.MEAN);
} catch (IOException e) {
throw new RuntimeException(e);

4
common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java
View File

@@ -1,8 +1,8 @@
package dev.langchain4j.provider;
import dev.langchain4j.model.embedding.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.model.embedding.BgeSmallZhEmbeddingModel;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.model.embedding.onnx.bgesmallzh.BgeSmallZhEmbeddingModel;
import org.springframework.stereotype.Service;
@Service

19
common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java
View File

@@ -57,6 +57,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
private final ConsistencyLevelEnum consistencyLevel;
private final boolean retrieveEmbeddingsOnSearch;
private final boolean autoFlushOnInsert;
private final FieldDefinition fieldDefinition;
public MilvusEmbeddingStore(String host, Integer port, String collectionName, Integer dimension,
IndexType indexType, MetricType metricType, String uri, String token, String username,
@@ -78,11 +79,15 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
this.retrieveEmbeddingsOnSearch = getOrDefault(retrieveEmbeddingsOnSearch, false);
this.autoFlushOnInsert = getOrDefault(autoFlushOnInsert, false);
// Define the field structure for the collection
this.fieldDefinition = new FieldDefinition(ID_FIELD_NAME, TEXT_FIELD_NAME,
METADATA_FIELD_NAME, VECTOR_FIELD_NAME);
if (!hasCollection(this.milvusClient, this.collectionName)) {
createCollection(this.milvusClient, this.collectionName,
ensureNotNull(dimension, "dimension"));
createIndex(this.milvusClient, this.collectionName, getOrDefault(indexType, FLAT),
this.metricType);
createCollection(this.milvusClient, this.collectionName, fieldDefinition,
ensureNotNull(dimension, "dimension"));
createIndex(this.milvusClient, this.collectionName, VECTOR_FIELD_NAME,
getOrDefault(indexType, FLAT), this.metricType);
}
loadCollectionInMemory(this.milvusClient, collectionName);
@@ -128,7 +133,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
public EmbeddingSearchResult<TextSegment> search(
EmbeddingSearchRequest embeddingSearchRequest) {
SearchParam searchParam = buildSearchRequest(collectionName,
SearchParam searchParam = buildSearchRequest(collectionName, fieldDefinition,
embeddingSearchRequest.queryEmbedding().vectorAsList(),
embeddingSearchRequest.filter(), embeddingSearchRequest.maxResults(), metricType,
consistencyLevel);
@@ -137,7 +142,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
CollectionOperationsExecutor.search(milvusClient, searchParam);
List<EmbeddingMatch<TextSegment>> matches = toEmbeddingMatches(milvusClient, resultsWrapper,
collectionName, consistencyLevel, retrieveEmbeddingsOnSearch);
collectionName, fieldDefinition, consistencyLevel, retrieveEmbeddingsOnSearch);
List<EmbeddingMatch<TextSegment>> result =
matches.stream().filter(match -> match.score() >= embeddingSearchRequest.minScore())
@@ -226,7 +231,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore<TextSegment> {
@Override
public void removeAll(Filter filter) {
ensureNotNull(filter, "filter");
removeForVector(this.milvusClient, this.collectionName, map(filter));
removeForVector(this.milvusClient, this.collectionName, map(filter, METADATA_FIELD_NAME));
}
/**