From cf41fa14e2e2ec994023a9d1959f33ec62a953f4 Mon Sep 17 00:00:00 2001 From: beat4ocean Date: Sat, 15 Mar 2025 13:15:18 +0800 Subject: [PATCH] [Improvement] Upgrade dependencies fix vulnerabilities. --- auth/api/pom.xml | 4 +- .../calcite/SemanticSqlConformance.java | 5 -- .../inmemory/spring/InMemoryAutoConfig.java | 4 +- .../model/embedding/S2OnnxEmbeddingModel.java | 7 ++- .../provider/EmbeddingModelConstant.java | 4 +- .../milvus/MilvusEmbeddingStore.java | 19 +++--- headless/core/pom.xml | 2 +- .../server/task/MetaEmbeddingTask.java | 4 +- launchers/chat/pom.xml | 4 +- launchers/headless/pom.xml | 8 +-- launchers/pom.xml | 4 +- launchers/standalone/pom.xml | 8 +-- pom.xml | 59 +++++++++++-------- 13 files changed, 73 insertions(+), 59 deletions(-) diff --git a/auth/api/pom.xml b/auth/api/pom.xml index c3f97fb87..9fa621fa7 100644 --- a/auth/api/pom.xml +++ b/auth/api/pom.xml @@ -34,8 +34,8 @@ - 8 - 8 + 21 + 21 \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlConformance.java b/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlConformance.java index 06107c2b6..8e900007b 100644 --- a/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlConformance.java +++ b/common/src/main/java/com/tencent/supersonic/common/calcite/SemanticSqlConformance.java @@ -77,11 +77,6 @@ public class SemanticSqlConformance implements SqlConformance { return SqlConformanceEnum.BIG_QUERY.isMinusAllowed(); } - @Override - public boolean isRegexReplaceCaptureGroupDollarIndexed() { - return SqlConformanceEnum.BIG_QUERY.isRegexReplaceCaptureGroupDollarIndexed(); - } - @Override public boolean isApplyAllowed() { return SqlConformanceEnum.BIG_QUERY.isApplyAllowed(); diff --git a/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java b/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java index 5b19c05e4..d78f8838e 100644 --- a/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java +++ b/common/src/main/java/dev/langchain4j/inmemory/spring/InMemoryAutoConfig.java @@ -1,9 +1,9 @@ package dev.langchain4j.inmemory.spring; -import dev.langchain4j.model.embedding.AllMiniLmL6V2QuantizedEmbeddingModel; -import dev.langchain4j.model.embedding.BgeSmallZhEmbeddingModel; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.embedding.S2OnnxEmbeddingModel; +import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel; +import dev.langchain4j.model.embedding.onnx.bgesmallzh.BgeSmallZhEmbeddingModel; import dev.langchain4j.provider.EmbeddingModelConstant; import dev.langchain4j.store.embedding.EmbeddingStoreFactory; import org.apache.commons.lang3.StringUtils; diff --git a/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java b/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java index 45db6c414..cfb13906a 100644 --- a/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java +++ b/common/src/main/java/dev/langchain4j/model/embedding/S2OnnxEmbeddingModel.java @@ -1,5 +1,8 @@ package dev.langchain4j.model.embedding; +import dev.langchain4j.model.embedding.onnx.AbstractInProcessEmbeddingModel; +import dev.langchain4j.model.embedding.onnx.OnnxBertBiEncoder; +import dev.langchain4j.model.embedding.onnx.PoolingMode; import org.apache.commons.lang3.StringUtils; import java.io.IOException; @@ -9,6 +12,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Objects; +import java.util.concurrent.Executors; /** * An embedding model that runs within your Java application's process. Any BERT-based model (e.g., @@ -25,6 +29,7 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel { private static volatile String cachedVocabularyPath; public S2OnnxEmbeddingModel(String pathToModel, String vocabularyPath) { + super(Executors.newSingleThreadExecutor()); if (shouldReloadModel(pathToModel, vocabularyPath)) { synchronized (S2OnnxEmbeddingModel.class) { if (shouldReloadModel(pathToModel, vocabularyPath)) { @@ -61,7 +66,7 @@ public class S2OnnxEmbeddingModel extends AbstractInProcessEmbeddingModel { static OnnxBertBiEncoder loadFromFileSystem(Path pathToModel, URL vocabularyFile) { try { - return new OnnxBertBiEncoder(Files.newInputStream(pathToModel), vocabularyFile, + return new OnnxBertBiEncoder(Files.newInputStream(pathToModel), vocabularyFile.openStream(), PoolingMode.MEAN); } catch (IOException e) { throw new RuntimeException(e); diff --git a/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java b/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java index ccadba509..85feb9b25 100644 --- a/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java +++ b/common/src/main/java/dev/langchain4j/provider/EmbeddingModelConstant.java @@ -1,8 +1,8 @@ package dev.langchain4j.provider; -import dev.langchain4j.model.embedding.AllMiniLmL6V2QuantizedEmbeddingModel; -import dev.langchain4j.model.embedding.BgeSmallZhEmbeddingModel; import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel; +import dev.langchain4j.model.embedding.onnx.bgesmallzh.BgeSmallZhEmbeddingModel; import org.springframework.stereotype.Service; @Service diff --git a/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java b/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java index a2eb5718e..1097964e1 100644 --- a/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java +++ b/common/src/main/java/dev/langchain4j/store/embedding/milvus/MilvusEmbeddingStore.java @@ -57,6 +57,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { private final ConsistencyLevelEnum consistencyLevel; private final boolean retrieveEmbeddingsOnSearch; private final boolean autoFlushOnInsert; + private final FieldDefinition fieldDefinition; public MilvusEmbeddingStore(String host, Integer port, String collectionName, Integer dimension, IndexType indexType, MetricType metricType, String uri, String token, String username, @@ -78,11 +79,15 @@ public class MilvusEmbeddingStore implements EmbeddingStore { this.retrieveEmbeddingsOnSearch = getOrDefault(retrieveEmbeddingsOnSearch, false); this.autoFlushOnInsert = getOrDefault(autoFlushOnInsert, false); + // Define the field structure for the collection + this.fieldDefinition = new FieldDefinition(ID_FIELD_NAME, TEXT_FIELD_NAME, + METADATA_FIELD_NAME, VECTOR_FIELD_NAME); + if (!hasCollection(this.milvusClient, this.collectionName)) { - createCollection(this.milvusClient, this.collectionName, - ensureNotNull(dimension, "dimension")); - createIndex(this.milvusClient, this.collectionName, getOrDefault(indexType, FLAT), - this.metricType); + createCollection(this.milvusClient, this.collectionName, fieldDefinition, + ensureNotNull(dimension, "dimension")); + createIndex(this.milvusClient, this.collectionName, VECTOR_FIELD_NAME, + getOrDefault(indexType, FLAT), this.metricType); } loadCollectionInMemory(this.milvusClient, collectionName); @@ -128,7 +133,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { public EmbeddingSearchResult search( EmbeddingSearchRequest embeddingSearchRequest) { - SearchParam searchParam = buildSearchRequest(collectionName, + SearchParam searchParam = buildSearchRequest(collectionName, fieldDefinition, embeddingSearchRequest.queryEmbedding().vectorAsList(), embeddingSearchRequest.filter(), embeddingSearchRequest.maxResults(), metricType, consistencyLevel); @@ -137,7 +142,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { CollectionOperationsExecutor.search(milvusClient, searchParam); List> matches = toEmbeddingMatches(milvusClient, resultsWrapper, - collectionName, consistencyLevel, retrieveEmbeddingsOnSearch); + collectionName, fieldDefinition, consistencyLevel, retrieveEmbeddingsOnSearch); List> result = matches.stream().filter(match -> match.score() >= embeddingSearchRequest.minScore()) @@ -226,7 +231,7 @@ public class MilvusEmbeddingStore implements EmbeddingStore { @Override public void removeAll(Filter filter) { ensureNotNull(filter, "filter"); - removeForVector(this.milvusClient, this.collectionName, map(filter)); + removeForVector(this.milvusClient, this.collectionName, map(filter, METADATA_FIELD_NAME)); } /** diff --git a/headless/core/pom.xml b/headless/core/pom.xml index d4e829879..c230b96de 100644 --- a/headless/core/pom.xml +++ b/headless/core/pom.xml @@ -64,7 +64,7 @@ com.alibaba transmittable-thread-local - ${transmittable.thread.local.version} + ${transmittable.version} org.aspectj diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java index 2b4dce434..e546a9847 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java @@ -44,10 +44,8 @@ public class MetaEmbeddingTask implements CommandLineRunner { private void embeddingStorePersistFile() { EmbeddingStoreFactory embeddingStoreFactory = EmbeddingStoreFactoryProvider.getFactory(); - if (embeddingStoreFactory instanceof InMemoryEmbeddingStoreFactory) { + if (embeddingStoreFactory instanceof InMemoryEmbeddingStoreFactory inMemoryFactory) { long startTime = System.currentTimeMillis(); - InMemoryEmbeddingStoreFactory inMemoryFactory = - (InMemoryEmbeddingStoreFactory) embeddingStoreFactory; inMemoryFactory.persistFile(); long duration = System.currentTimeMillis() - startTime; log.info("Embedding file has been regularly persisted in {} milliseconds", duration); diff --git a/launchers/chat/pom.xml b/launchers/chat/pom.xml index 10f396b7d..0132f67f0 100644 --- a/launchers/chat/pom.xml +++ b/launchers/chat/pom.xml @@ -60,7 +60,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 + 3.4.2 *.* @@ -70,7 +70,7 @@ org.apache.maven.plugins maven-assembly-plugin - 2.4 + 3.7.1 gnu false diff --git a/launchers/headless/pom.xml b/launchers/headless/pom.xml index 531a3f17f..6fe06b97e 100644 --- a/launchers/headless/pom.xml +++ b/launchers/headless/pom.xml @@ -12,8 +12,8 @@ launchers-headless - 8 - 8 + 21 + 21 com.tencent.supersonic.HeadlessLauncher @@ -71,7 +71,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 + 3.4.2 *.* @@ -81,7 +81,7 @@ org.apache.maven.plugins maven-assembly-plugin - 2.4 + 3.7.1 gnu false diff --git a/launchers/pom.xml b/launchers/pom.xml index 451cb27b9..773da1313 100644 --- a/launchers/pom.xml +++ b/launchers/pom.xml @@ -20,7 +20,7 @@ - 8 - 8 + 21 + 21 diff --git a/launchers/standalone/pom.xml b/launchers/standalone/pom.xml index f4d8322ee..71a087b3d 100644 --- a/launchers/standalone/pom.xml +++ b/launchers/standalone/pom.xml @@ -55,8 +55,8 @@ - mysql - mysql-connector-java + com.mysql + mysql-connector-j @@ -149,7 +149,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.4 + 3.4.2 *.* @@ -159,7 +159,7 @@ org.apache.maven.plugins maven-assembly-plugin - 2.4 + 3.7.1 gnu false diff --git a/pom.xml b/pom.xml index 954ff9cc7..92921c6f2 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.springframework.boot spring-boot-starter-parent - 3.2.4 + 3.3.9 @@ -32,55 +32,54 @@ 21 21 UTF-8 - 4.7 + 4.9 6.1.0 2.1.0 - 3.5.3 + 3.5.19 32.0.0-jre - portable-1.8.3 + portable-1.8.4 2.7.2 2.6 3.7 6.13.1 2.14.1 - 2.12.1 - 0.12.3 + 0.12.6 1.2.24 - 5.1.46 + 9.2.0 1.10.1 0.291 471 - 3.5.7 - 5.4.1 + 3.5.10.1 + 5.4.2 - 5.3.1 + 5.3.3 0.4.6 - 2.0.40 + 2.0.56 7.0.0 - 2.12.1 - 1.26.0 + 2.14.5 + 1.27.1 6.1.26 2.8.0 - 1.37.0 - 1.23.0 + 1.38.0 + 1.26.0 3.2.4 4.5.1 - 2.2.6 + 2.2.11 3.17 - 0.35.0 - 0.27.1 + 0.36.2 + 0.36.2 4.0.8 0.10.0 15.0.2 15.0.2 15.0.2 - 2.10.1 + 2.12.1 2.27.1 false - 4.2.1 + 4.2.2 1.12.780 1.5.2 @@ -208,8 +207,8 @@ ${stax2.version} - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.connector.java.version} @@ -324,12 +323,24 @@ org.sonarsource.scanner.maven sonar-maven-plugin - 3.6.0.1398 + 3.6.1.1688 org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + 3.5.2 + + + org.junit.jupiter + junit-jupiter + 5.11.4 + + + org.junit.platform + junit-platform-launcher + 1.11.4 + + true