From f96c0871f6956157d082775c5c16d1fb5b676a5b Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Sun, 30 Jun 2024 11:51:10 +0800 Subject: [PATCH] (improvement)(headless)Move task related classes to dedicated package. --- .../listener/FullMetaEmbeddingListener.java | 33 ------------------- .../DictionaryReloadTask.java} | 4 +-- .../FlightServerInitTask.java} | 17 ++++++---- .../MetaEmbeddingTask.java} | 18 ++++++++-- .../server/web/rest/KnowledgeController.java | 4 +-- .../web/service/impl/DictWordService.java | 4 +-- .../supersonic/headless/FlightSqlTest.java | 4 +-- 7 files changed, 34 insertions(+), 50 deletions(-) delete mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FullMetaEmbeddingListener.java rename headless/server/src/main/java/com/tencent/supersonic/headless/server/{listener/DictWordLoadStartedListener.java => task/DictionaryReloadTask.java} (90%) rename headless/server/src/main/java/com/tencent/supersonic/headless/server/{listener/FlightSqlListener.java => task/FlightServerInitTask.java} (88%) rename headless/server/src/main/java/com/tencent/supersonic/headless/server/{schedule/EmbeddingTask.java => task/MetaEmbeddingTask.java} (85%) diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FullMetaEmbeddingListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FullMetaEmbeddingListener.java deleted file mode 100644 index 766fe2b43..000000000 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FullMetaEmbeddingListener.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.tencent.supersonic.headless.server.listener; - -import com.tencent.supersonic.headless.chat.utils.ComponentFactory; -import com.tencent.supersonic.headless.chat.parser.llm.JavaLLMProxy; -import com.tencent.supersonic.headless.server.schedule.EmbeddingTask; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Component; - -@Slf4j -@Component -@Order(2) -public class FullMetaEmbeddingListener implements CommandLineRunner { - @Autowired - private EmbeddingTask embeddingTask; - - @Override - public void run(String... args) { - initMetaEmbedding(); - } - - public void initMetaEmbedding() { - try { - if (ComponentFactory.getLLMProxy() instanceof JavaLLMProxy) { - embeddingTask.reloadMetaEmbedding(); - } - } catch (Exception e) { - log.error("initMetaEmbedding error", e); - } - } -} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/DictWordLoadStartedListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java similarity index 90% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/DictWordLoadStartedListener.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java index 0d6db7a02..1df9b2694 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/DictWordLoadStartedListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.listener; +package com.tencent.supersonic.headless.server.task; import com.tencent.supersonic.headless.server.web.service.impl.DictWordService; import lombok.extern.slf4j.Slf4j; @@ -11,7 +11,7 @@ import org.springframework.stereotype.Component; @Slf4j @Component @Order(2) -public class DictWordLoadStartedListener implements CommandLineRunner { +public class DictionaryReloadTask implements CommandLineRunner { @Autowired private DictWordService dictWordService; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FlightSqlListener.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java similarity index 88% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FlightSqlListener.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java index 01e19e1bc..4585d88ee 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FlightSqlListener.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.listener; +package com.tencent.supersonic.headless.server.task; import com.tencent.supersonic.headless.server.facade.service.FlightService; import java.util.concurrent.ExecutorService; @@ -12,12 +12,14 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; +import javax.annotation.PreDestroy; + /** - * arrow flight jdbc server listener + * Initialize flight jdbc server */ @Component @Slf4j -public class FlightSqlListener implements CommandLineRunner { +public class FlightServerInitTask implements CommandLineRunner { @Value("${s2.flightSql.enable:false}") private Boolean enable = false; @@ -38,7 +40,7 @@ public class FlightSqlListener implements CommandLineRunner { private BufferAllocator allocator; private Boolean isRunning = false; - public FlightSqlListener(FlightService flightService) { + public FlightServerInitTask(FlightService flightService) { this.allocator = new RootAllocator(); this.flightService = flightService; this.flightService.setLocation(host, port); @@ -63,7 +65,7 @@ public class FlightSqlListener implements CommandLineRunner { flightServer.start(); isRunning = true; } catch (Exception e) { - log.error("FlightSqlListener start error {}", e); + log.error("FlightServerInitTask start error {}", e); } } @@ -72,13 +74,14 @@ public class FlightSqlListener implements CommandLineRunner { return isRunning; } - public void stop() { + @PreDestroy + public void onShutdown() { try { log.info("Arrow Flight JDBC server stop on {} {}", host, port); flightServer.close(); allocator.close(); } catch (Exception e) { - log.error("FlightSqlListener start error {}", e); + log.error("FlightServerInitTask start error {}", e); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/schedule/EmbeddingTask.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java similarity index 85% rename from headless/server/src/main/java/com/tencent/supersonic/headless/server/schedule/EmbeddingTask.java rename to headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java index dc250bad3..659c55640 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/schedule/EmbeddingTask.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.server.schedule; +package com.tencent.supersonic.headless.server.task; import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.pojo.DataItem; @@ -10,6 +10,8 @@ import dev.langchain4j.store.embedding.EmbeddingStoreFactory; import dev.langchain4j.store.embedding.TextSegmentConvert; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.core.annotation.Order; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -18,12 +20,15 @@ import java.util.List; @Component @Slf4j -public class EmbeddingTask { +@Order(2) +public class MetaEmbeddingTask implements CommandLineRunner { @Autowired private EmbeddingService embeddingService; + @Autowired private EmbeddingConfig embeddingConfig; + @Autowired private MetricService metricService; @@ -74,4 +79,13 @@ public class EmbeddingTask { log.info("reload.meta.embedding end"); } + + @Override + public void run(String... args) throws Exception { + try { + reloadMetaEmbedding(); + } catch (Exception e) { + log.error("initMetaEmbedding error", e); + } + } } \ No newline at end of file diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/KnowledgeController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/KnowledgeController.java index db2a87b2b..b4e4c2218 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/KnowledgeController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/KnowledgeController.java @@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; -import com.tencent.supersonic.headless.server.schedule.EmbeddingTask; +import com.tencent.supersonic.headless.server.task.MetaEmbeddingTask; import com.tencent.supersonic.headless.server.web.service.DictConfService; import com.tencent.supersonic.headless.server.web.service.DictTaskService; import org.springframework.beans.factory.annotation.Autowired; @@ -39,7 +39,7 @@ public class KnowledgeController { private DictConfService confService; @Autowired - private EmbeddingTask embeddingTask; + private MetaEmbeddingTask embeddingTask; /** * addDictConf-新增item的字典配置 diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DictWordService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DictWordService.java index 0afde4807..dde8e6597 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DictWordService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DictWordService.java @@ -40,10 +40,10 @@ public class DictWordService { List dictWords = getAllDictWords(); List preDictWords = getPreDictWords(); if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords, preDictWords)) { - log.debug("dictWords has not changed, reloadKnowledge end"); + log.debug("Dictionary hasn't been reloaded."); return; } - log.info("dictWords has changed"); + log.info("Dictionary has been reloaded."); setPreDictWords(dictWords); knowledgeBaseService.updateOnlineKnowledge(getAllDictWords()); } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java index 616704e6e..345de8549 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java @@ -5,7 +5,7 @@ import static org.junit.Assert.assertTrue; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.auth.authentication.strategy.FakeUserStrategy; -import com.tencent.supersonic.headless.server.listener.FlightSqlListener; +import com.tencent.supersonic.headless.server.task.FlightServerInitTask; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.flight.CallHeaders; import org.apache.arrow.flight.FlightCallHeaders; @@ -24,7 +24,7 @@ public class FlightSqlTest extends BaseTest { @Autowired - private FlightSqlListener flightSqlListener; + private FlightServerInitTask flightSqlListener; @Autowired private FakeUserStrategy fakeUserStrategy;