(improvement)(headless)Move task related classes to dedicated package.

This commit is contained in:
jerryjzhang
2024-06-30 11:51:10 +08:00
parent 1d0b288922
commit f96c0871f6
7 changed files with 34 additions and 50 deletions

View File

@@ -1,33 +0,0 @@
package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.headless.chat.utils.ComponentFactory;
import com.tencent.supersonic.headless.chat.parser.llm.JavaLLMProxy;
import com.tencent.supersonic.headless.server.schedule.EmbeddingTask;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@Order(2)
public class FullMetaEmbeddingListener implements CommandLineRunner {
@Autowired
private EmbeddingTask embeddingTask;
@Override
public void run(String... args) {
initMetaEmbedding();
}
public void initMetaEmbedding() {
try {
if (ComponentFactory.getLLMProxy() instanceof JavaLLMProxy) {
embeddingTask.reloadMetaEmbedding();
}
} catch (Exception e) {
log.error("initMetaEmbedding error", e);
}
}
}

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.listener; package com.tencent.supersonic.headless.server.task;
import com.tencent.supersonic.headless.server.web.service.impl.DictWordService; import com.tencent.supersonic.headless.server.web.service.impl.DictWordService;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -11,7 +11,7 @@ import org.springframework.stereotype.Component;
@Slf4j @Slf4j
@Component @Component
@Order(2) @Order(2)
public class DictWordLoadStartedListener implements CommandLineRunner { public class DictionaryReloadTask implements CommandLineRunner {
@Autowired @Autowired
private DictWordService dictWordService; private DictWordService dictWordService;

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.listener; package com.tencent.supersonic.headless.server.task;
import com.tencent.supersonic.headless.server.facade.service.FlightService; import com.tencent.supersonic.headless.server.facade.service.FlightService;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
@@ -12,12 +12,14 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner; import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import javax.annotation.PreDestroy;
/** /**
* arrow flight jdbc server listener * Initialize flight jdbc server
*/ */
@Component @Component
@Slf4j @Slf4j
public class FlightSqlListener implements CommandLineRunner { public class FlightServerInitTask implements CommandLineRunner {
@Value("${s2.flightSql.enable:false}") @Value("${s2.flightSql.enable:false}")
private Boolean enable = false; private Boolean enable = false;
@@ -38,7 +40,7 @@ public class FlightSqlListener implements CommandLineRunner {
private BufferAllocator allocator; private BufferAllocator allocator;
private Boolean isRunning = false; private Boolean isRunning = false;
public FlightSqlListener(FlightService flightService) { public FlightServerInitTask(FlightService flightService) {
this.allocator = new RootAllocator(); this.allocator = new RootAllocator();
this.flightService = flightService; this.flightService = flightService;
this.flightService.setLocation(host, port); this.flightService.setLocation(host, port);
@@ -63,7 +65,7 @@ public class FlightSqlListener implements CommandLineRunner {
flightServer.start(); flightServer.start();
isRunning = true; isRunning = true;
} catch (Exception e) { } catch (Exception e) {
log.error("FlightSqlListener start error {}", e); log.error("FlightServerInitTask start error {}", e);
} }
} }
@@ -72,13 +74,14 @@ public class FlightSqlListener implements CommandLineRunner {
return isRunning; return isRunning;
} }
public void stop() { @PreDestroy
public void onShutdown() {
try { try {
log.info("Arrow Flight JDBC server stop on {} {}", host, port); log.info("Arrow Flight JDBC server stop on {} {}", host, port);
flightServer.close(); flightServer.close();
allocator.close(); allocator.close();
} catch (Exception e) { } catch (Exception e) {
log.error("FlightSqlListener start error {}", e); log.error("FlightServerInitTask start error {}", e);
} }
} }

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.schedule; package com.tencent.supersonic.headless.server.task;
import com.tencent.supersonic.common.config.EmbeddingConfig; import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.common.pojo.DataItem; import com.tencent.supersonic.common.pojo.DataItem;
@@ -10,6 +10,8 @@ import dev.langchain4j.store.embedding.EmbeddingStoreFactory;
import dev.langchain4j.store.embedding.TextSegmentConvert; import dev.langchain4j.store.embedding.TextSegmentConvert;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@@ -18,12 +20,15 @@ import java.util.List;
@Component @Component
@Slf4j @Slf4j
public class EmbeddingTask { @Order(2)
public class MetaEmbeddingTask implements CommandLineRunner {
@Autowired @Autowired
private EmbeddingService embeddingService; private EmbeddingService embeddingService;
@Autowired @Autowired
private EmbeddingConfig embeddingConfig; private EmbeddingConfig embeddingConfig;
@Autowired @Autowired
private MetricService metricService; private MetricService metricService;
@@ -74,4 +79,13 @@ public class EmbeddingTask {
log.info("reload.meta.embedding end"); log.info("reload.meta.embedding end");
} }
@Override
public void run(String... args) throws Exception {
try {
reloadMetaEmbedding();
} catch (Exception e) {
log.error("initMetaEmbedding error", e);
}
}
} }

View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp; import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp; import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp; import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import com.tencent.supersonic.headless.server.schedule.EmbeddingTask; import com.tencent.supersonic.headless.server.task.MetaEmbeddingTask;
import com.tencent.supersonic.headless.server.web.service.DictConfService; import com.tencent.supersonic.headless.server.web.service.DictConfService;
import com.tencent.supersonic.headless.server.web.service.DictTaskService; import com.tencent.supersonic.headless.server.web.service.DictTaskService;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -39,7 +39,7 @@ public class KnowledgeController {
private DictConfService confService; private DictConfService confService;
@Autowired @Autowired
private EmbeddingTask embeddingTask; private MetaEmbeddingTask embeddingTask;
/** /**
* addDictConf-新增item的字典配置 * addDictConf-新增item的字典配置

View File

@@ -40,10 +40,10 @@ public class DictWordService {
List<DictWord> dictWords = getAllDictWords(); List<DictWord> dictWords = getAllDictWords();
List<DictWord> preDictWords = getPreDictWords(); List<DictWord> preDictWords = getPreDictWords();
if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords, preDictWords)) { if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords, preDictWords)) {
log.debug("dictWords has not changed, reloadKnowledge end"); log.debug("Dictionary hasn't been reloaded.");
return; return;
} }
log.info("dictWords has changed"); log.info("Dictionary has been reloaded.");
setPreDictWords(dictWords); setPreDictWords(dictWords);
knowledgeBaseService.updateOnlineKnowledge(getAllDictWords()); knowledgeBaseService.updateOnlineKnowledge(getAllDictWords());
} }

View File

@@ -5,7 +5,7 @@ import static org.junit.Assert.assertTrue;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.auth.authentication.strategy.FakeUserStrategy; import com.tencent.supersonic.auth.authentication.strategy.FakeUserStrategy;
import com.tencent.supersonic.headless.server.listener.FlightSqlListener; import com.tencent.supersonic.headless.server.task.FlightServerInitTask;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.arrow.flight.CallHeaders; import org.apache.arrow.flight.CallHeaders;
import org.apache.arrow.flight.FlightCallHeaders; import org.apache.arrow.flight.FlightCallHeaders;
@@ -24,7 +24,7 @@ public class FlightSqlTest extends BaseTest {
@Autowired @Autowired
private FlightSqlListener flightSqlListener; private FlightServerInitTask flightSqlListener;
@Autowired @Autowired
private FakeUserStrategy fakeUserStrategy; private FakeUserStrategy fakeUserStrategy;