topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)Move task related classes to dedicated package.

Browse Source
This commit is contained in:
jerryjzhang
2024-06-30 11:51:10 +08:00
parent 1d0b288922
commit f96c0871f6
7 changed files with 34 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FullMetaEmbeddingListener.java
View File

@@ -1,33 +0,0 @@
package com.tencent.supersonic.headless.server.listener;
import com.tencent.supersonic.headless.chat.utils.ComponentFactory;
import com.tencent.supersonic.headless.chat.parser.llm.JavaLLMProxy;
import com.tencent.supersonic.headless.server.schedule.EmbeddingTask;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@Order(2)
public class FullMetaEmbeddingListener implements CommandLineRunner {
@Autowired
private EmbeddingTask embeddingTask;
@Override
public void run(String... args) {
initMetaEmbedding();
}
public void initMetaEmbedding() {
try {
if (ComponentFactory.getLLMProxy() instanceof JavaLLMProxy) {
embeddingTask.reloadMetaEmbedding();
}
} catch (Exception e) {
log.error("initMetaEmbedding error", e);
}
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/DictWordLoadStartedListener.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/task/DictionaryReloadTask.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.listener;
package com.tencent.supersonic.headless.server.task;
import com.tencent.supersonic.headless.server.web.service.impl.DictWordService;
import lombok.extern.slf4j.Slf4j;
@@ -11,7 +11,7 @@ import org.springframework.stereotype.Component;
@Slf4j
@Component
@Order(2)
public class DictWordLoadStartedListener implements CommandLineRunner {
public class DictionaryReloadTask implements CommandLineRunner {
@Autowired
private DictWordService dictWordService;

17
headless/server/src/main/java/com/tencent/supersonic/headless/server/listener/FlightSqlListener.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.listener;
package com.tencent.supersonic.headless.server.task;
import com.tencent.supersonic.headless.server.facade.service.FlightService;
import java.util.concurrent.ExecutorService;
@@ -12,12 +12,14 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import javax.annotation.PreDestroy;
/**
* arrow flight jdbc server listener
* Initialize flight jdbc server
*/
@Component
@Slf4j
public class FlightSqlListener implements CommandLineRunner {
public class FlightServerInitTask implements CommandLineRunner {
@Value("${s2.flightSql.enable:false}")
private Boolean enable = false;
@@ -38,7 +40,7 @@ public class FlightSqlListener implements CommandLineRunner {
private BufferAllocator allocator;
private Boolean isRunning = false;
public FlightSqlListener(FlightService flightService) {
public FlightServerInitTask(FlightService flightService) {
this.allocator = new RootAllocator();
this.flightService = flightService;
this.flightService.setLocation(host, port);
@@ -63,7 +65,7 @@ public class FlightSqlListener implements CommandLineRunner {
flightServer.start();
isRunning = true;
} catch (Exception e) {
log.error("FlightSqlListener start error {}", e);
log.error("FlightServerInitTask start error {}", e);
}
}
@@ -72,13 +74,14 @@ public class FlightSqlListener implements CommandLineRunner {
return isRunning;
}
public void stop() {
@PreDestroy
public void onShutdown() {
try {
log.info("Arrow Flight JDBC server stop on {} {}", host, port);
flightServer.close();
allocator.close();
} catch (Exception e) {
log.error("FlightSqlListener start error {}", e);
log.error("FlightServerInitTask start error {}", e);
}
}

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/schedule/EmbeddingTask.java → headless/server/src/main/java/com/tencent/supersonic/headless/server/task/MetaEmbeddingTask.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.server.schedule;
package com.tencent.supersonic.headless.server.task;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.common.pojo.DataItem;
@@ -10,6 +10,8 @@ import dev.langchain4j.store.embedding.EmbeddingStoreFactory;
import dev.langchain4j.store.embedding.TextSegmentConvert;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@@ -18,12 +20,15 @@ import java.util.List;
@Component
@Slf4j
public class EmbeddingTask {
@Order(2)
public class MetaEmbeddingTask implements CommandLineRunner {
@Autowired
private EmbeddingService embeddingService;
@Autowired
private EmbeddingConfig embeddingConfig;
@Autowired
private MetricService metricService;
@@ -74,4 +79,13 @@ public class EmbeddingTask {
log.info("reload.meta.embedding end");
}
@Override
public void run(String... args) throws Exception {
try {
reloadMetaEmbedding();
} catch (Exception e) {
log.error("initMetaEmbedding error", e);
}
}
}

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/rest/KnowledgeController.java
View File

@@ -11,7 +11,7 @@ import com.tencent.supersonic.headless.api.pojo.request.DictValueReq;
import com.tencent.supersonic.headless.api.pojo.response.DictItemResp;
import com.tencent.supersonic.headless.api.pojo.response.DictTaskResp;
import com.tencent.supersonic.headless.api.pojo.response.DictValueResp;
import com.tencent.supersonic.headless.server.schedule.EmbeddingTask;
import com.tencent.supersonic.headless.server.task.MetaEmbeddingTask;
import com.tencent.supersonic.headless.server.web.service.DictConfService;
import com.tencent.supersonic.headless.server.web.service.DictTaskService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -39,7 +39,7 @@ public class KnowledgeController {
private DictConfService confService;
@Autowired
private EmbeddingTask embeddingTask;
private MetaEmbeddingTask embeddingTask;
/**
* addDictConf-新增item的字典配置

4
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/DictWordService.java
View File

@@ -40,10 +40,10 @@ public class DictWordService {
List<DictWord> dictWords = getAllDictWords();
List<DictWord> preDictWords = getPreDictWords();
if (org.apache.commons.collections.CollectionUtils.isEqualCollection(dictWords, preDictWords)) {
log.debug("dictWords has not changed, reloadKnowledge end");
log.debug("Dictionary hasn't been reloaded.");
return;
}
log.info("dictWords has changed");
log.info("Dictionary has been reloaded.");
setPreDictWords(dictWords);
knowledgeBaseService.updateOnlineKnowledge(getAllDictWords());
}

4
launchers/standalone/src/test/java/com/tencent/supersonic/headless/FlightSqlTest.java
View File

@@ -5,7 +5,7 @@ import static org.junit.Assert.assertTrue;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.auth.authentication.strategy.FakeUserStrategy;
import com.tencent.supersonic.headless.server.listener.FlightSqlListener;
import com.tencent.supersonic.headless.server.task.FlightServerInitTask;
import lombok.extern.slf4j.Slf4j;
import org.apache.arrow.flight.CallHeaders;
import org.apache.arrow.flight.FlightCallHeaders;
@@ -24,7 +24,7 @@ public class FlightSqlTest extends BaseTest {
@Autowired
private FlightSqlListener flightSqlListener;
private FlightServerInitTask flightSqlListener;
@Autowired
private FakeUserStrategy fakeUserStrategy;