topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-12 04:27:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) Upgrade to the latest version of langchain4j and add support for embedding deletion operation and reset. (#1660)

Browse Source
This commit is contained in:
lexluo09
2024-09-12 18:16:16 +08:00
committed by GitHub
parent 693356e46a
commit 4b1dab8e4a
16 changed files with 13307 additions and 16497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
headless/server/pom.xml
View File

@@ -114,6 +114,28 @@
<groupId>org.apache.arrow</groupId>
<artifactId>flight-sql</artifactId>
<version>${flight-sql.version}</version>
<exclusions>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
</exclusion>
<exclusion>
<groupId>io.grpc</groupId>
<artifactId>grpc-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>

16
headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/KnowledgeController.java
View File

@@ -7,6 +7,7 @@ import javax.validation.Valid;
import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.utils.UserHolder;
import com.tencent.supersonic.common.service.EmbeddingService;
import com.tencent.supersonic.common.service.ExemplarService;
import com.tencent.supersonic.headless.api.pojo.request.DictItemFilter;
import com.tencent.supersonic.headless.api.pojo.request.DictItemReq;
@@ -43,6 +44,8 @@ public class KnowledgeController {
@Autowired private ExemplarService exemplarService;
@Autowired private EmbeddingService embeddingService;
/**
* addDictConf-新增item的字典配置 Add configuration information for dictionary entries
*
@@ -115,7 +118,7 @@ public class KnowledgeController {
/** dailyDictTask-手动离线更新所有字典 */
@PutMapping("/task/all")
public Boolean dailyDictTask(HttpServletRequest request, HttpServletResponse response) {
public Boolean dailyDictTask() {
return taskService.dailyDictTask();
}
@@ -140,6 +143,12 @@ public class KnowledgeController {
return true;
}
@GetMapping("/embedding/reset")
public Object resetEmbedding() {
embeddingService.removeAll();
return reloadEmbedding();
}
@GetMapping("/embedding/persistFile")
public Object executePersistFileTask() {
metaEmbeddingTask.executePersistFileTask();
@@ -175,10 +184,7 @@ public class KnowledgeController {
}
@PostMapping("/dict/reload")
public boolean reloadKnowledge(
@RequestBody @Valid DictValueReq dictValueReq,
HttpServletRequest request,
HttpServletResponse response) {
public boolean reloadKnowledge() {
dictionaryReloadTask.reloadKnowledge();
return true;
}

117
headless/server/src/main/java/com/tencent/supersonic/headless/server/task/FlightServerInitTask.java
View File

@@ -1,117 +0,0 @@
package com.tencent.supersonic.headless.server.task;
import javax.annotation.PreDestroy;
import com.tencent.supersonic.headless.server.service.FlightService;
import lombok.extern.slf4j.Slf4j;
import org.apache.arrow.flight.FlightServer;
import org.apache.arrow.flight.Location;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/** Initialize flight jdbc server */
@Component
@Slf4j
public class FlightServerInitTask implements CommandLineRunner {
@Value("${s2.flightSql.enable:false}")
private Boolean enable = false;
@Value("${s2.flightSql.host:localhost}")
private String host = "localhost";
@Value("${s2.flightSql.port:9081}")
private Integer port = 9081;
@Value("${s2.flightSql.executor:4}")
private Integer executor = 4;
@Value("${s2.flightSql.queue:128}")
private Integer queue = 128;
@Value("${s2.flightSql.expireMinute:10}")
private Integer expireMinute = 10;
private final FlightService flightService;
private ExecutorService executorService;
private FlightServer flightServer;
private BufferAllocator allocator;
private Boolean isRunning = false;
public FlightServerInitTask(FlightService flightService) {
this.allocator = new RootAllocator();
this.flightService = flightService;
this.flightService.setLocation(host, port);
executorService = Executors.newFixedThreadPool(executor);
this.flightService.setExecutorService(executorService, queue, expireMinute);
Location listenLocation = Location.forGrpcInsecure(host, port);
flightServer = FlightServer.builder(allocator, listenLocation, this.flightService).build();
}
public String getHost() {
return host;
}
public Integer getPort() {
return port;
}
public void startServer() {
try {
log.info("Arrow Flight JDBC server started on {} {}", host, port);
flightServer.start();
isRunning = true;
} catch (Exception e) {
log.error("FlightServerInitTask start error {}", e);
}
}
public Boolean isRunning() {
return isRunning;
}
@PreDestroy
public void onShutdown() {
try {
log.info("Arrow Flight JDBC server stop on {} {}", host, port);
flightServer.close();
allocator.close();
} catch (Exception e) {
log.error("FlightServerInitTask start error {}", e);
}
}
@Override
public void run(String... args) throws Exception {
if (enable) {
new Thread() {
@Override
public void run() {
try {
startServer();
Runtime.getRuntime()
.addShutdownHook(
new Thread(
() -> {
try {
flightServer.close();
allocator.close();
} catch (Exception e) {
log.error("flightServer close error {}", e);
}
}));
// flightServer.awaitTermination();
} catch (Exception e) {
log.error("run error {}", e);
}
}
}.start();
}
}
}