topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(Headless) Add exemplars for building data-model by LLM #1319 (#1849)

Browse Source 
Co-authored-by: lxwcodemonkey
This commit is contained in:
LXW
2024-10-27 23:17:33 +08:00
committed by GitHub
parent b69ee81d58
commit c2785139f2
4 changed files with 270 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
headless/server/src/main/java/com/tencent/supersonic/headless/server/builder/ModelIntelligentBuilder.java
View File

@@ -1,5 +1,6 @@
package com.tencent.supersonic.headless.server.builder;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.tencent.supersonic.common.pojo.ChatApp;
import com.tencent.supersonic.common.pojo.ChatModelConfig;
import com.tencent.supersonic.common.pojo.enums.AppModule;
@@ -12,8 +13,11 @@ import dev.langchain4j.model.input.Prompt;
import dev.langchain4j.model.input.PromptTemplate;
import dev.langchain4j.service.AiServices;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Component;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -25,6 +29,8 @@ public class ModelIntelligentBuilder extends IntelligentBuilder {
public static final String APP_KEY = "BUILD_DATA_MODEL";
private static final String SYS_EXEMPLAR_FILE = "s2-buildModel-exemplar.json";
public static final String INSTRUCTION = ""
+ "Role: As an experienced data analyst with extensive modeling experience, "
+ " you are expected to have a deep understanding of data analysis and data modeling concepts."
@@ -44,12 +50,16 @@ public class ModelIntelligentBuilder extends IntelligentBuilder {
+ "\nDBSchema: {{DBSchema}}" + "\nOtherRelatedDBSchema: {{otherRelatedDBSchema}}"
+ "\nExemplar: {{exemplar}}";
private final ObjectMapper objectMapper = JsonUtil.INSTANCE.getObjectMapper();
@Value("${s2.model.building.exemplars.enabled:true}")
private Boolean enableExemplarLoading;
public ModelIntelligentBuilder() {
ChatAppManager.register(APP_KEY, ChatApp.builder().prompt(INSTRUCTION).name("构造数据语义模型")
.appModule(AppModule.HEADLESS).description("通过大模型来构造数据语义模型").enable(true).build());
}
interface ModelSchemaExtractor {
ModelSchema generateModelSchema(String text);
}
@@ -67,7 +77,8 @@ public class ModelIntelligentBuilder extends IntelligentBuilder {
Prompt prompt = generatePrompt(dbSchema, otherDbSchema, chatApp.get());
ModelSchema modelSchema =
extractor.generateModelSchema(prompt.toUserMessage().singleText());
log.info("dbSchema: {} modelSchema: {}", JsonUtil.toString(dbSchema),
log.info("dbSchema: {}\n otherRelatedDBSchema:{}\n modelSchema: {}",
JsonUtil.toString(dbSchema), JsonUtil.toString(otherDbSchema),
JsonUtil.toString(modelSchema));
return modelSchema;
}
@@ -82,7 +93,20 @@ public class ModelIntelligentBuilder extends IntelligentBuilder {
}
private String loadExemplars() {
// to add
if (!enableExemplarLoading) {
log.info("Not enable load model-building exemplars");
return "";
}
try {
ClassPathResource resource = new ClassPathResource(SYS_EXEMPLAR_FILE);
if (resource.exists()) {
InputStream inputStream = resource.getInputStream();
return objectMapper
.writeValueAsString(objectMapper.readValue(inputStream, Object.class));
}
} catch (Exception e) {
log.error("Failed to load model-building system exemplars", e);
}
return "";
}

10
headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/ModelServiceImpl.java
View File

@@ -103,7 +103,7 @@ public class ModelServiceImpl implements ModelService {
private ModelRelaService modelRelaService;
ExecutorService executor =
new ThreadPoolExecutor(0, 5, 100L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
new ThreadPoolExecutor(0, 5, 5L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
public ModelServiceImpl(ModelRepository modelRepository, DatabaseService databaseService,
@Lazy DimensionService dimensionService, @Lazy MetricService metricService,
@@ -219,12 +219,11 @@ public class ModelServiceImpl implements ModelService {
@Override
public Map<String, ModelSchema> buildModelSchema(ModelBuildReq modelBuildReq)
throws SQLException {
Map<String, List<DBColumn>> dbColumnMap = databaseService.getDbColumns(modelBuildReq);
if (modelBuildReq.isBuildByLLM() && modelBuildReq.getChatModelConfig() == null) {
ChatModel chatModel = chatModelService.getChatModel(modelBuildReq.getChatModelId());
modelBuildReq.setChatModelConfig(chatModel.getConfig());
}
List<DbSchema> dbSchemas = convert(dbColumnMap, modelBuildReq);
List<DbSchema> dbSchemas = getDbSchemes(modelBuildReq);
Map<String, ModelSchema> modelSchemaMap = new ConcurrentHashMap<>();
CompletableFuture.allOf(dbSchemas.stream()
.map(dbSchema -> CompletableFuture.runAsync(
@@ -246,6 +245,11 @@ public class ModelServiceImpl implements ModelService {
}
}
private List<DbSchema> getDbSchemes(ModelBuildReq modelBuildReq) throws SQLException {
Map<String, List<DBColumn>> dbColumnMap = databaseService.getDbColumns(modelBuildReq);
return convert(dbColumnMap, modelBuildReq);
}
private List<DbSchema> getOtherDbSchema(DbSchema curSchema, List<DbSchema> dbSchemas) {
return dbSchemas.stream()
.filter(dbSchema -> !dbSchema.getTable().equals(curSchema.getTable()))