From 72465cd88c5ccc3a423df8c67381b094b551c2a9 Mon Sep 17 00:00:00 2001 From: jerryjzhang Date: Fri, 5 Jul 2024 15:13:31 +0800 Subject: [PATCH] # This is a combination of 2 commits. (feature)(headless)Support custom prompt template. #1348. --- .../supersonic/chat/server/agent/Agent.java | 2 + .../chat/server/parser/NL2SQLParser.java | 8 +- .../persistence/dataobject/AgentDO.java | 3 +- .../server/service/impl/AgentServiceImpl.java | 3 + .../chat/server/util/QueryReqConverter.java | 1 + .../common/config/PromptConfig.java | 14 +++ .../headless/api/pojo/request/QueryReq.java | 4 +- .../headless/chat/QueryContext.java | 4 +- .../chat/parser/llm/LLMRequestService.java | 4 +- .../parser/llm/OnePassSCSqlGenStrategy.java | 26 ++-- .../chat/parser/llm/PromptHelper.java | 4 +- .../headless/chat/query/llm/s2sql/LLMReq.java | 6 +- .../resources/config.update/sql-update.sql | 7 +- .../src/main/resources/db/schema-h2.sql | 117 +++++++++--------- .../src/test/resources/db/schema-h2.sql | 115 ++++++++--------- 15 files changed, 180 insertions(+), 138 deletions(-) create mode 100644 common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java index 4101de7bc..b99380ca6 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/agent/Agent.java @@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.common.config.LLMConfig; +import com.tencent.supersonic.common.config.PromptConfig; import com.tencent.supersonic.common.config.VisualConfig; import com.tencent.supersonic.common.pojo.RecordInfo; import lombok.Data; @@ -33,6 +34,7 @@ public class Agent extends RecordInfo { private List examples; private String agentConfig; private LLMConfig llmConfig; + private PromptConfig promptConfig; private MultiTurnConfig multiTurnConfig; private VisualConfig visualConfig; diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java index 6ad549d98..0d850b254 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/parser/NL2SQLParser.java @@ -66,10 +66,10 @@ public class NL2SQLParser implements ChatParser { if (!chatParseContext.enableNL2SQL() || checkSkip(parseResp)) { return; } - processMultiTurn(chatParseContext); + processMultiTurn(chatParseContext); QueryReq queryReq = QueryReqConverter.buildText2SqlQueryReq(chatParseContext); - addExemplars(chatParseContext.getAgent().getId(), queryReq); + addDynamicExemplars(chatParseContext.getAgent().getId(), queryReq); ChatQueryService chatQueryService = ContextUtils.getBean(ChatQueryService.class); ParseResp text2SqlParseResp = chatQueryService.performParsing(queryReq); @@ -224,13 +224,13 @@ public class NL2SQLParser implements ChatParser { return contextualList; } - private void addExemplars(Integer agentId, QueryReq queryReq) { + private void addDynamicExemplars(Integer agentId, QueryReq queryReq) { ExemplarServiceImpl exemplarManager = ContextUtils.getBean(ExemplarServiceImpl.class); EmbeddingConfig embeddingConfig = ContextUtils.getBean(EmbeddingConfig.class); String memoryCollectionName = embeddingConfig.getMemoryCollectionName(agentId); List exemplars = exemplarManager.recallExemplars(memoryCollectionName, queryReq.getQueryText(), 5); - queryReq.getExemplars().addAll(exemplars); + queryReq.getDynamicExemplars().addAll(exemplars); } @Builder diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java index 00febd256..62314ff49 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/persistence/dataobject/AgentDO.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.chat.server.persistence.dataobject; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import com.tencent.supersonic.common.config.VisualConfig; import lombok.Data; import java.util.Date; @@ -63,4 +62,6 @@ public class AgentDO { private String visualConfig; + private String promptConfig; + } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java index 682dbecab..44cd3b394 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/service/impl/AgentServiceImpl.java @@ -13,6 +13,7 @@ import com.tencent.supersonic.chat.server.service.ChatService; import com.tencent.supersonic.chat.server.service.MemoryService; import com.tencent.supersonic.chat.server.util.LLMConnHelper; import com.tencent.supersonic.common.config.LLMConfig; +import com.tencent.supersonic.common.config.PromptConfig; import com.tencent.supersonic.common.config.VisualConfig; import com.tencent.supersonic.common.util.JsonUtil; import lombok.extern.slf4j.Slf4j; @@ -121,6 +122,7 @@ public class AgentServiceImpl extends ServiceImpl agent.setAgentConfig(agentDO.getConfig()); agent.setExamples(JsonUtil.toList(agentDO.getExamples(), String.class)); agent.setLlmConfig(JsonUtil.toObject(agentDO.getLlmConfig(), LLMConfig.class)); + agent.setPromptConfig(JsonUtil.toObject(agentDO.getPromptConfig(), PromptConfig.class)); agent.setMultiTurnConfig(JsonUtil.toObject(agentDO.getMultiTurnConfig(), MultiTurnConfig.class)); agent.setVisualConfig(JsonUtil.toObject(agentDO.getVisualConfig(), VisualConfig.class)); return agent; @@ -134,6 +136,7 @@ public class AgentServiceImpl extends ServiceImpl agentDO.setLlmConfig(JsonUtil.toString(agent.getLlmConfig())); agentDO.setMultiTurnConfig(JsonUtil.toString(agent.getMultiTurnConfig())); agentDO.setVisualConfig(JsonUtil.toString(agent.getVisualConfig())); + agentDO.setPromptConfig(JsonUtil.toString(agent.getPromptConfig())); if (agentDO.getStatus() == null) { agentDO.setStatus(1); } diff --git a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java index 3b2afad81..6cd91f4d9 100644 --- a/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java +++ b/chat/server/src/main/java/com/tencent/supersonic/chat/server/util/QueryReqConverter.java @@ -31,6 +31,7 @@ public class QueryReqConverter { queryReq.setMapInfo(queryReq.getMapInfo()); } queryReq.setLlmConfig(agent.getLlmConfig()); + queryReq.setPromptConfig(agent.getPromptConfig()); return queryReq; } diff --git a/common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java b/common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java new file mode 100644 index 000000000..bf3d8d6e7 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/config/PromptConfig.java @@ -0,0 +1,14 @@ +package com.tencent.supersonic.common.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class PromptConfig { + + private String promptTemplate; + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java index 020f04894..f7138af5c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryReq.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.config.PromptConfig; import com.tencent.supersonic.common.pojo.enums.Text2SQLType; import com.tencent.supersonic.common.config.LLMConfig; import com.tencent.supersonic.headless.api.pojo.QueryDataType; @@ -27,5 +28,6 @@ public class QueryReq { private SchemaMapInfo mapInfo = new SchemaMapInfo(); private QueryDataType queryDataType = QueryDataType.ALL; private LLMConfig llmConfig; - private List exemplars = Lists.newArrayList(); + private PromptConfig promptConfig; + private List dynamicExemplars = Lists.newArrayList(); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/QueryContext.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/QueryContext.java index da2e60c6d..b86e573db 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/QueryContext.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/QueryContext.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.chat; import com.fasterxml.jackson.annotation.JsonIgnore; import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.config.PromptConfig; import com.tencent.supersonic.common.pojo.enums.Text2SQLType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.config.LLMConfig; @@ -50,7 +51,8 @@ public class QueryContext { private WorkflowState workflowState; private QueryDataType queryDataType = QueryDataType.ALL; private LLMConfig llmConfig; - private List exemplars; + private PromptConfig promptConfig; + private List dynamicExemplars; public List getCandidateQueries() { ParserConfig parserConfig = ContextUtils.getBean(ParserConfig.class); diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java index 565e5ca21..a855f4082 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/LLMRequestService.java @@ -102,8 +102,8 @@ public class LLMRequestService { llmReq.setCurrentDate(DateUtils.getBeforeDate(0)); llmReq.setSqlGenType(LLMReq.SqlGenType.valueOf(parserConfig.getParameterValue(PARSER_STRATEGY_TYPE))); llmReq.setLlmConfig(queryCtx.getLlmConfig()); - - llmReq.setExemplars(queryCtx.getExemplars()); + llmReq.setPromptConfig(queryCtx.getPromptConfig()); + llmReq.setDynamicExemplars(queryCtx.getDynamicExemplars()); return llmReq; } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java index 94d889389..ad8ae6858 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/OnePassSCSqlGenStrategy.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.chat.parser.llm; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.config.PromptConfig; import com.tencent.supersonic.common.pojo.SqlExemplar; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMReq; import com.tencent.supersonic.headless.chat.query.llm.s2sql.LLMResp; @@ -10,10 +11,10 @@ import dev.langchain4j.model.input.Prompt; import dev.langchain4j.model.input.PromptTemplate; import dev.langchain4j.model.output.Response; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Service; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,10 +35,8 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy { + "2.ALWAYS use `datediff()` as the date function." + "3.DO NOT specify date filter in the where clause if not explicitly mentioned in the query." + "4.ONLY respond with the converted SQL statement.\n" - + "#Exemplars:\n%s" - + "#UserQuery: %s " - + "#Schema: %s " - + "#SQL: "; + + "#Exemplars:\n{{exemplar}}" + + "#Question:{{question}} #Schema:{{schema}} #SQL:"; @Override public LLMResp generate(LLMReq llmReq) { @@ -80,16 +79,25 @@ public class OnePassSCSqlGenStrategy extends SqlGenStrategy { private Prompt generatePrompt(LLMReq llmReq, List fewshotExampleList) { StringBuilder exemplarsStr = new StringBuilder(); for (SqlExemplar exemplar : fewshotExampleList) { - String exemplarStr = String.format("#UserQuery: %s #Schema: %s #SQL: %s\n", + String exemplarStr = String.format("#Question:%s #Schema:%s #SQL:%s\n", exemplar.getQuestion(), exemplar.getDbSchema(), exemplar.getSql()); exemplarsStr.append(exemplarStr); } - String dataSemanticsStr = promptHelper.buildSchemaStr(llmReq); String questionAugmented = promptHelper.buildAugmentedQuestion(llmReq); - String promptStr = String.format(INSTRUCTION, exemplarsStr, questionAugmented, dataSemanticsStr); - return PromptTemplate.from(promptStr).apply(Collections.EMPTY_MAP); + Map variable = new HashMap<>(); + variable.put("exemplar", exemplarsStr); + variable.put("question", questionAugmented); + variable.put("schema", dataSemanticsStr); + + // use custom prompt template if provided. + PromptConfig promptConfig = llmReq.getPromptConfig(); + String prompTemplate = INSTRUCTION; + if (promptConfig != null && StringUtils.isNotBlank(promptConfig.getPromptTemplate())) { + prompTemplate = promptConfig.getPromptTemplate(); + } + return PromptTemplate.from(prompTemplate).apply(variable); } @Override diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java index ca3f0fa77..76171069c 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/parser/llm/PromptHelper.java @@ -35,11 +35,11 @@ public class PromptHelper { int selfConsistencyNumber = Integer.valueOf(parserConfig.getParameterValue(PARSER_SELF_CONSISTENCY_NUMBER)); List exemplars = Lists.newArrayList(); - llmReq.getExemplars().stream().forEach(e -> { + llmReq.getDynamicExemplars().stream().forEach(e -> { exemplars.add(e); }); - int recallSize = exemplarRecallNumber - llmReq.getExemplars().size(); + int recallSize = exemplarRecallNumber - llmReq.getDynamicExemplars().size(); if (recallSize > 0) { exemplars.addAll(exemplarService.recallExemplars(llmReq.getQueryText(), recallSize)); } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java index de7c42f12..0f522a05e 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMReq.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.chat.query.llm.s2sql; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.collect.Lists; import com.tencent.supersonic.common.config.LLMConfig; +import com.tencent.supersonic.common.config.PromptConfig; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.common.pojo.SqlExemplar; import lombok.Data; @@ -28,7 +29,10 @@ public class LLMReq { private LLMConfig llmConfig; - private List exemplars; + private PromptConfig promptConfig; + + private List dynamicExemplars; + @Data public static class ElementValue { diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index de4e3e70e..37f090a0c 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -310,7 +310,7 @@ CREATE TABLE IF NOT EXISTS `s2_term` ( ); --20240520 -alter table s2_agent add column `llm_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL; +alter table s2_agent add column `llm_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '大模型配置'; alter table s2_agent add column `multi_turn_config` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL; alter table s2_model add column `ext` varchar(1000) DEFAULT NULL; @@ -347,4 +347,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat_memory` ( `created_by` varchar(100) NOT NULL , `updated_by` varchar(100) NOT NULL , PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +--20240705 +alter table s2_agent add column `prompt_config` varchar(6000) COLLATE utf8_unicode_ci DEFAULT NULL COMMENT '提示词配置'; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index 53e640a37..e115da152 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat_query` CREATE TABLE IF NOT EXISTS `s2_chat_parse` ( - `question_id` BIGINT NOT NULL, + `question_id` BIGINT NOT NULL, `chat_id` BIGINT NOT NULL , `parse_id` INT NOT NULL , `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -67,9 +67,9 @@ CREATE TABLE IF NOT EXISTS `s2_chat_statistics` ); CREATE TABLE IF NOT EXISTS `s2_chat_config` ( - `id` INT NOT NULL AUTO_INCREMENT, - `model_id` INT DEFAULT NULL , - `chat_detail_config` varchar(655) , + `id` INT NOT NULL AUTO_INCREMENT, + `model_id` INT DEFAULT NULL , + `chat_detail_config` varchar(655) , `chat_agg_config` varchar(655) , `recommended_questions` varchar(1500) , `created_at` TIMESTAMP NOT NULL , @@ -168,19 +168,19 @@ COMMENT ON TABLE s2_model IS 'model information'; CREATE TABLE IF NOT EXISTS `s2_database` ( - `id` INT NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL , - `description` varchar(500) DEFAULT NULL , - `version` varchar(64) DEFAULT NULL , - `type` varchar(20) NOT NULL , -- type: mysql,clickhouse,tdw - `config` varchar(655) NOT NULL , - `created_at` TIMESTAMP NOT NULL , - `created_by` varchar(100) NOT NULL , - `updated_at` TIMESTAMP NOT NULL , - `updated_by` varchar(100) NOT NULL, - `admin` varchar(500) NOT NULL, - `viewer` varchar(500) DEFAULT NULL, - PRIMARY KEY (`id`) + `id` INT NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL , + `description` varchar(500) DEFAULT NULL , + `version` varchar(64) DEFAULT NULL , + `type` varchar(20) NOT NULL , -- type: mysql,clickhouse,tdw + `config` varchar(655) NOT NULL , + `created_at` TIMESTAMP NOT NULL , + `created_by` varchar(100) NOT NULL , + `updated_at` TIMESTAMP NOT NULL , + `updated_by` varchar(100) NOT NULL, + `admin` varchar(500) NOT NULL, + `viewer` varchar(500) DEFAULT NULL, + PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_database IS 'database instance table'; @@ -219,9 +219,9 @@ COMMENT ON TABLE s2_metric IS 'metric information table'; CREATE TABLE IF NOT EXISTS `s2_dimension` ( - `id` INT NOT NULL AUTO_INCREMENT , - `model_id` INT NOT NULL , - `name` varchar(255) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT , + `model_id` INT NOT NULL , + `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) NOT NULL , `status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal @@ -270,46 +270,46 @@ COMMENT ON TABLE s2_canvas IS 'canvas table'; CREATE TABLE IF NOT EXISTS `s2_query_stat_info` ( - `id` INT NOT NULL AUTO_INCREMENT, - `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier - `model_id` INT DEFAULT NULL, - `data_set_id` INT DEFAULT NULL, - `user` varchar(200) DEFAULT NULL, - `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , - `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene - `query_type_back` INT DEFAULT '0' , -- query type, 0-normal query, 1-pre-refresh type - `query_sql_cmd`LONGVARCHAR , -- sql type request parameter - `sql_cmd_md5` varchar(200) DEFAULT NULL, -- sql type request parameter md5 - `query_struct_cmd`LONGVARCHAR , -- struct type request parameter - `struct_cmd_md5` varchar(200) DEFAULT NULL, -- struct type request parameter md5值 - `sql`LONGVARCHAR , - `sql_md5` varchar(200) DEFAULT NULL, -- sql md5 - `query_engine` varchar(20) DEFAULT NULL, - `elapsed_ms` bigINT DEFAULT NULL, - `query_state` varchar(20) DEFAULT NULL, - `native_query` INT DEFAULT NULL, -- 1-detail query, 0-aggregation query - `start_date` varchar(50) DEFAULT NULL, - `end_date` varchar(50) DEFAULT NULL, - `dimensions`LONGVARCHAR , -- dimensions involved in sql - `metrics`LONGVARCHAR , -- metric involved in sql - `select_cols`LONGVARCHAR , - `agg_cols`LONGVARCHAR , - `filter_cols`LONGVARCHAR , - `group_by_cols`LONGVARCHAR , - `order_by_cols`LONGVARCHAR , - `use_result_cache` TINYINT DEFAULT '-1' , -- whether to hit the result cache - `use_sql_cache` TINYINT DEFAULT '-1' , -- whether to hit the sql cache - `sql_cache_key`LONGVARCHAR , -- sql cache key - `result_cache_key`LONGVARCHAR , -- result cache key - `query_opt_mode` varchar(50) DEFAULT NULL , - PRIMARY KEY (`id`) + `id` INT NOT NULL AUTO_INCREMENT, + `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier + `model_id` INT DEFAULT NULL, + `data_set_id` INT DEFAULT NULL, + `user` varchar(200) DEFAULT NULL, + `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , + `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene + `query_type_back` INT DEFAULT '0' , -- query type, 0-normal query, 1-pre-refresh type + `query_sql_cmd`LONGVARCHAR , -- sql type request parameter + `sql_cmd_md5` varchar(200) DEFAULT NULL, -- sql type request parameter md5 + `query_struct_cmd`LONGVARCHAR , -- struct type request parameter + `struct_cmd_md5` varchar(200) DEFAULT NULL, -- struct type request parameter md5值 + `sql`LONGVARCHAR , + `sql_md5` varchar(200) DEFAULT NULL, -- sql md5 + `query_engine` varchar(20) DEFAULT NULL, + `elapsed_ms` bigINT DEFAULT NULL, + `query_state` varchar(20) DEFAULT NULL, + `native_query` INT DEFAULT NULL, -- 1-detail query, 0-aggregation query + `start_date` varchar(50) DEFAULT NULL, + `end_date` varchar(50) DEFAULT NULL, + `dimensions`LONGVARCHAR , -- dimensions involved in sql + `metrics`LONGVARCHAR , -- metric involved in sql + `select_cols`LONGVARCHAR , + `agg_cols`LONGVARCHAR , + `filter_cols`LONGVARCHAR , + `group_by_cols`LONGVARCHAR , + `order_by_cols`LONGVARCHAR , + `use_result_cache` TINYINT DEFAULT '-1' , -- whether to hit the result cache + `use_sql_cache` TINYINT DEFAULT '-1' , -- whether to hit the sql cache + `sql_cache_key`LONGVARCHAR , -- sql cache key + `result_cache_key`LONGVARCHAR , -- result cache key + `query_opt_mode` varchar(50) DEFAULT NULL , + PRIMARY KEY (`id`) ) ; COMMENT ON TABLE s2_query_stat_info IS 'query statistics table'; CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info` ( - `id` INT NOT NULL AUTO_INCREMENT, - `trace_id` varchar(200) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT, + `trace_id` varchar(200) NOT NULL , `model_id` INT NOT NULL , `dimensions`LONGVARCHAR , `metrics`LONGVARCHAR , @@ -329,9 +329,9 @@ COMMENT ON TABLE s2_semantic_pasre_info IS 'semantic layer sql parsing informati CREATE TABLE IF NOT EXISTS `s2_available_date_info` ( - `id` INT NOT NULL AUTO_INCREMENT , - `item_id` INT NOT NULL , - `type` varchar(255) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT , + `item_id` INT NOT NULL , + `type` varchar(255) NOT NULL , `date_format` varchar(64) NOT NULL , `start_date` varchar(64) , `end_date` varchar(64) , @@ -374,6 +374,7 @@ CREATE TABLE IF NOT EXISTS s2_agent examples varchar(500) null, config varchar(2000) null, llm_config varchar(2000) null, + prompt_config varchar(5000) null, multi_turn_config varchar(2000) null, visual_config varchar(2000) null, created_by varchar(100) null, diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 558032a5f..8ba9c4ba5 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -67,9 +67,9 @@ CREATE TABLE IF NOT EXISTS `s2_chat_statistics` ); CREATE TABLE IF NOT EXISTS `s2_chat_config` ( - `id` INT NOT NULL AUTO_INCREMENT, - `model_id` INT DEFAULT NULL , - `chat_detail_config` varchar(655) , + `id` INT NOT NULL AUTO_INCREMENT, + `model_id` INT DEFAULT NULL , + `chat_detail_config` varchar(655) , `chat_agg_config` varchar(655) , `recommended_questions` varchar(1500) , `created_at` TIMESTAMP NOT NULL , @@ -168,19 +168,19 @@ COMMENT ON TABLE s2_model IS 'model information'; CREATE TABLE IF NOT EXISTS `s2_database` ( - `id` INT NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL , - `description` varchar(500) DEFAULT NULL , - `version` varchar(64) DEFAULT NULL , - `type` varchar(20) NOT NULL , -- type: mysql,clickhouse,tdw - `config` varchar(655) NOT NULL , - `created_at` TIMESTAMP NOT NULL , - `created_by` varchar(100) NOT NULL , - `updated_at` TIMESTAMP NOT NULL , - `updated_by` varchar(100) NOT NULL, - `admin` varchar(500) NOT NULL, - `viewer` varchar(500) DEFAULT NULL, - PRIMARY KEY (`id`) + `id` INT NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL , + `description` varchar(500) DEFAULT NULL , + `version` varchar(64) DEFAULT NULL , + `type` varchar(20) NOT NULL , -- type: mysql,clickhouse,tdw + `config` varchar(655) NOT NULL , + `created_at` TIMESTAMP NOT NULL , + `created_by` varchar(100) NOT NULL , + `updated_at` TIMESTAMP NOT NULL , + `updated_by` varchar(100) NOT NULL, + `admin` varchar(500) NOT NULL, + `viewer` varchar(500) DEFAULT NULL, + PRIMARY KEY (`id`) ); COMMENT ON TABLE s2_database IS 'database instance table'; @@ -219,9 +219,9 @@ COMMENT ON TABLE s2_metric IS 'metric information table'; CREATE TABLE IF NOT EXISTS `s2_dimension` ( - `id` INT NOT NULL AUTO_INCREMENT , - `model_id` INT NOT NULL , - `name` varchar(255) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT , + `model_id` INT NOT NULL , + `name` varchar(255) NOT NULL , `biz_name` varchar(255) NOT NULL , `description` varchar(500) NOT NULL , `status` INT NOT NULL , -- status, 0 is off the shelf, 1 is normal @@ -270,46 +270,46 @@ COMMENT ON TABLE s2_canvas IS 'canvas table'; CREATE TABLE IF NOT EXISTS `s2_query_stat_info` ( - `id` INT NOT NULL AUTO_INCREMENT, - `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier - `model_id` INT DEFAULT NULL, - `data_set_id` INT DEFAULT NULL, - `user` varchar(200) DEFAULT NULL, - `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , - `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene - `query_type_back` INT DEFAULT '0' , -- query type, 0-normal query, 1-pre-refresh type - `query_sql_cmd`LONGVARCHAR , -- sql type request parameter - `sql_cmd_md5` varchar(200) DEFAULT NULL, -- sql type request parameter md5 - `query_struct_cmd`LONGVARCHAR , -- struct type request parameter - `struct_cmd_md5` varchar(200) DEFAULT NULL, -- struct type request parameter md5值 - `sql`LONGVARCHAR , - `sql_md5` varchar(200) DEFAULT NULL, -- sql md5 - `query_engine` varchar(20) DEFAULT NULL, - `elapsed_ms` bigINT DEFAULT NULL, - `query_state` varchar(20) DEFAULT NULL, - `native_query` INT DEFAULT NULL, -- 1-detail query, 0-aggregation query - `start_date` varchar(50) DEFAULT NULL, - `end_date` varchar(50) DEFAULT NULL, - `dimensions`LONGVARCHAR , -- dimensions involved in sql - `metrics`LONGVARCHAR , -- metric involved in sql - `select_cols`LONGVARCHAR , - `agg_cols`LONGVARCHAR , - `filter_cols`LONGVARCHAR , - `group_by_cols`LONGVARCHAR , - `order_by_cols`LONGVARCHAR , - `use_result_cache` TINYINT DEFAULT '-1' , -- whether to hit the result cache - `use_sql_cache` TINYINT DEFAULT '-1' , -- whether to hit the sql cache - `sql_cache_key`LONGVARCHAR , -- sql cache key - `result_cache_key`LONGVARCHAR , -- result cache key - `query_opt_mode` varchar(50) DEFAULT NULL , - PRIMARY KEY (`id`) + `id` INT NOT NULL AUTO_INCREMENT, + `trace_id` varchar(200) DEFAULT NULL, -- query unique identifier + `model_id` INT DEFAULT NULL, + `data_set_id` INT DEFAULT NULL, + `user` varchar(200) DEFAULT NULL, + `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP , + `query_type` varchar(200) DEFAULT NULL, -- the corresponding scene + `query_type_back` INT DEFAULT '0' , -- query type, 0-normal query, 1-pre-refresh type + `query_sql_cmd`LONGVARCHAR , -- sql type request parameter + `sql_cmd_md5` varchar(200) DEFAULT NULL, -- sql type request parameter md5 + `query_struct_cmd`LONGVARCHAR , -- struct type request parameter + `struct_cmd_md5` varchar(200) DEFAULT NULL, -- struct type request parameter md5值 + `sql`LONGVARCHAR , + `sql_md5` varchar(200) DEFAULT NULL, -- sql md5 + `query_engine` varchar(20) DEFAULT NULL, + `elapsed_ms` bigINT DEFAULT NULL, + `query_state` varchar(20) DEFAULT NULL, + `native_query` INT DEFAULT NULL, -- 1-detail query, 0-aggregation query + `start_date` varchar(50) DEFAULT NULL, + `end_date` varchar(50) DEFAULT NULL, + `dimensions`LONGVARCHAR , -- dimensions involved in sql + `metrics`LONGVARCHAR , -- metric involved in sql + `select_cols`LONGVARCHAR , + `agg_cols`LONGVARCHAR , + `filter_cols`LONGVARCHAR , + `group_by_cols`LONGVARCHAR , + `order_by_cols`LONGVARCHAR , + `use_result_cache` TINYINT DEFAULT '-1' , -- whether to hit the result cache + `use_sql_cache` TINYINT DEFAULT '-1' , -- whether to hit the sql cache + `sql_cache_key`LONGVARCHAR , -- sql cache key + `result_cache_key`LONGVARCHAR , -- result cache key + `query_opt_mode` varchar(50) DEFAULT NULL , + PRIMARY KEY (`id`) ) ; COMMENT ON TABLE s2_query_stat_info IS 'query statistics table'; CREATE TABLE IF NOT EXISTS `s2_semantic_pasre_info` ( - `id` INT NOT NULL AUTO_INCREMENT, - `trace_id` varchar(200) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT, + `trace_id` varchar(200) NOT NULL , `model_id` INT NOT NULL , `dimensions`LONGVARCHAR , `metrics`LONGVARCHAR , @@ -329,9 +329,9 @@ COMMENT ON TABLE s2_semantic_pasre_info IS 'semantic layer sql parsing informati CREATE TABLE IF NOT EXISTS `s2_available_date_info` ( - `id` INT NOT NULL AUTO_INCREMENT , - `item_id` INT NOT NULL , - `type` varchar(255) NOT NULL , + `id` INT NOT NULL AUTO_INCREMENT , + `item_id` INT NOT NULL , + `type` varchar(255) NOT NULL , `date_format` varchar(64) NOT NULL , `start_date` varchar(64) , `end_date` varchar(64) , @@ -374,6 +374,7 @@ CREATE TABLE IF NOT EXISTS s2_agent examples varchar(500) null, config varchar(2000) null, llm_config varchar(2000) null, + prompt_config varchar(5000) null, multi_turn_config varchar(2000) null, visual_config varchar(2000) null, created_by varchar(100) null,