topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-15 14:36:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(project) support for modifying filter conditions and fix group by pushdown and add windows scipt (#49)

Browse Source 
Co-authored-by: lexluo <lexluo@tencent.com>
This commit is contained in:
lexluo09
2023-09-03 23:51:47 +08:00
committed by GitHub
parent 8440f1f30e
commit 559ef974b0
317 changed files with 7449 additions and 9413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java
View File

@@ -71,7 +71,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
}
public void addSampleChats() throws Exception {
chatService.addChat(user, "样例对话1");
chatService.addChat(user, "样例对话1", 1);
parseAndExecute(1, "超音数 访问次数");
parseAndExecute(1, "按部门统计");
@@ -79,7 +79,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
}
public void addSampleChats2() throws Exception {
chatService.addChat(user, "样例对话2");
chatService.addChat(user, "样例对话2", 1);
parseAndExecute(2, "alice 停留时长");
parseAndExecute(2, "对比alice和lucy的访问次数");

3
launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java
View File

@@ -1,6 +1,5 @@
package com.tencent.supersonic.db;
import javax.sql.DataSource;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
@@ -9,6 +8,8 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import javax.sql.DataSource;
@Configuration
@MapperScan(value = "com.tencent.supersonic", annotationClass = Mapper.class)

5
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -32,9 +32,10 @@ com.tencent.supersonic.auth.api.authentication.adaptor.UserAdaptor=\
com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.corrector.DateFieldCorrector, \
com.tencent.supersonic.chat.corrector.FieldValueCorrector, \
com.tencent.supersonic.chat.corrector.FieldNameCorrector, \
com.tencent.supersonic.chat.corrector.FieldCorrector, \
com.tencent.supersonic.chat.corrector.FunctionCorrector, \
com.tencent.supersonic.chat.corrector.TableNameCorrector, \
com.tencent.supersonic.chat.corrector.QueryFilterAppend, \
com.tencent.supersonic.chat.corrector.SelectFieldAppendCorrector
com.tencent.supersonic.chat.corrector.SelectFieldAppendCorrector, \
com.tencent.supersonic.chat.corrector.FieldValueCorrector

2
launchers/standalone/src/main/resources/db/data-h2.sql
View File

@@ -9,7 +9,7 @@ insert into s2_user (id, `name`, password, display_name, email) values (5, 'alic
insert into s2_domain (id, `name`, biz_name, parent_id, status, created_at, created_by, updated_at, updated_by, `admin`, admin_org, viewer, view_org) VALUES(1, '超音数', 'supersonic', 0, 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 'admin,tom,jack', 'admin' );
insert into s2_model (id, `name`, biz_name, domain_id, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(1, '超音数', 'supersonic', 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','' );
insert into s2_model (id, `name`, biz_name, domain_id, created_at, created_by, updated_at, updated_by, `admin`, admin_org, is_open, viewer, view_org, entity) VALUES(2, '艺人库', 'singer', 1, '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin', '', 0, 'admin,tom,jack', 'admin','{"entityId": 7, "names": ["歌手", "艺人"]}' );
insert into s2_database (id, domain_id , `name`, description, `type` ,config ,created_at ,created_by ,updated_at ,updated_by) VALUES(1, 1, 'H2数据实例', '', 'h2', '{"password":"semantic","url":"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false","userName":"root"}', '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin');
insert into s2_database (id, `name`, description, `type` ,config ,created_at ,created_by ,updated_at ,updated_by, `admin`) VALUES(1, 'H2数据实例', '', 'h2', '{"password":"semantic","url":"jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false","userName":"root"}', '2023-05-24 00:00:00', 'admin', '2023-05-24 00:00:00', 'admin', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(1, 1, '停留时长统计', 's2_stay_time_statis', '停留时长统计', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"page","dateFormat":"yyyy-MM-dd","expr":"page","isCreateDimension":0,"type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[{"agg":"sum","bizName":"s2_stay_time_statis_stay_hours","expr":"stay_hours","isCreateMetric":1,"name":"停留时长"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date, page,user_name,stay_hours FROM s2_stay_time_statis"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(2, 1, 'PVUV统计', 's2_pv_uv_statis', 'PVUV统计', 1, '{"dimensions":[{"bizName":"imp_date","dateFormat":"yyyy-MM-dd","expr":"imp_date","isCreateDimension":0,"type":"time","typeParams":{"isPrimary":"true","timeGranularity":"day"}},{"bizName":"page","dateFormat":"yyyy-MM-dd","expr":"page","isCreateDimension":0,"type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[{"agg":"sum","bizName":"s2_pv_uv_statis_pv","expr":"pv","isCreateMetric":1,"name":"访问次数"},{"agg":"count_distinct","bizName":"s2_pv_uv_statis_uv","expr":"uv","isCreateMetric":1,"name":"访问人数"}],"queryType":"sql_query","sqlQuery":"SELECT imp_date, user_name,page,1 as pv, user_name as uv FROM s2_pv_uv_statis"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');
insert into s2_datasource (id , model_id, `name`, biz_name, description, database_id ,datasource_detail, created_at, created_by, updated_at, updated_by ) VALUES(3, 1, '用户部门', 'user_department', '用户部门', 1, '{"dimensions":[{"bizName":"department","dateFormat":"yyyy-MM-dd","expr":"department","isCreateDimension":1,"name":"部门","type":"categorical"}],"identifiers":[{"bizName":"user_name","name":"用户名","type":"primary"}],"measures":[],"queryType":"sql_query","sqlQuery":"SELECT user_name,department FROM s2_user_department"}', '2023-05-25 00:00:00', 'admin', '2023-05-25 00:00:00', 'admin');

4
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat_context`
CREATE TABLE IF NOT EXISTS `s2_chat`
(
`chat_id` BIGINT auto_increment ,-- AUTO_INCREMENT,
`agent_id` INT DEFAULT NULL,
`chat_name` varchar(100) DEFAULT NULL,
`create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
`last_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
@@ -133,7 +134,6 @@ COMMENT ON TABLE s2_model IS 'model information';
CREATE TABLE `s2_database` (
`id` INT NOT NULL AUTO_INCREMENT,
`domain_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`version` varchar(64) DEFAULT NULL ,
@@ -143,6 +143,8 @@ CREATE TABLE `s2_database` (
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP NOT NULL ,
`updated_by` varchar(100) NOT NULL,
`admin` varchar(500) NOT NULL,
`viewer` varchar(500) DEFAULT NULL,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_database IS 'database instance table';

363
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -1,79 +1,85 @@
create table supersonic_sit.s2_auth_groups
(
group_id int not null
group_id int not null
primary key,
config varchar(2048) null
) collate = utf8mb4_unicode_ci;
)
collate = utf8mb4_unicode_ci;
create table supersonic_sit.s2_available_date_info
(
id int auto_increment
primary key,
item_id int not null,
type varchar(255) not null,
date_format varchar(64) not null,
start_date varchar(64) null,
end_date varchar(64) null,
unavailable_date text null,
created_at timestamp default CURRENT_TIMESTAMP not null,
created_by varchar(100) not null,
updated_at timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP,
updated_by varchar(100) not null,
status int default 0 null
) collate = utf8mb4_unicode_ci;
item_id int not null,
type varchar(255) not null,
date_format varchar(64) not null,
start_date varchar(64) null,
end_date varchar(64) null,
unavailable_date text null,
created_at timestamp default CURRENT_TIMESTAMP not null,
created_by varchar(100) not null,
updated_at timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP,
updated_by varchar(100) not null,
status int default 0 null
)
collate = utf8mb4_unicode_ci;
create table supersonic_sit.s2_chat
(
chat_id bigint(8) auto_increment
primary key,
chat_name varchar(100) null,
create_time datetime null,
last_time datetime null,
creator varchar(30) null,
last_question varchar(200) null,
chat_name varchar(100) null,
create_time datetime null,
last_time datetime null,
creator varchar(30) null,
last_question varchar(200) null,
is_delete int(2) default 0 null comment 'is deleted',
is_top int(2) default 0 null comment 'is top'
) charset = utf8;
)
charset = utf8;
create table supersonic_sit.s2_chat_config
(
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`domain_id` bigint(20) DEFAULT NULL COMMENT '主题域id',
`chat_detail_config` mediumtext COMMENT '明细模式配置信息',
`chat_agg_config` mediumtext COMMENT '指标模式配置信息',
`recommended_questions` mediumtext COMMENT '推荐问题配置',
`created_at` datetime NOT NULL COMMENT '创建时间',
`updated_at` datetime NOT NULL COMMENT '更新时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`updated_by` varchar(100) NOT NULL COMMENT '更新人',
`status` int(10) NOT NULL COMMENT '主题域扩展信息状态, 0-删除，1-生效',
PRIMARY KEY (`id`)
) comment '主题域扩展信息表' charset = utf8;
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`domain_id` bigint(20) DEFAULT NULL COMMENT '主题域id',
`chat_detail_config` mediumtext COMMENT '明细模式配置信息',
`chat_agg_config` mediumtext COMMENT '指标模式配置信息',
`recommended_questions` mediumtext COMMENT '推荐问题配置',
`created_at` datetime NOT NULL COMMENT '创建时间',
`updated_at` datetime NOT NULL COMMENT '更新时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`updated_by` varchar(100) NOT NULL COMMENT '更新人',
`status` int(10) NOT NULL COMMENT '主题域扩展信息状态, 0-删除，1-生效',
PRIMARY KEY (`id`)
)
comment '主题域扩展信息表' charset = utf8;
create table supersonic_sit.s2_chat_context
(
chat_id bigint not null comment 'context chat id'
primary key,
modified_at datetime default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP comment 'row modify time',
user varchar(64) null comment 'row modify user',
query_text text null comment 'query text',
semantic_parse text null comment 'parse data',
ext_data text null comment 'extend data'
) charset = utf8;
user varchar(64) null comment 'row modify user',
query_text text null comment 'query text',
semantic_parse text null comment 'parse data',
ext_data text null comment 'extend data'
)
charset = utf8;
create table supersonic_sit.s2_chat_query
(
question_id bigint auto_increment
primary key,
create_time timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP,
query_text mediumtext null,
user_name varchar(150) null,
query_state int(1) null,
query_text mediumtext null,
user_name varchar(150) null,
query_state int(1) null,
chat_id bigint not null,
query_response mediumtext not null,
score int default 0 null,
feedback varchar(1024) default '' null
) charset = utf8;
score int default 0 null,
feedback varchar(1024) default '' null
)
charset = utf8;
create table supersonic_sit.s2_database
(
@@ -82,14 +88,15 @@ create table supersonic_sit.s2_database
domain_id bigint not null comment '主题域ID',
name varchar(255) not null comment '名称',
description varchar(500) null comment '描述',
version varchar(64) null comment '版本',
version varchar(64) null comment '版本',
type varchar(20) not null comment '类型 mysql,clickhouse,tdw',
config text not null comment '配置信息',
created_at datetime not null comment '创建时间',
created_by varchar(100) not null comment '创建人',
updated_at datetime not null comment '更新时间',
updated_by varchar(100) not null comment '更新人'
) comment '数据库实例表' charset = utf8;
)
comment '数据库实例表' charset = utf8;
create table supersonic_sit.s2_datasource
(
@@ -105,140 +112,147 @@ create table supersonic_sit.s2_datasource
created_by varchar(100) not null comment '创建人',
updated_at datetime not null comment '更新时间',
updated_by varchar(100) not null comment '更新人'
) charset = utf8;
)
charset = utf8;
create table supersonic_sit.s2_datasource_rela
(
id bigint auto_increment
primary key,
domain_id bigint null,
datasource_from bigint null,
datasource_to bigint null,
domain_id bigint null,
datasource_from bigint null,
datasource_to bigint null,
join_key varchar(100) null,
created_at datetime null,
created_at datetime null,
created_by varchar(100) null,
updated_at datetime null,
updated_at datetime null,
updated_by varchar(100) null
) charset = utf8;
)
charset = utf8;
create table supersonic_sit.s2_dimension
(
id bigint auto_increment comment '维度ID'
primary key,
domain_id bigint not null comment '主题域id',
datasource_id bigint not null comment '所属数据源id',
name varchar(255) not null comment '维度名称',
biz_name varchar(255) not null comment '字段名称',
description varchar(500) not null comment '描述',
status int(10) not null comment '维度状态,0正常,1下架,2删除',
sensitive_level int(10) null comment '敏感级别',
type varchar(50) not null comment '维度类型 categorical,time',
type_params text null comment '类型参数',
expr text not null comment '表达式',
created_at datetime not null comment '创建时间',
created_by varchar(100) not null comment '创建人',
updated_at datetime not null comment '更新时间',
updated_by varchar(100) not null comment '更新人',
semantic_type varchar(20) not null comment '语义类型DATE, ID, CATEGORY',
domain_id bigint not null comment '主题域id',
datasource_id bigint not null comment '所属数据源id',
name varchar(255) not null comment '维度名称',
biz_name varchar(255) not null comment '字段名称',
description varchar(500) not null comment '描述',
status int(10) not null comment '维度状态,0正常,1下架,2删除',
sensitive_level int(10) null comment '敏感级别',
type varchar(50) not null comment '维度类型 categorical,time',
type_params text null comment '类型参数',
expr text not null comment '表达式',
created_at datetime not null comment '创建时间',
created_by varchar(100) not null comment '创建人',
updated_at datetime not null comment '更新时间',
updated_by varchar(100) not null comment '更新人',
semantic_type varchar(20) not null comment '语义类型DATE, ID, CATEGORY',
alias varchar(500) collate utf8_unicode_ci null,
default_values varchar(500) DEFAULT NULL,
dim_value_maps varchar(500) DEFAULT NULL
) comment '维度表' charset = utf8;
default_values varchar(500) DEFAULT NULL,
dim_value_maps varchar(500) DEFAULT NULL
)
comment '维度表' charset = utf8;
create table supersonic_sit.s2_domain
(
id bigint auto_increment comment '自增ID'
primary key,
name varchar(255) null comment '主题域名称',
biz_name varchar(255) null comment '内部名称',
parent_id bigint default 0 null comment '父主题域ID',
status int(10) not null comment '主题域状态',
created_at datetime null comment '创建时间',
created_by varchar(100) null comment '创建人',
updated_at datetime null comment '更新时间',
updated_by varchar(100) null comment '更新人',
admin varchar(3000) null comment '主题域管理员',
admin_org varchar(3000) null comment '主题域管理员组织',
is_open int null comment '主题域是否公开',
viewer varchar(3000) null comment '主题域可用用户',
view_org varchar(3000) null comment '主题域可用组织',
name varchar(255) null comment '主题域名称',
biz_name varchar(255) null comment '内部名称',
parent_id bigint default 0 null comment '父主题域ID',
status int(10) not null comment '主题域状态',
created_at datetime null comment '创建时间',
created_by varchar(100) null comment '创建人',
updated_at datetime null comment '更新时间',
updated_by varchar(100) null comment '更新人',
admin varchar(3000) null comment '主题域管理员',
admin_org varchar(3000) null comment '主题域管理员组织',
is_open int null comment '主题域是否公开',
viewer varchar(3000) null comment '主题域可用用户',
view_org varchar(3000) null comment '主题域可用组织',
entity varchar(500) DEFAULT NULL COMMENT '主题域实体信息'
) comment '主题域基础信息表' charset = utf8;
)
comment '主题域基础信息表' charset = utf8;
create table supersonic_sit.s2_domain_extend
(
id bigint unsigned auto_increment
primary key,
domain_id bigint null comment '主题域id',
domain_id bigint null comment '主题域id',
default_metrics varchar(655) null comment '默认指标',
visibility mediumtext null comment '不可见的维度/指标信息',
entity_info mediumtext null comment '实体信息',
dictionary_info mediumtext null comment '字典相关的维度设置信息',
visibility mediumtext null comment '不可见的维度/指标信息',
entity_info mediumtext null comment '实体信息',
dictionary_info mediumtext null comment '字典相关的维度设置信息',
created_at datetime not null comment '创建时间',
updated_at datetime not null comment '更新时间',
created_by varchar(100) not null comment '创建人',
updated_by varchar(100) not null comment '更新人',
status int(10) not null comment '主题域扩展信息状态, 0-删除，1-生效'
) comment '主题域扩展信息表' collate = utf8mb4_unicode_ci;
status int(10) not null comment '主题域扩展信息状态, 0-删除，1-生效'
)
comment '主题域扩展信息表' collate = utf8mb4_unicode_ci;
create table supersonic_sit.s2_metric
(
id bigint auto_increment
primary key,
domain_id bigint not null comment '主体域ID',
name varchar(255) not null comment '指标名称',
biz_name varchar(255) not null comment '字段名称',
description varchar(500) null comment '描述',
status int(10) not null comment '指标状态,0正常,1下架,2删除',
sensitive_level int(10) not null comment '敏感级别',
type varchar(50) not null comment '指标类型 proxy,expr',
type_params text not null comment '类型参数',
created_at datetime not null comment '创建时间',
created_by varchar(100) not null comment '创建人',
updated_at datetime not null comment '更新时间',
updated_by varchar(100) not null comment '更新人',
data_format_type varchar(50) null comment '数值类型',
data_format varchar(500) null comment '数值类型参数',
domain_id bigint not null comment '主体域ID',
name varchar(255) not null comment '指标名称',
biz_name varchar(255) not null comment '字段名称',
description varchar(500) null comment '描述',
status int(10) not null comment '指标状态,0正常,1下架,2删除',
sensitive_level int(10) not null comment '敏感级别',
type varchar(50) not null comment '指标类型 proxy,expr',
type_params text not null comment '类型参数',
created_at datetime not null comment '创建时间',
created_by varchar(100) not null comment '创建人',
updated_at datetime not null comment '更新时间',
updated_by varchar(100) not null comment '更新人',
data_format_type varchar(50) null comment '数值类型',
data_format varchar(500) null comment '数值类型参数',
alias varchar(500) collate utf8_unicode_ci null
) comment '指标表' charset = utf8;
)
comment '指标表' charset = utf8;
create table supersonic_sit.s2_query_stat_info
(
id bigint unsigned auto_increment
primary key,
trace_id varchar(200) null comment '查询标识',
domain_id bigint null comment '主题域ID',
user varchar(200) null comment '执行sql的用户',
created_at datetime default CURRENT_TIMESTAMP null comment '创建时间',
query_type varchar(200) null comment '查询对应的场景',
query_type_back int(10) default 0 null comment '查询类型, 0-正常查询, 1-预刷类型',
query_sql_cmd mediumtext null comment '对应查询的struct',
sql_cmd_md5 varchar(200) null comment 'sql md5值',
query_struct_cmd mediumtext null comment '对应查询的struct',
struct_cmd_md5 varchar(200) null comment 'sql md5值',
`sql` mediumtext null comment '对应查询的sql',
sql_md5 varchar(200) null comment 'sql md5值',
query_engine varchar(20) null comment '查询引擎',
elapsed_ms bigint(10) null comment '查询耗时',
query_state varchar(20) null comment '查询最终状态',
native_query int(10) null comment '1-明细查询,0-聚合查询',
start_date varchar(50) null comment 'sql开始日期',
end_date varchar(50) null comment 'sql结束日期',
dimensions mediumtext null comment 'sql 涉及的维度',
metrics mediumtext null comment 'sql 涉及的指标',
select_cols mediumtext null comment 'sql select部分涉及的标签',
agg_cols mediumtext null comment 'sql agg部分涉及的标签',
filter_cols mediumtext null comment 'sql where部分涉及的标签',
group_by_cols mediumtext null comment 'sql grouy by部分涉及的标签',
order_by_cols mediumtext null comment 'sql order by部分涉及的标签',
use_result_cache tinyint(1) default -1 null comment '是否命中sql缓存',
use_sql_cache tinyint(1) default -1 null comment '是否命中sql缓存',
sql_cache_key mediumtext null comment '缓存的key',
result_cache_key mediumtext null comment '缓存的key'
) comment '查询统计信息表' collate = utf8mb4_unicode_ci;
trace_id varchar(200) null comment '查询标识',
domain_id bigint null comment '主题域ID',
user varchar(200) null comment '执行sql的用户',
created_at datetime default CURRENT_TIMESTAMP null comment '创建时间',
query_type varchar(200) null comment '查询对应的场景',
query_type_back int(10) default 0 null comment '查询类型, 0-正常查询, 1-预刷类型',
query_sql_cmd mediumtext null comment '对应查询的struct',
sql_cmd_md5 varchar(200) null comment 'sql md5值',
query_struct_cmd mediumtext null comment '对应查询的struct',
struct_cmd_md5 varchar(200) null comment 'sql md5值',
`sql` mediumtext null comment '对应查询的sql',
sql_md5 varchar(200) null comment 'sql md5值',
query_engine varchar(20) null comment '查询引擎',
elapsed_ms bigint(10) null comment '查询耗时',
query_state varchar(20) null comment '查询最终状态',
native_query int(10) null comment '1-明细查询,0-聚合查询',
start_date varchar(50) null comment 'sql开始日期',
end_date varchar(50) null comment 'sql结束日期',
dimensions mediumtext null comment 'sql 涉及的维度',
metrics mediumtext null comment 'sql 涉及的指标',
select_cols mediumtext null comment 'sql select部分涉及的标签',
agg_cols mediumtext null comment 'sql agg部分涉及的标签',
filter_cols mediumtext null comment 'sql where部分涉及的标签',
group_by_cols mediumtext null comment 'sql grouy by部分涉及的标签',
order_by_cols mediumtext null comment 'sql order by部分涉及的标签',
use_result_cache tinyint(1) default -1 null comment '是否命中sql缓存',
use_sql_cache tinyint(1) default -1 null comment '是否命中sql缓存',
sql_cache_key mediumtext null comment '缓存的key',
result_cache_key mediumtext null comment '缓存的key'
)
comment '查询统计信息表' collate = utf8mb4_unicode_ci;
create index domain_index
on supersonic_sit.s2_query_stat_info (domain_id);
@@ -247,42 +261,67 @@ create table supersonic_sit.s2_semantic_pasre_info
(
id bigint unsigned auto_increment
primary key,
trace_id varchar(200) not null comment '查询标识',
domain_id bigint not null comment '主体域ID',
dimensions mediumtext null comment '查询相关的维度信息',
metrics mediumtext null comment '查询相关的指标信息',
orders mediumtext null comment '查询相关的排序信息',
filters mediumtext null comment '查询相关的过滤信息',
date_info mediumtext null comment '查询相关的日期信息',
`limit` bigint not null comment '查询相关的limit信息',
trace_id varchar(200) not null comment '查询标识',
domain_id bigint not null comment '主体域ID',
dimensions mediumtext null comment '查询相关的维度信息',
metrics mediumtext null comment '查询相关的指标信息',
orders mediumtext null comment '查询相关的排序信息',
filters mediumtext null comment '查询相关的过滤信息',
date_info mediumtext null comment '查询相关的日期信息',
`limit` bigint not null comment '查询相关的limit信息',
native_query tinyint(1) default 0 not null comment '1-明细查询,0-聚合查询',
`sql` mediumtext null comment '解析后的sql',
created_at datetime not null comment '创建时间',
created_by varchar(100) not null comment '创建人',
status int(10) not null comment '运行状态',
elapsed_ms bigint(10) null comment 'sql解析耗时'
`sql` mediumtext null comment '解析后的sql',
created_at datetime not null comment '创建时间',
created_by varchar(100) not null comment '创建人',
status int(10) not null comment '运行状态',
elapsed_ms bigint(10) null comment 'sql解析耗时'
) comment '语义层sql解析信息表' charset = utf8;
create table supersonic_sit.s2_view_info
(
id bigint auto_increment
primary key,
domain_id bigint null,
type varchar(20) null comment 'datasource、dimension、metric',
config text null comment 'config detail',
created_at datetime null,
domain_id bigint null,
type varchar(20) null comment 'datasource、dimension、metric',
config text null comment 'config detail',
created_at datetime null,
created_by varchar(100) null,
updated_at datetime null,
updated_at datetime null,
updated_by varchar(100) not null
) charset = utf8;
CREATE TABLE `s2_user`
(
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(100) NOT NULL,
`display_name` varchar(100) DEFAULT NULL,
`password` varchar(100) DEFAULT NULL,
`email` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8;
CREATE TABLE `s2_user` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(100) NOT NULL,
`display_name` varchar(100) DEFAULT NULL,
`password` varchar(100) DEFAULT NULL,
`email` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8;
CREATE TABLE s2_chat_statistics (
`question_id` BIGINT NOT NULL,
`chat_id` BIGINT NOT NULL ,
`user_name` varchar(150) DEFAULT NULL COMMENT '',
`query_text` varchar(200),
`interface_name` varchar(100) DEFAULT NULL COMMENT '',
`cost` INT(6) DEFAULT 0 ,
`type` INT DEFAULT NULL ,
`create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
index `commonIndex` (`question_id`)
);
CREATE TABLE `s2_chat_parse` (
`question_id` bigint(20) NOT NULL,
`chat_id` bigint(20) NOT NULL,
`parse_id` int(11) NOT NULL,
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`query_text` varchar(500) DEFAULT NULL,
`user_name` varchar(150) DEFAULT NULL,
`parse_info` mediumtext NOT NULL,
`is_candidate` int DEFAULT 1 COMMENT '1是candidate,0是selected',
index `commonIndex` (`question_id`)
)

8
launchers/standalone/src/main/resources/db/sql-update.sql
View File

@@ -38,3 +38,11 @@ update s2_plugin set config = replace(config, 'domain', 'model');
--20230823
alter table s2_chat_query add column agent_id int after question_id;
alter table s2_chat_query change column query_response query_result mediumtext;
--20230829
alter table s2_database add column admin varchar(500);
alter table s2_database add column viewer varchar(500);
alter table s2_database drop column domain_id;
--20230831
alter table s2_chat add column agent_id int after chat_id;

10
launchers/standalone/src/main/resources/logback-spring.xml
View File

@@ -1,13 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true">
<contextName>logback</contextName>
<!-- <property name="LOG_PATH" value="${logback.logdir:-logs}"/>-->
<!-- <property name="LOG_PATH" value="${logback.logdir:-logs}"/>-->
<property name="LOG_PATH" value="${LOG_PATH:-logs}"/>
<property name="LOG_APPNAME" value="chat"/>
<!--输出到控制台-->
<appender name="consoleLog" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} %line - %msg%n</pattern>
<pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} %line - %msg%n</pattern>
</encoder>
</appender>
@@ -46,8 +46,7 @@
<!--滚动策略，按照时间滚动 TimeBasedRollingPolicy-->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--文件路径,定义了日志的切分方式——把每一天的日志归档到一个文件中,以防止日志填满整个磁盘空间-->
<FileNamePattern>${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz
</FileNamePattern>
<FileNamePattern>${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz</FileNamePattern>
<!--只保留最近90天的日志-->
<maxHistory>90</maxHistory>
<!--用来指定日志文件的上限大小，那么到了这个值，就会删除旧的日志-->
@@ -75,8 +74,7 @@
<!--滚动策略，按照时间滚动 TimeBasedRollingPolicy-->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--文件路径,定义了日志的切分方式——把每一天的日志归档到一个文件中,以防止日志填满整个磁盘空间-->
<FileNamePattern>${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz
</FileNamePattern>
<FileNamePattern>${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz</FileNamePattern>
<!--只保留最近30天的日志-->
<maxHistory>30</maxHistory>
<!--用来指定日志文件的上限大小，那么到了这个值，就会删除旧的日志-->

18
launchers/standalone/src/main/resources/optimization.properties Normal file
View File

@@ -0,0 +1,18 @@
#hanlp mapper helper
one.detection.size=8
one.detection.max.size=20
metric.dimension.min.threshold=0.3
metric.dimension.threshold=0.3
dimension.value.threshold=0.5
function.bonus.threshold=201
#SatisfactionChecker
long.text.threshold=0.8
short.text.threshold=0.5
query.text.length.threshold=10
candidate.threshold=0.2

11
launchers/standalone/src/test/java/com/tencent/supersonic/integration/BaseQueryTest.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.integration;
import static org.junit.Assert.assertEquals;
import com.tencent.supersonic.StandaloneLauncher;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
@@ -16,9 +14,6 @@ import com.tencent.supersonic.chat.service.ChatService;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.QueryService;
import com.tencent.supersonic.util.DataUtils;
import java.time.LocalDate;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -27,6 +22,12 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import java.time.LocalDate;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = StandaloneLauncher.class)
@ActiveProfiles("local")

5
launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.integration;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -11,9 +9,10 @@ import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
public class EntityQueryTest extends BaseQueryTest {

9
launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.integration;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
@@ -9,11 +7,14 @@ import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import org.junit.Test;
import org.junit.jupiter.api.Order;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
public class MultiTurnsTest extends BaseQueryTest {
@Test

43
launchers/standalone/src/test/java/com/tencent/supersonic/integration/llm/LLMDslParserTest.java
View File

@@ -1,43 +0,0 @@
package com.tencent.supersonic.integration.llm;
import static org.mockito.Mockito.when;
import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.config.LLMConfig;
import com.tencent.supersonic.chat.parser.llm.dsl.LLMDslParser;
import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.integration.BaseQueryTest;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Test;
import org.springframework.boot.test.mock.mockito.MockBean;
public class LLMDslParserTest extends BaseQueryTest {
@MockBean
protected LLMConfig llmConfig;
@Test
public void parse() throws Exception {
String queryText = "周杰伦专辑十一月的萧邦有哪些歌曲";
QueryReq queryReq = DataUtils.getQueryContextReq(10, queryText);
QueryContext queryContext = new QueryContext();
queryContext.setRequest(queryReq);
SemanticParser dslParser = ComponentFactory.getSemanticParsers().stream().filter(parser -> {
if (parser instanceof LLMDslParser) {
return true;
} else {
return false;
}
}
).findFirst().get();
when(llmConfig.getUrl()).thenReturn("llmUrl");
ChatContext chatCtx = new ChatContext();
dslParser.parse(queryContext, chatCtx);
}
}

6
launchers/standalone/src/test/resources/META-INF/spring.factories
View File

@@ -1,5 +1,6 @@
com.tencent.supersonic.chat.api.component.SchemaMapper=\
com.tencent.supersonic.chat.mapper.HanlpDictMapper
com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.QueryModeParser, \
com.tencent.supersonic.chat.parser.rule.ContextInheritParser, \
@@ -8,13 +9,18 @@ com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.AggregateTypeParser, \
com.tencent.supersonic.chat.parser.llm.interpret.MetricInterpretParser
# com.tencent.supersonic.chat.parser.llm.DSLQueryFunction
com.tencent.supersonic.chat.api.component.QueryProcessor=\
com.tencent.supersonic.chat.application.processor.SemanticQueryProcessor
com.tencent.supersonic.chat.api.component.SemanticLayer=\
com.tencent.supersonic.knowledge.semantic.LocalSemanticLayer
com.tencent.supersonic.chat.query.QuerySelector=\
com.tencent.supersonic.chat.query.HeuristicQuerySelector
com.tencent.supersonic.chat.application.query.DomainResolver=\
com.tencent.supersonic.chat.application.query.HeuristicDomainResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

3386
launchers/standalone/src/test/resources/db/data-h2.sql
View File

File diff suppressed because it is too large Load Diff

4
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS `s2_chat_context`
CREATE TABLE IF NOT EXISTS `s2_chat`
(
`chat_id` BIGINT auto_increment ,-- AUTO_INCREMENT,
`agent_id` INT DEFAULT NULL,
`chat_name` varchar(100) DEFAULT NULL,
`create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
`last_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ,
@@ -148,7 +149,6 @@ COMMENT ON TABLE s2_model IS 'model information';
CREATE TABLE `s2_database` (
`id` INT NOT NULL AUTO_INCREMENT,
`domain_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`version` varchar(64) DEFAULT NULL ,
@@ -158,6 +158,8 @@ CREATE TABLE `s2_database` (
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP NOT NULL ,
`updated_by` varchar(100) NOT NULL,
`admin` varchar(500) NOT NULL,
`viewer` varchar(500) DEFAULT NULL,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_database IS 'database instance table';

10
launchers/standalone/src/test/resources/logback-spring.xml
View File

@@ -1,13 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true">
<contextName>logback</contextName>
<!-- <property name="LOG_PATH" value="${logback.logdir:-logs}"/>-->
<!-- <property name="LOG_PATH" value="${logback.logdir:-logs}"/>-->
<property name="LOG_PATH" value="${LOG_PATH:-logs}"/>
<property name="LOG_APPNAME" value="chat"/>
<!--输出到控制台-->
<appender name="consoleLog" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} %line - %msg%n</pattern>
<pattern>%d{HH:mm:ss} [%thread] %-5level %logger{36} %line - %msg%n</pattern>
</encoder>
</appender>
@@ -46,8 +46,7 @@
<!--滚动策略，按照时间滚动 TimeBasedRollingPolicy-->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--文件路径,定义了日志的切分方式——把每一天的日志归档到一个文件中,以防止日志填满整个磁盘空间-->
<FileNamePattern>${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz
</FileNamePattern>
<FileNamePattern>${LOG_PATH}/error.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz</FileNamePattern>
<!--只保留最近90天的日志-->
<maxHistory>90</maxHistory>
<!--用来指定日志文件的上限大小，那么到了这个值，就会删除旧的日志-->
@@ -75,8 +74,7 @@
<!--滚动策略，按照时间滚动 TimeBasedRollingPolicy-->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--文件路径,定义了日志的切分方式——把每一天的日志归档到一个文件中,以防止日志填满整个磁盘空间-->
<FileNamePattern>${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz
</FileNamePattern>
<FileNamePattern>${LOG_PATH}/serviceinfo.${LOG_APPNAME}.%d{yyyy-MM-dd}.log.gz</FileNamePattern>
<!--只保留最近30天的日志-->
<maxHistory>30</maxHistory>
<!--用来指定日志文件的上限大小，那么到了这个值，就会删除旧的日志-->