topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(semantic) Supply and compatible with mysql demo data (#464)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-12-02 23:17:27 +08:00
committed by GitHub
parent c98d15059b
commit 4280aad0a7
9 changed files with 1394 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java → launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java
View File

@@ -20,9 +20,6 @@ import com.tencent.supersonic.semantic.model.domain.ModelRelaService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
@@ -32,27 +29,18 @@ import java.util.List;
@Component
@Slf4j
@Order(2)
public class LoadBenchMarkDemo implements CommandLineRunner {
public class BenchMarkDemoDataLoader {
private User user = User.getFakeUser();
@Value("${spring.h2.demo.enabled:false}")
private boolean demoEnable;
@Autowired
private DomainService domainService;
@Autowired
private ModelService modelService;
@Autowired
private ModelRelaService modelRelaService;
@Override
public void run(String... args) {
if (!demoEnable) {
return;
}
public void doRun() {
try {
addDomain();
addModel_1();

54
launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java → launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java
View File

@@ -34,9 +34,6 @@ import com.tencent.supersonic.common.util.JsonUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
@@ -46,7 +43,7 @@ import java.util.List;
@Component
@Slf4j
public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent> {
public class ChatDemoLoader {
private User user = User.getFakeUser();
@Qualifier("chatQueryService")
@@ -63,8 +60,27 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
@Autowired
private SysParameterService sysParameterService;
@Value("${spring.h2.demo.enabled:false}")
private boolean demoEnable;
public void doRun() {
try {
addSysParameter();
addDemoChatConfig_1();
addDemoChatConfig_2();
addDemoChatConfig_3();
addDemoChatConfig_4();
addDemoChatConfig_5();
addDemoChatConfig_6();
addDemoChatConfig_7();
addDemoChatConfig_8();
addPlugin_1();
addAgent1();
addAgent2();
addAgent3();
addSampleChats();
addSampleChats2();
} catch (Exception e) {
log.error("Failed to add sample chats", e);
}
}
private void parseAndExecute(int chatId, String queryText) throws Exception {
QueryReq queryRequest = new QueryReq();
@@ -474,30 +490,4 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
agentService.createAgent(agent, User.getFakeUser());
}
@Override
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
if (!demoEnable) {
return;
}
try {
addSysParameter();
addDemoChatConfig_1();
addDemoChatConfig_2();
addDemoChatConfig_3();
addDemoChatConfig_4();
addDemoChatConfig_5();
addDemoChatConfig_6();
addDemoChatConfig_7();
addDemoChatConfig_8();
addPlugin_1();
addAgent1();
addAgent2();
addAgent3();
addSampleChats();
addSampleChats2();
} catch (Exception e) {
log.error("Failed to add sample chats", e);
}
}
}

50
launchers/standalone/src/main/java/com/tencent/supersonic/DemoLoader.java Normal file
View File

@@ -0,0 +1,50 @@
package com.tencent.supersonic;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@Order(1)
public class DemoLoader implements CommandLineRunner {
@Autowired
private DomainService domainService;
@Autowired
private ModelDemoDataLoader modelDataDemoLoader;
@Autowired
private BenchMarkDemoDataLoader benchMarkDemoLoader;
@Autowired
private ChatDemoLoader chatDemoLoader;
@Value("${demo.enabled:false}")
private boolean demoEnabled;
@Override
public void run(String... args) {
if (!checkLoadDemo()) {
log.info("skip load demo");
return;
}
modelDataDemoLoader.doRun();
benchMarkDemoLoader.doRun();
chatDemoLoader.doRun();
}
private boolean checkLoadDemo() {
if (!demoEnabled) {
return false;
}
return CollectionUtils.isEmpty(domainService.getDomainList());
}
}

50
launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java → launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
View File

@@ -34,10 +34,10 @@ import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelRelaService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
@@ -47,14 +47,12 @@ import java.util.List;
@Component
@Slf4j
@Order(1)
public class LoadModelDataDemo implements CommandLineRunner {
public class ModelDemoDataLoader {
private User user = User.getFakeUser();
@Value("${spring.h2.demo.enabled:false}")
private boolean demoEnable;
@Value("${demo.dbType:mysql}")
private String demoDb;
@Autowired
private DatabaseService databaseService;
@Autowired
@@ -69,12 +67,10 @@ public class LoadModelDataDemo implements CommandLineRunner {
private MetricService metricService;
@Autowired
private AuthService authService;
@Autowired
private DataSourceProperties dataSourceProperties;
@Override
public void run(String... args) {
if (!demoEnable) {
return;
}
public void doRun() {
try {
addDatabase();
addDomain();
@@ -96,13 +92,20 @@ public class LoadModelDataDemo implements CommandLineRunner {
}
public void addDatabase() {
String url = dataSourceProperties.getUrl();
DatabaseReq databaseReq = new DatabaseReq();
databaseReq.setName("H2数据实例");
databaseReq.setName("数据实例");
databaseReq.setDescription("样例数据库实例");
databaseReq.setType(DataTypeEnum.H2.getFeature());
databaseReq.setUrl("jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false");
databaseReq.setUsername("root");
databaseReq.setPassword("semantic");
if (StringUtils.isNotBlank(url)
&& url.toLowerCase().contains(DataTypeEnum.MYSQL.getFeature().toLowerCase())) {
databaseReq.setType(DataTypeEnum.MYSQL.getFeature());
databaseReq.setVersion("5.7");
} else {
databaseReq.setType(DataTypeEnum.H2.getFeature());
}
databaseReq.setUrl(url);
databaseReq.setUsername(dataSourceProperties.getUsername());
databaseReq.setPassword(dataSourceProperties.getPassword());
databaseService.createOrUpdateDatabase(databaseReq, user);
}
@@ -141,8 +144,8 @@ public class LoadModelDataDemo implements CommandLineRunner {
modelDetail.setDimensions(dimensions);
modelDetail.setMeasures(Collections.emptyList());
modelDetail.setQueryType("table_query");
modelDetail.setSqlQuery("select user_name,department from PUBLIC.s2_user_department");
modelDetail.setQueryType("sql_query");
modelDetail.setSqlQuery("select user_name,department from s2_user_department");
modelReq.setModelDetail(modelDetail);
modelReq.setDomainId(1L);
modelService.createModel(modelReq, user);
@@ -218,8 +221,8 @@ public class LoadModelDataDemo implements CommandLineRunner {
modelDetail.setMeasures(measures);
modelDetail.setSqlQuery(
"select imp_date,user_name as stay_hours_user_name,stay_hours,page from PUBLIC.s2_stay_time_statis");
modelDetail.setQueryType("table_query");
"select imp_date,user_name as stay_hours_user_name,stay_hours,page from s2_stay_time_statis");
modelDetail.setQueryType("sql_query");
modelReq.setDomainId(1L);
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
@@ -296,8 +299,9 @@ public class LoadModelDataDemo implements CommandLineRunner {
Measure measure2 = new Measure("下载量", "down_cnt", "sum", 1);
Measure measure3 = new Measure("收藏量", "favor_cnt", "sum", 1);
modelDetail.setMeasures(Lists.newArrayList(measure1, measure2, measure3));
modelDetail.setQueryType("table_query");
modelDetail.setTableQuery("PUBLIC.singer");
modelDetail.setQueryType("sql_query");
modelDetail.setSqlQuery("select imp_date, singer_name, act_area, song_name, genre, "
+ "js_play_cnt, down_cnt, favor_cnt from singer");
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
}

5
launchers/standalone/src/main/resources/application-local.yaml
View File

@@ -4,8 +4,6 @@ spring:
path: /h2-console/semantic
# enabled web
enabled: true
demo:
enabled: true
datasource:
driver-class-name: org.h2.Driver
schema: classpath:db/schema-h2.sql
@@ -14,6 +12,9 @@ spring:
username: root
password: semantic
demo:
enabled: true
server:
port: 9080

1213
launchers/standalone/src/main/resources/db/data-mysql.sql Normal file
View File

File diff suppressed because it is too large Load Diff

82
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -1,3 +1,70 @@
-------demo for semantic and chat
CREATE TABLE `s2_user_department` (
`user_name` varchar(200) NOT NULL,
`department` varchar(200) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `s2_pv_uv_statis` (
`imp_date` varchar(200) NOT NULL,
`user_name` varchar(200) NOT NULL,
`page` varchar(200) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `s2_stay_time_statis` (
`imp_date` varchar(200) NOT NULL,
`user_name` varchar(200) NOT NULL,
`stay_hours` DOUBLE NOT NULL,
`page` varchar(200) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `singer` (
`imp_date` varchar(200) NOT NULL,
`singer_name` varchar(200) NOT NULL,
`act_area` varchar(200) NOT NULL,
`song_name` varchar(200) NOT NULL,
`genre` varchar(200) NOT NULL,
`js_play_cnt` bigint DEFAULT NULL,
`down_cnt` bigint DEFAULT NULL,
`favor_cnt` bigint DEFAULT NULL
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- benchmark
CREATE TABLE IF NOT EXISTS `genre` (
`g_name` varchar(20) NOT NULL , -- genre name
`rating` INT ,
`most_popular_in` varchar(50) ,
PRIMARY KEY (`g_name`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `artist` (
`artist_name` varchar(50) NOT NULL , -- genre name
`country` varchar(20) ,
`gender` varchar(20) ,
`g_name` varchar(50)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `files` (
`f_id` bigINT NOT NULL,
`artist_name` varchar(50) ,
`file_size` varchar(20) ,
`duration` varchar(20) ,
`formats` varchar(20) ,
PRIMARY KEY (`f_id`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS `song` (
`imp_date` varchar(50) ,
`song_name` varchar(50) ,
`artist_name` varchar(50) ,
`country` varchar(20) ,
`f_id` bigINT ,
`g_name` varchar(20) ,
`rating` int ,
`languages` varchar(20) ,
`releasedate` varchar(50) ,
`resolution` bigINT NOT NULL
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `s2_agent` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
@@ -15,9 +82,9 @@ CREATE TABLE `s2_agent` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
CREATE TABLE `s2_auth_groups` (
`group_id` int(11) NOT NULL,
`config` varchar(2048) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`group_id`)
`group_id` int(11) NOT NULL,
`config` varchar(2048) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`group_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
@@ -387,9 +454,6 @@ create table s2_user
PRIMARY KEY (`id`)
);
insert into s2_user (id, `name`, password, display_name, email, is_admin) values (1, 'admin','admin','admin','admin@xx.com', 1);
CREATE TABLE `s2_materialization`
(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
@@ -456,7 +520,7 @@ CREATE TABLE s2_sys_parameter
id int primary key AUTO_INCREMENT COMMENT '主键id',
admin varchar(500) COMMENT '系统管理员',
parameters text null COMMENT '配置项'
);
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE s2_model_rela
(
@@ -466,7 +530,7 @@ CREATE TABLE s2_model_rela
to_model_id bigint,
join_type VARCHAR(255),
join_condition VARCHAR(255)
);
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE `s2_collect` (
`id` bigint NOT NULL AUTO_INCREMENT,
@@ -476,4 +540,4 @@ CREATE TABLE `s2_collect` (
`create_time` datetime,
`update_time` datetime,
PRIMARY KEY (`id`)
);
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

4
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatabaseReq.java
View File

@@ -36,8 +36,8 @@ public class DatabaseReq {
private List<String> viewers = Lists.newArrayList();
public String getUrl() {
if (type.equalsIgnoreCase(DataTypeEnum.H2.getFeature())) {
public String getConnectUrl() {
if (StringUtils.isNotBlank(url)) {
return url;
}
String databaseUrl = database;

4
semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatabaseConverter.java
View File

@@ -6,9 +6,9 @@ import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
import com.tencent.supersonic.semantic.model.domain.dataobject.DatabaseDO;
import com.tencent.supersonic.semantic.model.domain.pojo.ConnectInfo;
import com.tencent.supersonic.semantic.model.domain.pojo.Database;
import java.util.Arrays;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import java.util.Arrays;
public class DatabaseConverter {
public static Database convert(DatabaseReq databaseReq) {
@@ -17,7 +17,7 @@ public class DatabaseConverter {
ConnectInfo connectInfo = new ConnectInfo();
connectInfo.setUserName(databaseReq.getUsername());
connectInfo.setPassword(databaseReq.getPassword());
connectInfo.setUrl(databaseReq.getUrl());
connectInfo.setUrl(databaseReq.getConnectUrl());
connectInfo.setDatabase(databaseReq.getDatabase());
database.setConnectInfo(connectInfo);
database.setVersion(databaseReq.getVersion());