(improvement)(semantic) Supply and compatible with mysql demo data (#464)

Co-authored-by: jolunoluo

launchers/standalone/src/main/java/com/tencent/supersonic/BenchMarkDemoDataLoader.java

@@ -20,9 +20,6 @@ import com.tencent.supersonic.semantic.model.domain.ModelRelaService;
 import com.tencent.supersonic.semantic.model.domain.ModelService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.CommandLineRunner;
-import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
@@ -32,27 +29,18 @@ import java.util.List;
 
 @Component
 @Slf4j
-@Order(2)
+public class BenchMarkDemoDataLoader {
-public class LoadBenchMarkDemo implements CommandLineRunner {
 
     private User user = User.getFakeUser();
 
-    @Value("${spring.h2.demo.enabled:false}")
-    private boolean demoEnable;
-
     @Autowired
     private DomainService domainService;
     @Autowired
     private ModelService modelService;
-
     @Autowired
     private ModelRelaService modelRelaService;
 
-    @Override
+    public void doRun() {
-    public void run(String... args) {
-        if (!demoEnable) {
-            return;
-        }
         try {
             addDomain();
             addModel_1();

launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java

@@ -34,9 +34,6 @@ import com.tencent.supersonic.common.util.JsonUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.ApplicationListener;
 import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
@@ -46,7 +43,7 @@ import java.util.List;
 
 @Component
 @Slf4j
-public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent> {
+public class ChatDemoLoader {
 
     private User user = User.getFakeUser();
     @Qualifier("chatQueryService")
@@ -63,8 +60,27 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
     @Autowired
     private SysParameterService sysParameterService;
 
-    @Value("${spring.h2.demo.enabled:false}")
+    public void doRun() {
-    private boolean demoEnable;
+        try {
+            addSysParameter();
+            addDemoChatConfig_1();
+            addDemoChatConfig_2();
+            addDemoChatConfig_3();
+            addDemoChatConfig_4();
+            addDemoChatConfig_5();
+            addDemoChatConfig_6();
+            addDemoChatConfig_7();
+            addDemoChatConfig_8();
+            addPlugin_1();
+            addAgent1();
+            addAgent2();
+            addAgent3();
+            addSampleChats();
+            addSampleChats2();
+        } catch (Exception e) {
+            log.error("Failed to add sample chats", e);
+        }
+    }
 
     private void parseAndExecute(int chatId, String queryText) throws Exception {
         QueryReq queryRequest = new QueryReq();
@@ -474,30 +490,4 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
         agentService.createAgent(agent, User.getFakeUser());
     }
 
-    @Override
-    public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
-        if (!demoEnable) {
-            return;
-        }
-        try {
-            addSysParameter();
-            addDemoChatConfig_1();
-            addDemoChatConfig_2();
-            addDemoChatConfig_3();
-            addDemoChatConfig_4();
-            addDemoChatConfig_5();
-            addDemoChatConfig_6();
-            addDemoChatConfig_7();
-            addDemoChatConfig_8();
-            addPlugin_1();
-            addAgent1();
-            addAgent2();
-            addAgent3();
-            addSampleChats();
-            addSampleChats2();
-        } catch (Exception e) {
-            log.error("Failed to add sample chats", e);
-        }
-    }
-
 }

launchers/standalone/src/main/java/com/tencent/supersonic/DemoLoader.java

@@ -0,0 +1,50 @@
+package com.tencent.supersonic;
+
+import com.tencent.supersonic.semantic.model.domain.DomainService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+@Component
+@Slf4j
+@Order(1)
+public class DemoLoader implements CommandLineRunner {
+
+    @Autowired
+    private DomainService domainService;
+
+    @Autowired
+    private ModelDemoDataLoader modelDataDemoLoader;
+
+    @Autowired
+    private BenchMarkDemoDataLoader benchMarkDemoLoader;
+
+    @Autowired
+    private ChatDemoLoader chatDemoLoader;
+
+    @Value("${demo.enabled:false}")
+    private boolean demoEnabled;
+
+    @Override
+    public void run(String... args) {
+        if (!checkLoadDemo()) {
+            log.info("skip load demo");
+            return;
+        }
+        modelDataDemoLoader.doRun();
+        benchMarkDemoLoader.doRun();
+        chatDemoLoader.doRun();
+    }
+
+    private boolean checkLoadDemo() {
+        if (!demoEnabled) {
+            return false;
+        }
+        return CollectionUtils.isEmpty(domainService.getDomainList());
+    }
+
+}

launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java

@@ -34,10 +34,10 @@ import com.tencent.supersonic.semantic.model.domain.MetricService;
 import com.tencent.supersonic.semantic.model.domain.ModelRelaService;
 import com.tencent.supersonic.semantic.model.domain.ModelService;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
-import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Component;
 
 import java.util.ArrayList;
@@ -47,14 +47,12 @@ import java.util.List;
 
 @Component
 @Slf4j
-@Order(1)
+public class ModelDemoDataLoader {
-public class LoadModelDataDemo implements CommandLineRunner {
 
     private User user = User.getFakeUser();
 
-    @Value("${spring.h2.demo.enabled:false}")
+    @Value("${demo.dbType:mysql}")
-    private boolean demoEnable;
+    private String demoDb;
-
     @Autowired
     private DatabaseService databaseService;
     @Autowired
@@ -69,12 +67,10 @@ public class LoadModelDataDemo implements CommandLineRunner {
     private MetricService metricService;
     @Autowired
     private AuthService authService;
+    @Autowired
+    private DataSourceProperties dataSourceProperties;
 
-    @Override
+    public void doRun() {
-    public void run(String... args) {
-        if (!demoEnable) {
-            return;
-        }
         try {
             addDatabase();
             addDomain();
@@ -96,13 +92,20 @@ public class LoadModelDataDemo implements CommandLineRunner {
     }
 
     public void addDatabase() {
+        String url = dataSourceProperties.getUrl();
         DatabaseReq databaseReq = new DatabaseReq();
-        databaseReq.setName("H2数据实例");
+        databaseReq.setName("数据实例");
         databaseReq.setDescription("样例数据库实例");
-        databaseReq.setType(DataTypeEnum.H2.getFeature());
+        if (StringUtils.isNotBlank(url)
-        databaseReq.setUrl("jdbc:h2:mem:semantic;DATABASE_TO_UPPER=false");
+                && url.toLowerCase().contains(DataTypeEnum.MYSQL.getFeature().toLowerCase())) {
-        databaseReq.setUsername("root");
+            databaseReq.setType(DataTypeEnum.MYSQL.getFeature());
-        databaseReq.setPassword("semantic");
+            databaseReq.setVersion("5.7");
+        } else {
+            databaseReq.setType(DataTypeEnum.H2.getFeature());
+        }
+        databaseReq.setUrl(url);
+        databaseReq.setUsername(dataSourceProperties.getUsername());
+        databaseReq.setPassword(dataSourceProperties.getPassword());
         databaseService.createOrUpdateDatabase(databaseReq, user);
     }
 
@@ -141,8 +144,8 @@ public class LoadModelDataDemo implements CommandLineRunner {
         modelDetail.setDimensions(dimensions);
 
         modelDetail.setMeasures(Collections.emptyList());
-        modelDetail.setQueryType("table_query");
+        modelDetail.setQueryType("sql_query");
-        modelDetail.setSqlQuery("select user_name,department from PUBLIC.s2_user_department");
+        modelDetail.setSqlQuery("select user_name,department from s2_user_department");
         modelReq.setModelDetail(modelDetail);
         modelReq.setDomainId(1L);
         modelService.createModel(modelReq, user);
@@ -218,8 +221,8 @@ public class LoadModelDataDemo implements CommandLineRunner {
 
         modelDetail.setMeasures(measures);
         modelDetail.setSqlQuery(
-                "select imp_date,user_name as stay_hours_user_name,stay_hours,page from PUBLIC.s2_stay_time_statis");
+                "select imp_date,user_name as stay_hours_user_name,stay_hours,page from s2_stay_time_statis");
-        modelDetail.setQueryType("table_query");
+        modelDetail.setQueryType("sql_query");
         modelReq.setDomainId(1L);
         modelReq.setModelDetail(modelDetail);
         modelService.createModel(modelReq, user);
@@ -296,8 +299,9 @@ public class LoadModelDataDemo implements CommandLineRunner {
         Measure measure2 = new Measure("下载量", "down_cnt", "sum", 1);
         Measure measure3 = new Measure("收藏量", "favor_cnt", "sum", 1);
         modelDetail.setMeasures(Lists.newArrayList(measure1, measure2, measure3));
-        modelDetail.setQueryType("table_query");
+        modelDetail.setQueryType("sql_query");
-        modelDetail.setTableQuery("PUBLIC.singer");
+        modelDetail.setSqlQuery("select imp_date, singer_name, act_area, song_name, genre, "
+                + "js_play_cnt, down_cnt, favor_cnt from singer");
         modelReq.setModelDetail(modelDetail);
         modelService.createModel(modelReq, user);
     }

launchers/standalone/src/main/resources/application-local.yaml

@@ -4,8 +4,6 @@ spring:
       path: /h2-console/semantic
       # enabled web
       enabled: true
-    demo:
-      enabled: true
   datasource:
     driver-class-name: org.h2.Driver
     schema: classpath:db/schema-h2.sql
@@ -14,6 +12,9 @@ spring:
     username: root
     password: semantic
 
+demo:
+  enabled: true
+
 server:
   port: 9080
 

launchers/standalone/src/main/resources/db/schema-mysql.sql

@@ -1,3 +1,70 @@
+-------demo for semantic and chat
+CREATE TABLE `s2_user_department` (
+      `user_name` varchar(200) NOT NULL,
+       `department` varchar(200) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE `s2_pv_uv_statis` (
+      `imp_date` varchar(200) NOT NULL,
+      `user_name` varchar(200) NOT NULL,
+      `page` varchar(200) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE `s2_stay_time_statis` (
+       `imp_date` varchar(200) NOT NULL,
+       `user_name` varchar(200) NOT NULL,
+       `stay_hours` DOUBLE NOT NULL,
+       `page` varchar(200) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS `singer` (
+    `imp_date` varchar(200) NOT NULL,
+    `singer_name` varchar(200) NOT NULL,
+    `act_area` varchar(200) NOT NULL,
+    `song_name` varchar(200) NOT NULL,
+    `genre` varchar(200) NOT NULL,
+    `js_play_cnt` bigint DEFAULT NULL,
+    `down_cnt` bigint DEFAULT NULL,
+    `favor_cnt` bigint DEFAULT NULL
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- benchmark
+CREATE TABLE IF NOT EXISTS `genre` (
+    `g_name` varchar(20) NOT NULL , -- genre name
+    `rating` INT ,
+    `most_popular_in` varchar(50) ,
+    PRIMARY KEY (`g_name`)
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS `artist` (
+    `artist_name` varchar(50) NOT NULL , -- genre name
+    `country` varchar(20) ,
+    `gender` varchar(20) ,
+    `g_name` varchar(50)
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS `files` (
+     `f_id` bigINT NOT NULL,
+     `artist_name` varchar(50) ,
+    `file_size` varchar(20) ,
+    `duration` varchar(20) ,
+    `formats` varchar(20) ,
+    PRIMARY KEY (`f_id`)
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS `song` (
+    `imp_date` varchar(50) ,
+    `song_name` varchar(50) ,
+    `artist_name` varchar(50) ,
+    `country` varchar(20) ,
+    `f_id` bigINT ,
+    `g_name` varchar(20) ,
+    `rating` int ,
+    `languages` varchar(20) ,
+    `releasedate` varchar(50) ,
+    `resolution` bigINT NOT NULL
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
 CREATE TABLE `s2_agent` (
                             `id` int(11) NOT NULL AUTO_INCREMENT,
                             `name` varchar(100) COLLATE utf8_unicode_ci DEFAULT NULL,
@@ -15,9 +82,9 @@ CREATE TABLE `s2_agent` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 
 CREATE TABLE `s2_auth_groups` (
-                                  `group_id` int(11) NOT NULL,
+      `group_id` int(11) NOT NULL,
-                                  `config` varchar(2048) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+      `config` varchar(2048) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
-                                  PRIMARY KEY (`group_id`)
+      PRIMARY KEY (`group_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 
@@ -387,9 +454,6 @@ create table s2_user
     PRIMARY KEY (`id`)
 );
 
-insert into s2_user (id, `name`, password, display_name, email, is_admin) values (1, 'admin','admin','admin','admin@xx.com', 1);
-
-
 CREATE TABLE `s2_materialization`
 (
     `id`                bigint(20) NOT NULL AUTO_INCREMENT,
@@ -456,7 +520,7 @@ CREATE TABLE s2_sys_parameter
     id  int primary key AUTO_INCREMENT COMMENT '主键id',
     admin varchar(500) COMMENT '系统管理员',
     parameters text null COMMENT '配置项'
-);
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 CREATE TABLE s2_model_rela
 (
@@ -466,7 +530,7 @@ CREATE TABLE s2_model_rela
     to_model_id      bigint,
     join_type       VARCHAR(255),
     join_condition  VARCHAR(255)
-);
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 CREATE TABLE `s2_collect` (
     `id` bigint NOT NULL AUTO_INCREMENT,
@@ -476,4 +540,4 @@ CREATE TABLE `s2_collect` (
     `create_time` datetime,
     `update_time` datetime,
     PRIMARY KEY (`id`)
-);
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

semantic/api/src/main/java/com/tencent/supersonic/semantic/api/model/request/DatabaseReq.java

@@ -36,8 +36,8 @@ public class DatabaseReq {
 
     private List<String> viewers = Lists.newArrayList();
 
-    public String getUrl() {
+    public String getConnectUrl() {
-        if (type.equalsIgnoreCase(DataTypeEnum.H2.getFeature())) {
+        if (StringUtils.isNotBlank(url)) {
             return url;
         }
         String databaseUrl = database;

semantic/model/src/main/java/com/tencent/supersonic/semantic/model/domain/utils/DatabaseConverter.java

@@ -6,9 +6,9 @@ import com.tencent.supersonic.semantic.api.model.response.DatabaseResp;
 import com.tencent.supersonic.semantic.model.domain.dataobject.DatabaseDO;
 import com.tencent.supersonic.semantic.model.domain.pojo.ConnectInfo;
 import com.tencent.supersonic.semantic.model.domain.pojo.Database;
-import java.util.Arrays;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
+import java.util.Arrays;
 public class DatabaseConverter {
 
     public static Database convert(DatabaseReq databaseReq) {
@@ -17,7 +17,7 @@ public class DatabaseConverter {
         ConnectInfo connectInfo = new ConnectInfo();
         connectInfo.setUserName(databaseReq.getUsername());
         connectInfo.setPassword(databaseReq.getPassword());
-        connectInfo.setUrl(databaseReq.getUrl());
+        connectInfo.setUrl(databaseReq.getConnectUrl());
         connectInfo.setDatabase(databaseReq.getDatabase());
         database.setConnectInfo(connectInfo);
         database.setVersion(databaseReq.getVersion());