[improvement][launcher]Change supersonic docker deployment from mysql to postgres.

2025-12-10 11:07:06 +00:00 · 2024-12-02 10:18:19 +08:00
parent b8924ed45e
commit 02f0063846
6 changed files with 125 additions and 108 deletions
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -7,21 +7,20 @@ WORKDIR /usr/src/app
 # Argument to pass in the supersonic version at build time
 ARG SUPERSONIC_VERSION

-# Install necessary packages, including MySQL client
-RUN apt-get update && \
-    apt-get install -y default-mysql-client unzip && \
-    rm -rf /var/lib/apt/lists/*
+RUN apt-get update
+
+# Install necessary packages, including Postgres client
+RUN apt-get install -y postgresql-client

 # Install the vim editor.
-RUN apt-get update && \
-    apt-get install -y vim && \
+RUN apt-get install -y vim && \
    rm -rf /var/lib/apt/lists/*

 # Update the package list and install iputils-ping.
-RUN apt-get update && apt-get install -y iputils-ping
+RUN apt-get install -y iputils-ping

 # 更新包列表并安装 dnsutils 包
-RUN apt-get update && apt-get install -y dnsutils
+RUN apt-get install -y dnsutils

 # Copy the supersonic standalone zip file into the container
 COPY assembly/build/supersonic-standalone-${SUPERSONIC_VERSION}.zip .
@@ -40,4 +39,4 @@ WORKDIR /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION}
 EXPOSE 9080
 # Command to run the supersonic daemon
 RUN chmod +x bin/supersonic-daemon.sh
-CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone prd && tail -f /dev/null"]
+CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone docker && tail -f /dev/null"]
--- a/docker/DockerfileS2
+++ b/docker/DockerfileS2
@@ -0,0 +1,32 @@
+# Use an official OpenJDK runtime as a parent image
+FROM supersonicbi/supersonic:0.9.8
+
+# Set the working directory in the container
+WORKDIR /usr/src/app
+
+# Argument to pass in the supersonic version at build time
+ARG SUPERSONIC_VERSION
+
+# Install necessary packages, including Postgres client
+RUN apt-get install -y postgresql-client
+
+RUN rm /usr/src/app/supersonic-standalone-latest
+
+# Copy the supersonic standalone zip file into the container
+COPY assembly/build/supersonic-standalone-${SUPERSONIC_VERSION}.zip .
+
+# Unzip the supersonic standalone zip
+RUN unzip supersonic-standalone-${SUPERSONIC_VERSION}.zip && \
+    rm supersonic-standalone-${SUPERSONIC_VERSION}.zip
+
+# Create a symbolic link to the supersonic installation directory
+RUN ln -s /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION} /usr/src/app/supersonic-standalone-latest
+
+# Set the working directory to the supersonic installation directory
+WORKDIR /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION}
+
+# Expose the default port
+EXPOSE 9080
+# Command to run the supersonic daemon
+RUN chmod +x bin/supersonic-daemon.sh
+CMD ["bash", "-c", "bin/supersonic-daemon.sh restart standalone docker && tail -f /dev/null"]
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -1,51 +1,28 @@
 services:
-  chroma:
-    image: chromadb/chroma:0.5.3
+  postgres:
+    image: pgvector/pgvector:pg17
    privileged: true
-    container_name: supersonic_chroma
-    ports:
-      - "8000:8000"
-    volumes:
-      - chroma_data:/chroma
-    networks:
-      - supersonic_network
-    dns:
-      - 114.114.114.114
-      - 8.8.8.8
-      - 8.8.4.4
-    healthcheck:
-      test: ["CMD", "curl", "http://0.0.0.0:8000"]
-      interval: 10s
-      timeout: 5s
-      retries: 10
-
-  mysql:
-    image: mysql:8.0
-    privileged: true
-    container_name: supersonic_mysql
+    container_name: supersonic_postgres
    environment:
      LANG: 'C.UTF-8'  # 设置环境变量
-      MYSQL_ROOT_PASSWORD: root_password
-      MYSQL_DATABASE: supersonic_db
-      MYSQL_USER: supersonic_user
-      MYSQL_PASSWORD: supersonic_password
+      POSTGRES_ROOT_PASSWORD: root_password
+      POSTGRES_DATABASE: postgres
+      POSTGRES_USER: supersonic_user
+      POSTGRES_PASSWORD: supersonic_password
    ports:
-      - "13306:3306"
+      - "15432:5432"
    volumes:
-      - mysql_data:/var/lib/mysql
+      - postgres_data:/var/lib/postgresql
    networks:
      - supersonic_network
    dns:
      - 114.114.114.114
      - 8.8.8.8
      - 8.8.4.4
-    depends_on:
-      chroma:
-        condition: service_healthy
    healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
-      interval: 10s
-      timeout: 5s
+      test: ["CMD-SHELL", "sh -c 'pg_isready -U supersonic_user -d postgres'"]
+      interval: 30s
+      timeout: 10s
      retries: 5

  db_init:
@@ -53,22 +30,21 @@ services:
    privileged: true
    container_name: supersonic_db_init
    depends_on:
-      mysql:
+      postgres:
        condition: service_healthy
    networks:
      - supersonic_network
    command: >
      sh -c "
-      sleep 15 &&
-      if ! mysql -h supersonic_mysql -usupersonic_user -psupersonic_password -e 'use supersonic_db; show tables;' | grep -q 's2_database'; then
-        mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-mysql.sql &&
-        mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-mysql-demo.sql &&
-        mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-mysql.sql &&
-        mysql -h supersonic_mysql -usupersonic_user -psupersonic_password supersonic_db < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-mysql-demo.sql
-      else
-        echo 'Database already initialized.'
-      fi
-      "
+        if ! PGPASSWORD=supersonic_password psql -h supersonic_postgres -U supersonic_user -d postgres -c 'select * from s2_database limit 1' > /dev/null;
+        then
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-postgres.sql
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/schema-postgres-demo.sql
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-postgres.sql
+          PGPASSWORD=supersonic_password psql -hsupersonic_postgres -U supersonic_user -d postgres < /usr/src/app/supersonic-standalone-${SUPERSONIC_VERSION:-latest}/conf/db/data-postgres-demo.sql 
+        else
+          echo 'Database already initialized.'
+        fi"
    dns:
      - 114.114.114.114
      - 8.8.8.8
@@ -79,17 +55,14 @@ services:
    privileged: true
    container_name: supersonic_standalone
    environment:
-      DB_HOST: supersonic_mysql
-      DB_NAME: supersonic_db
+      DB_HOST: supersonic_postgres
+      DB_NAME: postgres
      DB_USERNAME: supersonic_user
      DB_PASSWORD: supersonic_password
-      CHROMA_HOST: supersonic_chroma
    ports:
      - "9080:9080"
    depends_on:
-      chroma:
-        condition: service_healthy
-      mysql:
+      postgres:
        condition: service_healthy
      db_init:
        condition: service_completed_successfully
@@ -112,8 +85,7 @@ services:
      #          propagation: rprivate
      #          create_host_path: true
 volumes:
-  mysql_data:
-  chroma_data:
+  postgres_data:
  supersonic_data:

 networks:
--- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
+++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
@@ -18,7 +18,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;

 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -85,15 +84,4 @@ public class ModelYamlManager {
        return identifyYamlTpl;
    }

-    private static void addInterCntMetric(String datasourceEnName, ModelDetail datasourceDetail) {
-        Measure measure = new Measure();
-        measure.setExpr("1");
-        if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) {
-            measure.setExpr(datasourceDetail.getIdentifiers().get(0).getBizName());
-        }
-        measure.setAgg("count");
-        measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
-        measure.setIsCreateMetric(1);
-        datasourceDetail.getMeasures().add(measure);
-    }
 }
--- a/launchers/standalone/src/main/resources/application-docker.yaml
+++ b/launchers/standalone/src/main/resources/application-docker.yaml
@@ -0,0 +1,26 @@
+spring:
+  datasource:
+    url: jdbc:postgresql://${DB_HOST}:${DB_PORT:5432}/${DB_NAME}?stringtype=unspecified
+    username: ${DB_USERNAME}
+    password: ${DB_PASSWORD}
+    driver-class-name: org.postgresql.Driver
+  sql:
+    init:
+      enabled: false
+      mode: always
+      username: ${DB_USERNAME}
+      password: ${DB_PASSWORD}
+      schema-locations: classpath:db/schema-postgres.sql,classpath:db/schema-postgres-demo.sql
+      data-locations: classpath:db/data-postgres.sql,classpath:db/data-postgres-demo.sql
+
+s2:
+  embedding:
+    store:
+      provider: PGVECTOR
+      base:
+        url: ${DB_HOST}
+      port: ${DB_PORT:5432}
+      databaseName: ${DB_NAME}
+      user: ${DB_USERNAME}
+      password: ${DB_PASSWORD}
+      dimension: 512
--- a/launchers/standalone/src/main/resources/db/data-postgres-demo.sql
+++ b/launchers/standalone/src/main/resources/db/data-postgres-demo.sql
@@ -1,10 +1,10 @@
 -- S2VisitsDemo
-MERGE into s2_user_department (user_name, department) values ('jack','HR');
-MERGE into s2_user_department (user_name, department) values ('tom','sales');
-MERGE into s2_user_department (user_name, department) values ('lucy','marketing');
-MERGE into s2_user_department (user_name, department) values ('john','strategy');
-MERGE into s2_user_department (user_name, department) values ('alice','sales');
-MERGE into s2_user_department (user_name, department) values ('dean','marketing');
+INSERT into s2_user_department (user_name, department) values ('jack','HR');
+INSERT into s2_user_department (user_name, department) values ('tom','sales');
+INSERT into s2_user_department (user_name, department) values ('lucy','marketing');
+INSERT into s2_user_department (user_name, department) values ('john','strategy');
+INSERT into s2_user_department (user_name, department) values ('alice','sales');
+INSERT into s2_user_department (user_name, department) values ('dean','marketing');

 INSERT INTO s2_pv_uv_statis (imp_date, user_name, page) VALUES (CURRENT_DATE, 'lucy', 'p1');
 INSERT INTO s2_pv_uv_statis (imp_date, user_name, page) VALUES (CURRENT_DATE, 'jack', 'p1');
@@ -1016,43 +1016,43 @@ INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (
 INSERT INTO s2_stay_time_statis (imp_date, user_name, stay_hours, page) VALUES (CURRENT_DATE - INTERVAL '8 DAY', 'lucy', '0.039935860913407284', 'p2');

 -- S2ArtistDemo
-MERGE INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
+INSERT INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
 VALUES ('周杰伦', '港台', '青花瓷', '国风', 1000000, 1000000, 1000000);
-MERGE INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
+INSERT INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
 VALUES ('陈奕迅', '港台', '爱情转移', '流行', 1000000, 1000000, 1000000);
-MERGE INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
+INSERT INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
 VALUES ('林俊杰', '港台', '美人鱼', '流行', 1000000, 1000000, 1000000);
-MERGE INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
+INSERT INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
 VALUES ('张碧晨', '内地', '光的方向', '流行', 1000000, 1000000, 1000000);
-MERGE INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
+INSERT INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
 VALUES ('程响', '内地', '人间烟火', '国风', 1000000, 1000000, 1000000);
-MERGE INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
+INSERT INTO singer (singer_name, act_area, song_name, genre, js_play_cnt, down_cnt, favor_cnt)
 VALUES ('Taylor Swift', '欧美', 'Love Story', '流行', 1000000, 1000000, 1000000);

-MERGE INTO genre(g_name,rating,most_popular_in) VALUES ('tagore',8,'孟加拉国');
-MERGE INTO genre(g_name,rating,most_popular_in) VALUES ('nazrul',7,'孟加拉国');
-MERGE INTO genre(g_name,rating,most_popular_in) VALUES ('民间',9,'锡尔赫特、吉大港、库斯蒂亚');
-MERGE INTO genre(g_name,rating,most_popular_in) VALUES ('现代',8,'孟加拉国');
-MERGE INTO genre(g_name,rating,most_popular_in) VALUES ('蓝调',7,'加拿大');
-MERGE INTO genre(g_name,rating,most_popular_in) VALUES ('流行',9,'美国');
+INSERT INTO genre(g_name,rating,most_popular_in) VALUES ('tagore',8,'孟加拉国');
+INSERT INTO genre(g_name,rating,most_popular_in) VALUES ('nazrul',7,'孟加拉国');
+INSERT INTO genre(g_name,rating,most_popular_in) VALUES ('民间',9,'锡尔赫特、吉大港、库斯蒂亚');
+INSERT INTO genre(g_name,rating,most_popular_in) VALUES ('现代',8,'孟加拉国');
+INSERT INTO genre(g_name,rating,most_popular_in) VALUES ('蓝调',7,'加拿大');
+INSERT INTO genre(g_name,rating,most_popular_in) VALUES ('流行',9,'美国');

 -- S2CompanyDemo
-MERGE into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_131','微软','西雅图','1975','盖茨','纳德拉',102300000000,210000);
-MERGE into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_132','特斯拉','加州','2003','艾伯哈德','马斯克',376800000000,140473);
-MERGE into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_133','谷歌','加州','1998','拉里佩奇','劈柴',321600000000,182503);
-MERGE into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_134','亚马逊','加州','1994','贝索斯','贝索斯',28800000000,950000);
-MERGE into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_135','英伟达','杭州','1993','黄仁勋','黄仁勋',67500000000,29000);
+INSERT into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_131','微软','西雅图','1975','盖茨','纳德拉',102300000000,210000);
+INSERT into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_132','特斯拉','加州','2003','艾伯哈德','马斯克',376800000000,140473);
+INSERT into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_133','谷歌','加州','1998','拉里佩奇','劈柴',321600000000,182503);
+INSERT into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_134','亚马逊','加州','1994','贝索斯','贝索斯',28800000000,950000);
+INSERT into company(company_id,company_name,headquarter_address,company_established_time,founder,ceo,annual_turnover,employee_count) VALUES ('item_enterprise_13_135','英伟达','杭州','1993','黄仁勋','黄仁勋',67500000000,29000);

-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_131','Office','1990','item_enterprise_13_131','盖茨',50000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_132','Windows','1991','item_enterprise_13_131','盖茨',50000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_133','Model 3','2017','item_enterprise_13_132','马斯克',100000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_134','Model Y','2020','item_enterprise_13_132','马斯克',100000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_135','Google','2003','item_enterprise_13_133','拉里佩奇',50000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_136','Android','2007','item_enterprise_13_133','拉里佩奇',50000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_137','aws','2004','item_enterprise_13_134','贝索斯',100000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_138','kindle','2007','item_enterprise_13_134','贝索斯',100000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_139','H100','2022','item_enterprise_13_135','黄仁勋',100000000);
-MERGE into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_140','A100','2021','item_enterprise_13_135','黄仁勋',100000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_131','Office','1990','item_enterprise_13_131','盖茨',50000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_132','Windows','1991','item_enterprise_13_131','盖茨',50000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_133','Model 3','2017','item_enterprise_13_132','马斯克',100000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_134','Model Y','2020','item_enterprise_13_132','马斯克',100000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_135','Google','2003','item_enterprise_13_133','拉里佩奇',50000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_136','Android','2007','item_enterprise_13_133','拉里佩奇',50000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_137','aws','2004','item_enterprise_13_134','贝索斯',100000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_138','kindle','2007','item_enterprise_13_134','贝索斯',100000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_139','H100','2022','item_enterprise_13_135','黄仁勋',100000000);
+INSERT into brand(brand_id,brand_name,brand_established_time,company_id,legal_representative,registered_capital) VALUES ('item_brand_13_140','A100','2021','item_enterprise_13_135','黄仁勋',100000000);

 insert into brand_revenue(year_time,brand_id,revenue,profit,revenue_growth_year_on_year,profit_growth_year_on_year) VALUES ('2023','item_brand_13_131',12100000000, 2100000000,10,10);
 insert into brand_revenue(year_time,brand_id,revenue,profit,revenue_growth_year_on_year,profit_growth_year_on_year) VALUES ('2023','item_brand_13_132',12200000000, 2200000000,20,20);