diff --git a/.gitignore b/.gitignore index 55c7381fb..8d4de8975 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ assembly/runtime/* **/.flattened-pom.xml chm_db/ __pycache__/ -/dict \ No newline at end of file +/dict +assembly/build/*-SNAPSHOT \ No newline at end of file diff --git a/assembly/bin/supersonic-build.bat b/assembly/bin/supersonic-build.bat index 1f64e5344..c11d0fc47 100644 --- a/assembly/bin/supersonic-build.bat +++ b/assembly/bin/supersonic-build.bat @@ -1,72 +1,98 @@ @echo off -setlocal +setlocal enabledelayedexpansion chcp 65001 -set "sbinDir=%~dp0" -set "baseDir=%~dp0.." -set "buildDir=%baseDir%\build" -set "runtimeDir=%baseDir%\..\runtime" -set "pip_path=pip3" + +call supersonic-common.bat %* + set "service=%~1" - -rem 1. build backend java modules -del /q "%buildDir%\*.tar.gz" 2>NUL -call mvn -f "%baseDir%\..\pom.xml" clean package -DskipTests - -IF ERRORLEVEL 1 ( - ECHO Failed to build backend Java modules. - EXIT /B 1 +cd %projectDir% +if "%service%"=="" ( + set service=%standalone_service% ) -rem 2. move package to build -echo f|xcopy "%baseDir%\..\launchers\standalone\target\*.tar.gz" "%buildDir%\supersonic-standalone.tar.gz" +call mvn help:evaluate -Dexpression=project.version > temp.txt +for /f "delims=" %%i in (temp.txt) do ( + set line=%%i + if not "!line:~0,1!"=="[" ( + set MVN_VERSION=!line! + ) +) +del temp.txt +cd %baseDir% -rem 3. build frontend webapp -cd "%baseDir%\..\webapp" -call start-fe-prod.bat -copy /y "%baseDir%\..\webapp\supersonic-webapp.tar.gz" "%buildDir%\" -IF ERRORLEVEL 1 ( - ECHO Failed to build frontend webapp. - EXIT /B 1 +if "%service%"=="%pyllm_service%" ( + echo start installing python modules required by supersonic-pyllm: %pip_path% + %pip_path% install -r %projectDir%\headless\python\requirements.txt" + echo install python modules success + goto :EOF +) else if "%service%"=="webapp" ( + call :buildWebapp + tar xvf supersonic-webapp.tar.gz + move /y supersonic-webapp webapp + move /y webapp %projectDir%\launchers\%STANDALONE_SERVICE%\target\classes + goto :EOF +) else ( + call :buildJavaService + call :buildWebapp + call :packageRelease + goto :EOF ) -rem 4. copy webapp to java classpath -cd "%buildDir%" -tar -zxvf supersonic-webapp.tar.gz -move supersonic-webapp webapp -move webapp ..\..\launchers\standalone\target\classes -rem 5. build backend python modules -if "%service%"=="pyllm" ( - echo "start installing python modules with pip: ${pip_path}" - set requirementPath="%baseDir%/../headless/python/requirements.txt" - %pip_path% install -r %requirementPath% - echo "install python modules success" -) +:buildJavaService + set "model_name=%service%" + echo "starting building supersonic-%model_name% service" + call mvn -f %projectDir%\launchers\%model_name% clean package -DskipTests + IF ERRORLEVEL 1 ( + ECHO Failed to build backend Java modules. + EXIT /B 1 + ) + copy /y %projectDir%\launchers\%model_name%\target\*.tar.gz %buildDir%\ + echo "finished building supersonic-%model_name% service" + goto :EOF -call :BUILD_RUNTIME -:BUILD_RUNTIME - rem 6. reset runtime - IF EXIST "%runtimeDir%" ( - echo begin to delete dir : %runtimeDir% - rd /s /q "%runtimeDir%" - ) ELSE ( - echo %runtimeDir% does not exist, create directly - ) - mkdir "%runtimeDir%" - tar -zxvf "%buildDir%\supersonic-standalone.tar.gz" -C "%runtimeDir%" - for /d %%f in ("%runtimeDir%\launchers-standalone-*") do ( - move "%%f" "%runtimeDir%\supersonic-standalone" - ) +:buildWebapp + echo "starting building supersonic webapp" + cd %projectDir%\webapp + call start-fe-prod.bat + copy /y supersonic-webapp.tar.gz %buildDir%\ + rem check build result + IF ERRORLEVEL 1 ( + ECHO Failed to build frontend webapp. + EXIT /B 1 + ) + echo "finished building supersonic webapp" + goto :EOF - rem 7. copy webapp to runtime - tar -zxvf "%buildDir%\supersonic-webapp.tar.gz" -C "%buildDir%" - if not exist "%runtimeDir%\supersonic-standalone\webapp" mkdir "%runtimeDir%\supersonic-standalone\webapp" - xcopy /s /e /h /y "%buildDir%\supersonic-webapp\*" "%runtimeDir%\supersonic-standalone\webapp" - if not exist "%runtimeDir%\supersonic-standalone\conf\webapp" mkdir "%runtimeDir%\supersonic-standalone\conf\webapp" - xcopy /s /e /h /y "%runtimeDir%\supersonic-standalone\webapp\*" "%runtimeDir%\supersonic-standalone\conf\webapp" - rd /s /q "%buildDir%\supersonic-webapp" + +:packageRelease + set "model_name=%service%" + set "release_dir=supersonic-%model_name%-%MVN_VERSION%" + set "service_name=launchers-%model_name%-%MVN_VERSION%" + echo "starting packaging supersonic release" + cd %buildDir% + if exist %release_dir% rmdir /s /q %release_dir% + if exist %release_dir%.zip del %release_dir%.zip + mkdir %release_dir% + rem package webapp + tar xvf supersonic-webapp.tar.gz + move /y supersonic-webapp webapp + echo {"env": ""} > webapp\supersonic.config.json + move /y webapp %release_dir% + rem package java service + tar xvf %service_name%-bin.tar.gz + for /d %%D in ("%service_name%\*") do ( + move "%%D" "%release_dir%" + ) + rem generate zip file + powershell Compress-Archive -Path %release_dir% -DestinationPath %release_dir%.zip + del %service_name%-bin.tar.gz + del supersonic-webapp.tar.gz + rmdir /s /q %service_name% + echo "finished packaging supersonic release" + goto :EOF endlocal \ No newline at end of file diff --git a/assembly/bin/supersonic-common.bat b/assembly/bin/supersonic-common.bat new file mode 100644 index 000000000..5d1e218d8 --- /dev/null +++ b/assembly/bin/supersonic-common.bat @@ -0,0 +1,9 @@ +set "sbinDir=%~dp0" +set "baseDir=%~dp0.." +set "buildDir=%baseDir%\build" +set "main_class=com.tencent.supersonic.StandaloneLauncher" +set "python_path=python" +set "pip_path=pip3" +set "standalone_service=standalone" +set "pyllm_service=pyllm" +set "projectDir=%baseDir%\.." diff --git a/assembly/bin/supersonic-daemon.bat b/assembly/bin/supersonic-daemon.bat index c04ce86fc..7490db55d 100644 --- a/assembly/bin/supersonic-daemon.bat +++ b/assembly/bin/supersonic-daemon.bat @@ -1,118 +1,101 @@ @echo off setlocal chcp 65001 -set "sbinDir=%~dp0" -set "baseDir=%~dp0.." -set "runtimeDir=%baseDir%\..\runtime" -set "buildDir=%baseDir%\build" -set "main_class=com.tencent.supersonic.StandaloneLauncher" -set "python_path=python" -set "pip_path=pip3" -set "standalone_service=standalone" -set "pyllm_service=pyllm" -set "javaRunDir=%runtimeDir%\supersonic-standalone" -set "pythonRunDir=%runtimeDir%\supersonic-standalone\pyllm" +call supersonic-common.bat %* set "command=%~1" set "service=%~2" - if "%service%"=="" ( set "service=%standalone_service%" ) - +set "model_name=%service%" IF "%service%"=="pyllm" ( - SET "llmProxy=PythonLLMProxy" + set "llmProxy=PythonLLMProxy" + set "model_name=%standalone_service%" ) -call :BUILD_RUNTIME +cd %baseDir% if "%command%"=="restart" ( - call :STOP - call :START + call :stop + call :start goto :EOF ) else if "%command%"=="start" ( - call :START + call :start goto :EOF ) else if "%command%"=="stop" ( - call :STOP - goto :EOF + call :stop + goto :EOF ) else if "%command%"=="reload" ( - call :RELOAD_EXAMPLE - goto :EOF + call :reloadExamples + goto :EOF ) else ( echo "Use command {start|stop|restart} to run." goto :EOF ) -:START - if "%service%"=="%pyllm_service%" ( - call :START_PYTHON - call :START_JAVA + +: start + if "%service%"=="%pyllm_service%" ( + call :runPythonService + call :runJavaService goto :EOF - ) - call :START_JAVA - goto :EOF - -:STOP - call :STOP_PYTHON - call :STOP_JAVA - goto :EOF - -:START_PYTHON - echo 'python service starting, see logs in pyllm/pyllm.log' - cd "%pythonRunDir%" - start /B %python_path% supersonic_pyllm.py > %pythonRunDir%\pyllm.log 2>&1 - timeout /t 10 >nul - echo 'python service started' + ) + call :runJavaService goto :EOF -:START_JAVA - echo 'java service starting, see logs in logs/' - cd "%javaRunDir%" - if not exist "%runtimeDir%\supersonic-standalone\logs" mkdir "%runtimeDir%\supersonic-standalone\logs" - set "libDir=%runtimeDir%\supersonic-standalone\lib" - set "confDir=%runtimeDir%\supersonic-standalone\conf" - set "webDir=%runtimeDir%\supersonic-standalone\webapp" - set "classpath=%confDir%;%webDir%;%libDir%\*" + +: stop + call :stopPythonService + call :stopJavaService + goto :EOF + + +: reloadExamples + set "pythonRunDir=%baseDir%\pyllm" + cd "%pythonRunDir%\sql" + start %python_path% examples_reload_run.py + goto :EOF + + +: runJavaService + echo 'java service starting, see logs in logs/' + set "libDir=%baseDir%\lib" + set "confDir=%baseDir%\conf" + set "webDir=%baseDir%\webapp" + set "logDir=%baseDir%\logs" + set "classpath=%baseDir%;%webDir%;%libDir%\*;%confDir%" set "java-command=-Dfile.encoding=UTF-8 -Duser.language=Zh -Duser.region=CN -Duser.timezone=GMT+08 -Xms1024m -Xmx2048m -cp %CLASSPATH% %MAIN_CLASS%" + if not exist %logDir% mkdir %logDir% start /B java %java-command% >nul 2>&1 timeout /t 10 >nul echo 'java service started' goto :EOF -:STOP_PYTHON + +: runPythonService + echo 'python service starting, see logs in pyllm\pyllm.log' + set "pythonRunDir=%baseDir%\pyllm" + start /B %python_path% %pythonRunDir%\supersonic_pyllm.py > %pythonRunDir%\pyllm.log 2>&1 + timeout /t 10 >nul + echo 'python service started' + goto :EOF + + +: stopPythonService for /f "tokens=2" %%i in ('tasklist ^| findstr /i "python"') do ( taskkill /PID %%i /F echo "python service (PID = %%i) is killed." ) goto :EOF -:STOP_JAVA + +: stopJavaService for /f "tokens=2" %%i in ('tasklist ^| findstr /i "java"') do ( taskkill /PID %%i /F echo "java service (PID = %%i) is killed." ) goto :EOF -:RELOAD_EXAMPLE - cd "%runtimeDir%\supersonic-standalone\pyllm\sql" - start %python_path% examples_reload_run.py - goto :EOF - -:BUILD_RUNTIME - rem 6. reset runtime - if exist "%runtimeDir%" goto :EOF - mkdir "%runtimeDir%" - tar -zxvf "%buildDir%\supersonic-standalone.tar.gz" -C "%runtimeDir%" - for /d %%f in ("%runtimeDir%\launchers-standalone-*") do ( - move "%%f" "%runtimeDir%\supersonic-standalone" - ) - - rem 7. copy webapp to runtime - tar -zxvf "%buildDir%\supersonic-webapp.tar.gz" -C "%buildDir%" - if not exist "%runtimeDir%\supersonic-standalone\webapp" mkdir "%runtimeDir%\supersonic-standalone\webapp" - xcopy /s /e /h /y "%buildDir%\supersonic-webapp\*" "%runtimeDir%\supersonic-standalone\webapp" - if not exist "%runtimeDir%\supersonic-standalone\conf\webapp" mkdir "%runtimeDir%\supersonic-standalone\conf\webapp" - xcopy /s /e /h /y "%runtimeDir%\supersonic-standalone\webapp\*" "%runtimeDir%\supersonic-standalone\conf\webapp" - rd /s /q "%buildDir%\supersonic-webapp" \ No newline at end of file +endlocal \ No newline at end of file diff --git a/webapp/.gitignore b/webapp/.gitignore index 679cdeb79..33d65bce4 100644 --- a/webapp/.gitignore +++ b/webapp/.gitignore @@ -15,8 +15,8 @@ /supersonic-webapp +../assembly/build/supersonic-webapp.tar.gz supersonic-webapp.tar.gz - package-lock.json yarn.lock