From 9337d7c91fa8eaef153050198171919d86b6afff Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 25 Dec 2020 21:54:46 -0500 Subject: [PATCH] e2e test workflow (#885) --- .github/workflows/e2etest.yml | 335 ++++++++++++++++++ .../workflows/runner-basic-e2e-test-case.yml | 31 ++ 2 files changed, 366 insertions(+) create mode 100644 .github/workflows/e2etest.yml create mode 100644 .github/workflows/runner-basic-e2e-test-case.yml diff --git a/.github/workflows/e2etest.yml b/.github/workflows/e2etest.yml new file mode 100644 index 000000000..826b54a6f --- /dev/null +++ b/.github/workflows/e2etest.yml @@ -0,0 +1,335 @@ +name: Runner E2E Test + +on: + workflow_dispatch: + push: + branches: + - main + - releases/* + +jobs: + init: + name: Initialize workflow ☕ + runs-on: ubuntu-latest + outputs: + unique_runner_label: ${{steps.generator.outputs.runner_label}} + steps: + - name: Delete all runners + uses: actions/github-script@v3 + with: + debug: true + script: | + var runnersResp = await github.actions.listSelfHostedRunnersForRepo({ + owner: 'actions', + repo: 'runner', + per_page: '100' + }); + for(var i=0; i setTimeout(resolve, ms)); } + async function dispatchWorkflow(runner) { + await github.actions.createWorkflowDispatch({ + owner: 'actions', + repo: 'runner', + workflow_id: 'runner-basic-e2e-test-case.yml', + ref: 'main', + inputs: {target_runner: runner} + }); + } + var runWin64 = false, runLinux64 = false, runOsx64 = false, runLinuxARM64 = false; + while (true) { + core.info(`------------- Waiting for runners to be configured --------------`) + await sleep(10000); + var runnersResp = await github.actions.listSelfHostedRunnersForRepo({owner: 'actions', repo: 'runner', per_page: '100'}); + for (var i = 0; i < runnersResp.data.total_count; i++) { + core.debug(JSON.stringify(runnersResp.data.runners[i])) + var labels = runnersResp.data.runners[i].labels; + for (var j = 0; j < labels.length; j++) { + core.debug(`Comparing: ${labels[j].name} to win-x64/linux-x64/osx-x64/linux-arm64-${{ needs.init.outputs.unique_runner_label }}`) + if (labels[j].name == 'win-x64-${{needs.init.outputs.unique_runner_label}}' && runWin64 == false) { + core.info(`------------------- Windows runner is configured, queue Windows Run -------------------------`) + runWin64 = true; + await dispatchWorkflow('win-x64-${{needs.init.outputs.unique_runner_label}}'); + break; + } else if (labels[j].name == 'linux-x64-${{needs.init.outputs.unique_runner_label}}' && runLinux64 == false) { + core.info(`------------------- Linux runner is configured, queue Linux Run -------------------------`) + runLinux64 = true; + await dispatchWorkflow('linux-x64-${{needs.init.outputs.unique_runner_label}}'); + break; + } else if (labels[j].name == 'osx-x64-${{needs.init.outputs.unique_runner_label}}' && runOsx64 == false) { + core.info(`------------------- macOS runner is configured, queue macOS Run -------------------------`) + runOsx64 = true; + await dispatchWorkflow('osx-x64-${{needs.init.outputs.unique_runner_label}}'); + break; + } else if (labels[j].name == 'linux-arm64-${{needs.init.outputs.unique_runner_label}}' && runLinuxARM64 == false) { + core.info(`------------------- Linux ARM64 runner is configured, queue Linux ARM64 Run-------------------------`) + runLinuxARM64 = true; + await dispatchWorkflow('linux-arm64-${{needs.init.outputs.unique_runner_label}}'); + break; + } + } + } + if (runWin64 && runLinux64 && runOsx64 && runLinuxARM64) { + core.info(`--------------------- ALL runner are running jobs --------------------------`) + break; + } else { + core.info(`---------- Windows running: ${runWin64} -- Linux running: ${runLinux64} -- macOS running: ${runOsx64} -- Linux ARM64 running: ${runLinuxARM64} -----------`) + } + } + github-token: ${{secrets.PAT}} + + LinuxE2E: + needs: [build, init] + runs-on: ubuntu-latest + steps: + - name: Download Runner + uses: actions/download-artifact@v2 + with: + name: runner-package-linux-x64 + - name: Unzip Runner Package + run: | + tar -xzf *.tar.gz + - name: Configure Runner + env: + unique_runner_name: linux-x64-${{needs.init.outputs.unique_runner_label}} + run: | + ./config.sh --url ${{github.event.repository.html_url}} --unattended --name $unique_runner_name --pat ${{secrets.PAT}} --labels $unique_runner_name --replace + - name: Start Runner and Wait for Job + timeout-minutes: 5 + run: | + ./run.sh --once + - name: Remove Runner + if: always() + continue-on-error: true + run: | + ./config.sh remove --pat ${{secrets.PAT}} + - name: Upload Runner Logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: linux_x64_logs + path: _diag + macOSE2E: + needs: [build, init] + runs-on: macos-latest + steps: + - name: Download Runner + uses: actions/download-artifact@v2 + with: + name: runner-package-osx-x64 + - name: Unzip Runner Package + run: | + tar -xzf *.tar.gz + - name: Configure Runner + env: + unique_runner_name: osx-x64-${{needs.init.outputs.unique_runner_label}} + run: | + ./config.sh --url ${{github.event.repository.html_url}} --unattended --name $unique_runner_name --pat ${{secrets.PAT}} --labels $unique_runner_name --replace + - name: Start Runner and Wait for Job + timeout-minutes: 5 + run: | + ./run.sh --once + - name: Remove Runner + if: always() + continue-on-error: true + run: | + ./config.sh remove --pat ${{secrets.PAT}} + - name: Upload Runner Logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: osx_x64_logs + path: _diag + + ARM64E2E: + needs: [build, init] + runs-on: ubuntu-latest + steps: + - name: Download Runner + uses: actions/download-artifact@v2 + with: + name: runner-package-linux-arm64 + - name: Unzip Runner Package + run: | + tar -xzf *.tar.gz + - name: Prepare QEMU + run: | + docker run --rm --privileged multiarch/qemu-user-static:register --reset + - name: Configure Runner + uses: docker://multiarch/ubuntu-core:arm64-bionic + with: + args: 'bash -c "apt-get update && apt-get install -y curl && ./bin/installdependencies.sh && ./config.sh --unattended --name $unique_runner_name --url ${{github.event.repository.html_url}} --pat ${{secrets.PAT}} --labels $unique_runner_name --replace"' + env: + RUNNER_ALLOW_RUNASROOT: 1 + unique_runner_name: linux-arm64-${{needs.init.outputs.unique_runner_label}} + + - name: Start Runner and Wait for Job + timeout-minutes: 5 + uses: docker://multiarch/ubuntu-core:arm64-bionic + with: + args: 'bash -c "apt-get update && apt-get install -y curl git && ./bin/installdependencies.sh && ./run.sh --once"' + env: + RUNNER_ALLOW_RUNASROOT: 1 + + - name: Remove Runner + if: always() + continue-on-error: true + uses: docker://multiarch/ubuntu-core:arm64-bionic + with: + args: 'bash -c "apt-get update && apt-get install -y curl && ./bin/installdependencies.sh && ./config.sh remove --pat ${{secrets.PAT}}"' + env: + RUNNER_ALLOW_RUNASROOT: 1 + + - name: Upload Runner Logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: linux_arm64_logs + path: _diag + + WindowsE2E: + needs: [build, init] + runs-on: windows-latest + steps: + - name: Download Runner + uses: actions/download-artifact@v2 + with: + name: runner-package-win-x64 + - name: Unzip Runner Package + run: | + Get-ChildItem *.zip | Expand-Archive -DestinationPath $PWD + - name: Configure Runner + shell: cmd + run: | + config.cmd --unattended --url ${{github.event.repository.html_url}} --name %unique_runner_name% --pat ${{secrets.PAT}} --labels %unique_runner_name% --replace + env: + unique_runner_name: win-x64-${{needs.init.outputs.unique_runner_label}} + + - name: Start Runner and Wait for Job + shell: cmd + timeout-minutes: 5 + run: | + run.cmd --once + - name: Remove Runner + shell: cmd + if: always() + continue-on-error: true + run: | + config.cmd remove --pat ${{secrets.PAT}} + - name: Upload Runner Logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: win_x64_logs + path: _diag + + check: + name: Check runner logs 🕵️‍♂️ + needs: [WindowsE2E, LinuxE2E, macOSE2E, ARM64E2E] + runs-on: ubuntu-latest + steps: + - name: Download Linux Runner Logs + uses: actions/download-artifact@v2 + with: + name: linux_x64_logs + path: linux_x64_logs + - name: Download macOS Runner Logs + uses: actions/download-artifact@v2 + with: + name: osx_x64_logs + path: osx_x64_logs + - name: Download Linux ARM64 Runner Logs + uses: actions/download-artifact@v2 + with: + name: linux_arm64_logs + path: linux_arm64_logs + - name: Download Windows Runner Logs + uses: actions/download-artifact@v2 + with: + name: win_x64_logs + path: win_x64_logs + - name: Check Runner Logs + run: | + function failed() + { + local error=${1:-Undefined error} + echo "Failed: $error" >&2 + exit 1 + } + grep -R "Job runner-basics completed with result: Succeeded" ./win_x64_logs || failed "Windows Runner fail to run the job, please check logs" + grep -R "Job runner-basics completed with result: Succeeded" ./linux_x64_logs || failed "Linux Runner fail to run the job, please check logs" + grep -R "Job runner-basics completed with result: Succeeded" ./osx_x64_logs || failed "macOS Runner fail to run the job, please check logs" + grep -R "Job runner-basics completed with result: Succeeded" ./linux_arm64_logs || failed "Linux ARM64 Runner fail to run the job, please check logs" diff --git a/.github/workflows/runner-basic-e2e-test-case.yml b/.github/workflows/runner-basic-e2e-test-case.yml new file mode 100644 index 000000000..55d437296 --- /dev/null +++ b/.github/workflows/runner-basic-e2e-test-case.yml @@ -0,0 +1,31 @@ +name: Runner Basics Test Case + +on: + workflow_dispatch: + inputs: + target_runner: + description: 'Self-hosted runner will run the job' + required: true + +jobs: + test: + runs-on: + - self-hosted + - ${{github.event.inputs.target_runner}} + + name: Runner Basic Test 🛠 + steps: + - uses: actions/checkout@v2 + - name: Run a one-line script + run: echo Hello, world! + - name: Run a multi-line script + shell: bash + run: | + printenv|sort + cat $GITHUB_EVENT_PATH + - name: Validate GitHub Context + shell: bash + run: | + declare -a context_vars=("GITHUB_ACTION" "GITHUB_ACTIONS" "GITHUB_REPOSITORY" "GITHUB_WORKSPACE" "GITHUB_SHA" "GITHUB_RUN_ID" "GITHUB_RUN_NUMBER") + for var in ${context_vars[@]}; + do [ -z "${!var}" ] && echo "##[error]$var not found" && exit 1 || echo "$var: ${!var}"; done \ No newline at end of file