e2e test workflow (#885)

.github/workflows/e2etest.yml

@@ -0,0 +1,335 @@
+name: Runner E2E Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - main
+    - releases/*
+
+jobs:
+  init:
+    name: Initialize workflow ☕
+    runs-on: ubuntu-latest
+    outputs:
+      unique_runner_label: ${{steps.generator.outputs.runner_label}}
+    steps:
+    - name: Delete all runners
+      uses: actions/github-script@v3
+      with:
+        debug: true
+        script: |
+          var runnersResp = await github.actions.listSelfHostedRunnersForRepo({
+              owner: 'actions',
+              repo: 'runner',
+              per_page: '100'
+          });
+          for(var i=0; i<runnersResp.data.total_count; i++){
+            core.debug(JSON.stringify(runnersResp.data.runners[i]))
+            await github.actions.deleteSelfHostedRunnerFromRepo({
+              owner: 'actions',
+              repo: 'runner',
+              runner_id: runnersResp.data.runners[i].id
+            });
+          }
+        github-token: ${{secrets.PAT}}
+    - name: Generate Unique Runner label
+      id: generator
+      run: |
+        label=$(openssl rand -hex 16)
+        echo ::set-output name=runner_label::$label
+
+  build:
+    name: Build runner packages 🏗 📦
+    strategy:
+      matrix:
+        runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, osx-x64 ]
+        include:
+        - runtime: linux-x64
+          os: ubuntu-latest
+          devScript: ./dev.sh
+
+        - runtime: linux-arm64
+          os: ubuntu-latest
+          devScript: ./dev.sh
+
+        - runtime: linux-arm
+          os: ubuntu-latest
+          devScript: ./dev.sh
+
+        - runtime: osx-x64
+          os: macOS-latest
+          devScript: ./dev.sh
+
+        - runtime: win-x64
+          os: windows-latest
+          devScript: ./dev
+
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v1
+
+    # Build runner layout
+    - name: Build & Layout Release
+      run: |
+        ${{ matrix.devScript }} layout Release ${{ matrix.runtime }}
+      working-directory: src
+    
+    # Create runner package tar.gz/zip
+    - name: Package Release
+      run: |
+         ${{ matrix.devScript }} package Release ${{ matrix.runtime }}
+      working-directory: src
+
+    # Upload runner package tar.gz/zip as artifact
+    - name: Publish Artifact
+      uses: actions/upload-artifact@v1
+      with:
+        name: runner-package-${{ matrix.runtime }}
+        path: _package
+
+  dispatch_workflow:
+    name: Dispatch workflow to runners 🚨
+    needs: [init, build]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Dispatch workflow
+      timeout-minutes: 10
+      uses: actions/github-script@v3
+      with:
+        debug: true
+        script: |
+          function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); }
+          async function dispatchWorkflow(runner) {
+            await github.actions.createWorkflowDispatch({
+              owner: 'actions',
+              repo: 'runner',
+              workflow_id: 'runner-basic-e2e-test-case.yml',
+              ref: 'main',
+              inputs: {target_runner: runner}
+            });
+          }
+          var runWin64 = false, runLinux64 = false, runOsx64 = false, runLinuxARM64 = false;
+          while (true) {
+            core.info(`------------- Waiting for runners to be configured --------------`)
+            await sleep(10000);
+            var runnersResp = await github.actions.listSelfHostedRunnersForRepo({owner: 'actions', repo: 'runner', per_page: '100'});
+            for (var i = 0; i < runnersResp.data.total_count; i++) {
+              core.debug(JSON.stringify(runnersResp.data.runners[i]))
+              var labels = runnersResp.data.runners[i].labels;
+              for (var j = 0; j < labels.length; j++) {
+                core.debug(`Comparing: ${labels[j].name} to win-x64/linux-x64/osx-x64/linux-arm64-${{ needs.init.outputs.unique_runner_label }}`)
+                if (labels[j].name == 'win-x64-${{needs.init.outputs.unique_runner_label}}' && runWin64 == false) {
+                  core.info(`------------------- Windows runner is configured, queue Windows Run -------------------------`)
+                  runWin64 = true;
+                  await dispatchWorkflow('win-x64-${{needs.init.outputs.unique_runner_label}}');
+                  break;
+                } else if (labels[j].name == 'linux-x64-${{needs.init.outputs.unique_runner_label}}' && runLinux64 == false) {
+                  core.info(`------------------- Linux runner is configured, queue Linux Run -------------------------`)
+                  runLinux64 = true;
+                  await dispatchWorkflow('linux-x64-${{needs.init.outputs.unique_runner_label}}');
+                  break;
+                } else if (labels[j].name == 'osx-x64-${{needs.init.outputs.unique_runner_label}}' && runOsx64 == false) {
+                  core.info(`------------------- macOS runner is configured, queue macOS Run -------------------------`)
+                  runOsx64 = true;
+                  await dispatchWorkflow('osx-x64-${{needs.init.outputs.unique_runner_label}}');
+                  break;
+                } else if (labels[j].name == 'linux-arm64-${{needs.init.outputs.unique_runner_label}}' && runLinuxARM64 == false) {
+                  core.info(`------------------- Linux ARM64 runner is configured, queue Linux ARM64 Run-------------------------`)
+                  runLinuxARM64 = true;
+                  await dispatchWorkflow('linux-arm64-${{needs.init.outputs.unique_runner_label}}');
+                  break;
+                }
+              }
+            }
+            if (runWin64 && runLinux64 && runOsx64 && runLinuxARM64) {
+              core.info(`--------------------- ALL runner are running jobs --------------------------`)
+              break;
+            } else {
+              core.info(`---------- Windows running: ${runWin64} -- Linux running: ${runLinux64} -- macOS running: ${runOsx64} -- Linux ARM64 running: ${runLinuxARM64} -----------`)
+            }
+          }
+        github-token: ${{secrets.PAT}}
+
+  LinuxE2E:
+    needs: [build, init]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Runner
+        uses: actions/download-artifact@v2
+        with:
+          name: runner-package-linux-x64
+      - name: Unzip Runner Package
+        run: |
+          tar -xzf *.tar.gz
+      - name: Configure Runner
+        env:
+          unique_runner_name: linux-x64-${{needs.init.outputs.unique_runner_label}}
+        run: |
+          ./config.sh --url ${{github.event.repository.html_url}} --unattended --name $unique_runner_name --pat ${{secrets.PAT}} --labels $unique_runner_name --replace
+      - name: Start Runner and Wait for Job
+        timeout-minutes: 5
+        run: |
+          ./run.sh --once
+      - name: Remove Runner
+        if: always()
+        continue-on-error: true
+        run: |
+          ./config.sh remove --pat ${{secrets.PAT}}
+      - name: Upload Runner Logs
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: linux_x64_logs
+          path: _diag
+  macOSE2E:
+    needs: [build, init]
+    runs-on: macos-latest
+    steps:
+      - name: Download Runner
+        uses: actions/download-artifact@v2
+        with:
+          name: runner-package-osx-x64
+      - name: Unzip Runner Package
+        run: |
+          tar -xzf *.tar.gz
+      - name: Configure Runner
+        env:
+          unique_runner_name: osx-x64-${{needs.init.outputs.unique_runner_label}}
+        run: |
+          ./config.sh --url ${{github.event.repository.html_url}} --unattended --name $unique_runner_name --pat ${{secrets.PAT}} --labels $unique_runner_name --replace
+      - name: Start Runner and Wait for Job
+        timeout-minutes: 5
+        run: |
+          ./run.sh --once
+      - name: Remove Runner
+        if: always()
+        continue-on-error: true
+        run: |
+          ./config.sh remove --pat ${{secrets.PAT}}
+      - name: Upload Runner Logs
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: osx_x64_logs
+          path: _diag
+
+  ARM64E2E:
+    needs: [build, init]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Runner
+        uses: actions/download-artifact@v2
+        with:
+          name: runner-package-linux-arm64
+      - name: Unzip Runner Package
+        run: |
+          tar -xzf *.tar.gz
+      - name: Prepare QEMU
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static:register --reset
+      - name: Configure Runner
+        uses: docker://multiarch/ubuntu-core:arm64-bionic
+        with:
+          args: 'bash -c "apt-get update && apt-get install -y curl && ./bin/installdependencies.sh && ./config.sh --unattended --name $unique_runner_name --url ${{github.event.repository.html_url}} --pat ${{secrets.PAT}} --labels $unique_runner_name  --replace"'
+        env:
+          RUNNER_ALLOW_RUNASROOT: 1
+          unique_runner_name: linux-arm64-${{needs.init.outputs.unique_runner_label}}
+
+      - name: Start Runner and Wait for Job
+        timeout-minutes: 5
+        uses: docker://multiarch/ubuntu-core:arm64-bionic
+        with:
+          args: 'bash -c "apt-get update && apt-get install -y curl git && ./bin/installdependencies.sh && ./run.sh --once"'
+        env:
+          RUNNER_ALLOW_RUNASROOT: 1
+
+      - name: Remove Runner
+        if: always()
+        continue-on-error: true
+        uses: docker://multiarch/ubuntu-core:arm64-bionic
+        with:
+          args: 'bash -c "apt-get update && apt-get install -y curl && ./bin/installdependencies.sh && ./config.sh remove --pat ${{secrets.PAT}}"'
+        env:
+          RUNNER_ALLOW_RUNASROOT: 1
+
+      - name: Upload Runner Logs
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: linux_arm64_logs
+          path: _diag
+
+  WindowsE2E:
+    needs: [build, init]
+    runs-on: windows-latest
+    steps:
+      - name: Download Runner
+        uses: actions/download-artifact@v2
+        with:
+          name: runner-package-win-x64
+      - name: Unzip Runner Package
+        run: |
+          Get-ChildItem *.zip | Expand-Archive -DestinationPath $PWD
+      - name: Configure Runner
+        shell: cmd
+        run: |
+          config.cmd --unattended --url ${{github.event.repository.html_url}} --name %unique_runner_name% --pat ${{secrets.PAT}} --labels %unique_runner_name% --replace
+        env:
+          unique_runner_name: win-x64-${{needs.init.outputs.unique_runner_label}}
+
+      - name: Start Runner and Wait for Job
+        shell: cmd
+        timeout-minutes: 5
+        run: |
+          run.cmd --once
+      - name: Remove Runner
+        shell: cmd
+        if: always()
+        continue-on-error: true
+        run: |
+          config.cmd remove --pat ${{secrets.PAT}}
+      - name: Upload Runner Logs
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: win_x64_logs
+          path: _diag
+  
+  check:
+    name: Check runner logs 🕵️‍♂️
+    needs: [WindowsE2E, LinuxE2E, macOSE2E, ARM64E2E]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download Linux Runner Logs
+      uses: actions/download-artifact@v2
+      with:
+        name: linux_x64_logs
+        path: linux_x64_logs
+    - name: Download macOS Runner Logs
+      uses: actions/download-artifact@v2
+      with:
+        name: osx_x64_logs
+        path: osx_x64_logs
+    - name: Download Linux ARM64 Runner Logs
+      uses: actions/download-artifact@v2
+      with:
+        name: linux_arm64_logs
+        path: linux_arm64_logs
+    - name: Download Windows Runner Logs
+      uses: actions/download-artifact@v2
+      with:
+        name: win_x64_logs
+        path: win_x64_logs
+    - name: Check Runner Logs
+      run: |
+        function failed()
+        {
+            local error=${1:-Undefined error}
+            echo "Failed: $error" >&2
+            exit 1
+        }
+        grep -R "Job runner-basics completed with result: Succeeded" ./win_x64_logs || failed "Windows Runner fail to run the job, please check logs"
+        grep -R "Job runner-basics completed with result: Succeeded" ./linux_x64_logs || failed "Linux Runner fail to run the job, please check logs"
+        grep -R "Job runner-basics completed with result: Succeeded" ./osx_x64_logs || failed "macOS Runner fail to run the job, please check logs"
+        grep -R "Job runner-basics completed with result: Succeeded" ./linux_arm64_logs || failed "Linux ARM64 Runner fail to run the job, please check logs"

.github/workflows/runner-basic-e2e-test-case.yml

@@ -0,0 +1,31 @@
+name: Runner Basics Test Case
+
+on:
+  workflow_dispatch:
+    inputs: 
+      target_runner:
+        description: 'Self-hosted runner will run the job'
+        required: true
+
+jobs:
+  test:
+    runs-on:
+    - self-hosted
+    - ${{github.event.inputs.target_runner}}
+
+    name: Runner Basic Test 🛠
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run a one-line script
+      run: echo Hello, world!
+    - name: Run a multi-line script
+      shell: bash
+      run: |
+        printenv|sort
+        cat $GITHUB_EVENT_PATH
+    - name: Validate GitHub Context
+      shell: bash
+      run: |
+        declare -a context_vars=("GITHUB_ACTION" "GITHUB_ACTIONS" "GITHUB_REPOSITORY" "GITHUB_WORKSPACE" "GITHUB_SHA" "GITHUB_RUN_ID" "GITHUB_RUN_NUMBER")
+        for var in ${context_vars[@]};
+        do [ -z "${!var}" ] && echo "##[error]$var not found" && exit 1 || echo "$var: ${!var}"; done