chart: Bump chart and app versions for ARC 0.26.0 (#1799)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -17,6 +17,12 @@ body:
     label: Helm Chart Version
     description: Run `helm list` and see what's shown under CHART VERSION. Any release tags prefixed with `actions-runner-controller-` are for chart releases
     placeholder: ex. 0.11.0
+- type: input
+  id: cert-manager-version
+  attributes:
+    label: CertManager Version
+    description: Run `kubectl get po -o yaml $CERT_MANAGER_POD` and see the image tag, or run `helm list` and see what's shown under APP VERSION for your cert-manager Helm release.
+    placeholder: ex. 1.8
 - type: dropdown
   id: deployment-method
   attributes:
@@ -29,6 +35,17 @@ body:
       - Other
   validations:
     required: true
+- type: textarea
+  id: cert-manager
+  attributes:
+    label: cert-manager installation
+    description: Confirm that you've installed cert-manager correctly by answering a few questions
+    placeholder: |
+      - Did you follow https://github.com/actions-runner-controller/actions-runner-controller#installation? If not, describe the installation process so that we can reproduce your environment.
+      - Are you sure you've installed cert-manager from an official source?
+      (Note that we won't provide user support for cert-manager itself. Make sure cert-manager is fully working before testing ARC or reporting a bug
+  validations:
+    required: true
 - type: checkboxes
   id: checks
   attributes:
@@ -41,7 +58,7 @@ body:
       required: true
     - label: My actions-runner-controller version (v0.x.y) does support the feature
       required: true
-    - label: I've already upgraded ARC to the latest and it didn't fix the issue
+    - label: I've already upgraded ARC (including the CRDs, see charts/actions-runner-controller/docs/UPGRADING.md for details) to the latest and it didn't fix the issue
       required: true
 - type: textarea
   id: resource-definitions
@@ -113,9 +130,11 @@ body:
   id: controller-logs
   attributes:
     label: Controller Logs
-    description: "Include logs from `actions-runner-controller`'s controller-manager pod"
+    description: "NEVER EVER OMIT THIS! Include logs from `actions-runner-controller`'s controller-manager pod"
     render: shell
     placeholder: |
+      PROVIDE THE LOGS VIA A GIST LINK (https://gist.github.com/), NOT DIRECTLY IN THIS TEXT AREA
+      
       To grab controller logs:
 
       # Set NS according to your setup
@@ -125,8 +144,6 @@ body:
       kubectl -n $NS get po
 
       kubectl -n $NS logs $POD_NAME > arc.log
-
-      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
   validations:
     required: true
 - type: textarea
@@ -136,6 +153,8 @@ body:
     description: "Include logs from runner pod(s)"
     render: shell
     placeholder: |
+      PROVIDE THE LOGS VIA A GIST LINK (https://gist.github.com/), NOT DIRECTLY IN THIS TEXT AREA
+      
       To grab the runner pod logs:
 
       # Set NS according to your setup. It should match your RunnerDeployment's metadata.namespace.
@@ -146,8 +165,6 @@ body:
 
       kubectl -n $NS logs $POD_NAME -c runner > runnerpod_runner.log
       kubectl -n $NS logs $POD_NAME -c docker > runnerpod_docker.log
-
-      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
   validations:
     required: true
 - type: textarea

.github/actions/setup-docker-environment/action.yaml

@@ -29,23 +29,23 @@ runs:
       shell: bash
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
       with:
         version: latest
 
     - name: Login to DockerHub
-      if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
-      uses: docker/login-action@v1
+      if: ${{ github.event_name == 'release' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+      uses: docker/login-action@v2
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
 
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
-      if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+      if: ${{ github.event_name == 'release' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+      uses: docker/login-action@v2
       with:
         registry: ghcr.io
         username: ${{ inputs.ghcr_username }}

.github/lock.yml

@@ -1,25 +0,0 @@
-# Configuration for Lock Threads
-# Repo: https://github.com/dessant/lock-threads-app
-# App: https://github.com/apps/lock
-
-# Number of days of inactivity before a closed issue or pull request is locked
-daysUntilLock: 7
-
-# Skip issues and pull requests created before a given timestamp. Timestamp must
-# follow ISO 8601 (`YYYY-MM-DD`). Set to `false` to disable
-skipCreatedBefore: false
-
-# Issues and pull requests with these labels will be ignored. Set to `[]` to disable
-exemptLabels: []
-
-# Label to add before locking, such as `outdated`. Set to `false` to disable
-lockLabel: false
-
-# Comment to post before locking. Set to `false` to disable
-lockComment: >
-  This thread has been automatically locked since there has not been
-  any recent activity after it was closed. Please open a new issue for
-  related bugs.
-
-# Assign `resolved` as the reason for locking. Set to `false` to disable
-setLockReason: true

.github/renovate.json5

@@ -13,7 +13,7 @@
     {
       // use https://github.com/actions/runner/releases
       "fileMatch": [
-        ".github/workflows/runners.yml"
+        ".github/workflows/runners.yaml"
       ],
       "matchStrings": ["RUNNER_VERSION: +(?<currentValue>.*?)\\n"],
       "depNameTemplate": "actions/runner",
@@ -30,8 +30,9 @@
     },
     {
       "fileMatch": [
-        "runner/Dockerfile",
-        "runner/Dockerfile.dindrunner"
+        "runner/actions-runner.dockerfile",
+        "runner/actions-runner-dind.dockerfile",
+        "runner/actions-runner-dind-rootless.dockerfile"
       ],
       "matchStrings": ["RUNNER_VERSION=+(?<currentValue>.*?)\\n"],
       "depNameTemplate": "actions/runner",

.github/workflows/publish-arc.yaml

@@ -1,26 +1,28 @@
-name: Publish Controller Image
+name: Publish ARC
 
 on:
   release:
-    types: [published]
+    types:
+      - published
+
+# https://docs.github.com/en/rest/overview/permissions-required-for-github-apps
+permissions:
+ contents: write 
+ packages: write
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  release-controller:
     name: Release
+    runs-on: ubuntu-latest
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
     steps:
-      - name: Set outputs
-        id: vars
-        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
-
       - name: Checkout
         uses: actions/checkout@v3
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.17.7'
+          go-version: '1.18.2'
 
       - name: Install tools
         run: |
@@ -39,31 +41,31 @@ jobs:
       - name: Upload artifacts
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: make github-release
+        run: |
+          make github-release
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
+      - name: Setup Docker Environment
+        id: vars
+        uses: ./.github/actions/setup-docker-environment
         with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
+          username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+          ghcr_username: ${{ github.actor }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and Push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           file: Dockerfile
           platforms: linux/amd64,linux/arm64
+          build-args: VERSION=${{ env.VERSION }}
           push: true
           tags: |
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
-
+            ghcr.io/actions-runner-controller/actions-runner-controller:latest
+            ghcr.io/actions-runner-controller/actions-runner-controller:${{ env.VERSION }}
+            ghcr.io/actions-runner-controller/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/publish-canary.yaml

@@ -0,0 +1,59 @@
+name: Publish Canary Image
+
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - '**.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/workflows/validate-chart.yaml'
+      - '.github/workflows/publish-chart.yaml'
+      - '.github/workflows/publish-arc.yaml'
+      - '.github/workflows/runners.yaml'
+      - '.github/workflows/validate-entrypoint.yaml'
+      - '.github/renovate.*'
+      - 'runner/**'
+      - '.gitignore'
+      - 'PROJECT'
+      - 'LICENSE'
+      - 'Makefile'
+
+# https://docs.github.com/en/rest/overview/permissions-required-for-github-apps
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  canary-build:
+    name: Build and Publish Canary Image
+    runs-on: ubuntu-latest
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Docker Environment
+        id: vars
+        uses: ./.github/actions/setup-docker-environment
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+          ghcr_username: ${{ github.actor }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Considered unstable builds
+      # See Issue #285, PR #286, and PR #323 for more information
+      - name: Build and Push
+        uses: docker/build-push-action@v3
+        with:
+          file: Dockerfile
+          platforms: linux/amd64,linux/arm64
+          build-args: VERSION=canary-${{ github.sha }}
+          push: true
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary
+            ghcr.io/${{ github.repository }}:canary
+          cache-from: type=gha,scope=arc-canary
+          cache-to: type=gha,mode=max,scope=arc-canary

.github/workflows/publish-chart.yaml

@@ -1,4 +1,4 @@
-name: Publish helm chart
+name: Publish Helm Chart
 
 on:
   push:
@@ -6,7 +6,7 @@ on:
       - master
     paths:
       - 'charts/**'
-      - '.github/workflows/on-push-master-publish-chart.yml'
+      - '.github/workflows/publish-chart.yaml'
       - '!charts/actions-runner-controller/docs/**'
       - '!**.md'
   workflow_dispatch:
@@ -15,10 +15,13 @@ env:
   KUBE_SCORE_VERSION: 1.10.0
   HELM_VERSION: v3.8.0
 
+permissions:
+  contents: read
+
 jobs:
   lint-chart:
-    runs-on: ubuntu-latest
     name: Lint Chart
+    runs-on: ubuntu-latest
     outputs:
       publish-chart: ${{ steps.publish-chart-step.outputs.publish }}
     steps:
@@ -28,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.1
+        uses: azure/setup-helm@v3.3
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -49,12 +52,12 @@ jobs:
               --enable-optional-test container-security-context-readonlyrootfilesystem
 
       # python is a requirement for the chart-testing action below (supports yamllint among other tests)
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: '3.7'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.2.1
+        uses: helm/chart-testing-action@v2.3.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -65,22 +68,23 @@ jobs:
           fi
 
       - name: Run chart-testing (lint)
-        run: ct lint --config charts/.ci/ct-config.yaml
+        run: |
+          ct lint --config charts/.ci/ct-config.yaml
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.2.0
         if: steps.list-changed.outputs.changed == 'true'
+        uses: helm/kind-action@v1.3.0
 
       # We need cert-manager already installed in the cluster because we assume the CRDs exist
       - name: Install cert-manager
+        if: steps.list-changed.outputs.changed == 'true'      
         run: |
           helm repo add jetstack https://charts.jetstack.io --force-update
           helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
-        if: steps.list-changed.outputs.changed == 'true'
 
       - name: Run chart-testing (install)
-        run: ct install --config charts/.ci/ct-config.yaml
         if: steps.list-changed.outputs.changed == 'true'
+        run: ct install --config charts/.ci/ct-config.yaml
 
       # WARNING: This relies on the latest release being inat the top of the JSON from GitHub and a clean chart.yaml
       - name: Check if Chart Publish is Needed
@@ -99,8 +103,11 @@ jobs:
   publish-chart:
     if: needs.lint-chart.outputs.publish-chart == 'true'
     needs: lint-chart
-    runs-on: ubuntu-latest
     name: Publish Chart
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # for helm/chart-releaser-action to push chart release and create a release
+    
 
     steps:
       - name: Checkout

.github/workflows/run-codeql.yaml

@@ -0,0 +1,32 @@
+name: Run CodeQL
+
+on:
+  push:
+    branches: 
+      - master
+  pull_request:
+    branches:
+      - master
+  schedule:
+    - cron: '30 1 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: go
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/run-stale.yaml

@@ -1,12 +1,18 @@
-name: 'Close stale issues and PRs'
+name: Run Stale Bot
 on:
   schedule:
-    # 01:30 every day
     - cron: '30 1 * * *'
 
+permissions:
+  contents: read
+
 jobs:
   stale:
+    name: Run Stale
     runs-on: ubuntu-latest
+    permissions:
+      issues: write         # for actions/stale to close stale issues
+      pull-requests: write  # for actions/stale to close stale PRs
     steps:
       - uses: actions/stale@v5
         with:

.github/workflows/runners.yaml

@@ -2,31 +2,41 @@ name: Runners
 
 on:
   pull_request:
-    types: 
+    types:
       - opened
       - synchronize
       - reopened
-      - closed
     branches:
       - 'master'
     paths:
       - 'runner/**'
       - '!runner/Makefile'
-      - .github/workflows/runners.yml
+      - '.github/workflows/runners.yaml'
+      - '!**.md'
+  # We must do a trigger on a push: instead of a types: closed so GitHub Secrets 
+  # are available to the workflow run
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'runner/**'
+      - '!runner/Makefile'
+      - '.github/workflows/runners.yaml'
       - '!**.md'
 
 env:
-  RUNNER_VERSION: 2.290.1
+  RUNNER_VERSION: 2.296.2
   DOCKER_VERSION: 20.10.12
+  RUNNER_CONTAINER_HOOKS_VERSION: 0.1.2
   DOCKERHUB_USERNAME: summerwind
 
 jobs:
-  build:
+  build-runners:
+    name: Build ${{ matrix.name }}-${{ matrix.os-name }}-${{ matrix.os-version }}
     runs-on: ubuntu-latest
     permissions:
       packages: write
       contents: read
-    name: Build ${{ matrix.name }}-${{ matrix.os-name }}-${{ matrix.os-version }}
     strategy:
       fail-fast: false
       matrix:
@@ -34,11 +44,12 @@ jobs:
           - name: actions-runner
             os-name: ubuntu
             os-version: 20.04
-            dockerfile: Dockerfile
           - name: actions-runner-dind
             os-name: ubuntu
             os-version: 20.04
-            dockerfile: Dockerfile.dindrunner
+          - name: actions-runner-dind-rootless
+            os-name: ubuntu
+            os-version: 20.04
 
     steps:
       - name: Checkout
@@ -47,22 +58,23 @@ jobs:
       - name: Setup Docker Environment
         id: vars
         uses: ./.github/actions/setup-docker-environment
-        with: 
+        with:
           username: ${{ env.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
           ghcr_username: ${{ github.actor }}
           ghcr_password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and Push Versioned Tags
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: ./runner
-          file: ./runner/${{ matrix.dockerfile }}
+          file: ./runner/${{ matrix.name }}.dockerfile
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
           build-args: |
             RUNNER_VERSION=${{ env.RUNNER_VERSION }}
             DOCKER_VERSION=${{ env.DOCKER_VERSION }}
+            RUNNER_CONTAINER_HOOKS_VERSION=${{ env.RUNNER_CONTAINER_HOOKS_VERSION }}
           tags: |
             ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
             ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
@@ -70,5 +82,5 @@ jobs:
             ghcr.io/${{ github.repository }}/${{ matrix.name }}:latest
             ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
             ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=build-${{ matrix.name }}
+          cache-to: type=gha,mode=max,scope=build-${{ matrix.name }}

.github/workflows/validate-arc.yaml

@@ -1,45 +1,59 @@
-name: CI
+name: Validate ARC
 
 on:
   pull_request:
     branches:
       - master
     paths-ignore:
-      - .github/workflows/runners.yml
-      - .github/workflows/on-push-lint-charts.yml
-      - .github/workflows/on-push-master-publish-chart.yml
-      - .github/workflows/release.yml
-      - .github/workflows/test-entrypoint.yml
-      - .github/workflows/wip.yml
-      - 'runner/**'
       - '**.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/workflows/publish-canary.yaml'
+      - '.github/workflows/validate-chart.yaml'
+      - '.github/workflows/publish-chart.yaml'
+      - '.github/workflows/runners.yaml'
+      - '.github/workflows/publish-arc.yaml'
+      - '.github/workflows/validate-entrypoint.yaml'
+      - '.github/renovate.*'
+      - 'runner/**'
       - '.gitignore'
+      - 'PROJECT'
+      - 'LICENSE'
+      - 'Makefile'
+
+permissions:
+  contents: read
 
 jobs:
-  test:
+  test-controller:
+    name: Test ARC
     runs-on: ubuntu-latest
-    name: Test
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-    - uses: actions/setup-go@v3
+
+    - name: Set-up Go
+      uses: actions/setup-go@v3
       with:
-        go-version: '1.17.7'
+        go-version: '1.18.2'
         check-latest: false
-    - run: go version
+    
     - uses: actions/cache@v3
       with:
         path: ~/go/pkg/mod
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
+
     - name: Install kubebuilder
       run: |
         curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_linux_amd64.tar.gz
         tar zxvf kubebuilder_2.3.2_linux_amd64.tar.gz
         sudo mv kubebuilder_2.3.2_linux_amd64 /usr/local/kubebuilder
+
     - name: Run tests
-      run: make test
+      run: |
+        make test
+
     - name: Verify manifests are up-to-date
       run: |
         make manifests

.github/workflows/validate-chart.yaml

@@ -1,10 +1,10 @@
-name: Lint and Test Charts
+name: Validate Helm Chart
 
 on:
   push:
     paths:
       - 'charts/**'
-      - '.github/workflows/on-push-lint-charts.yml'
+      - '.github/workflows/validate-chart.yaml'
       - '!charts/actions-runner-controller/docs/**'
       - '!**.md'
   workflow_dispatch:
@@ -12,10 +12,13 @@ env:
   KUBE_SCORE_VERSION: 1.10.0
   HELM_VERSION: v3.8.0
 
+permissions:
+  contents: read
+
 jobs:
-  lint-test:
-    runs-on: ubuntu-latest
+  validate-chart:
     name: Lint Chart
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -23,7 +26,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.1
+        uses: azure/setup-helm@v3.3
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -44,12 +47,12 @@ jobs:
               --enable-optional-test container-security-context-readonlyrootfilesystem
 
       # python is a requirement for the chart-testing action below (supports yamllint among other tests)
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: '3.7'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.2.1
+        uses: helm/chart-testing-action@v2.3.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -60,18 +63,20 @@ jobs:
           fi
 
       - name: Run chart-testing (lint)
-        run: ct lint --config charts/.ci/ct-config.yaml
+        run: |
+          ct lint --config charts/.ci/ct-config.yaml
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.3.0
         if: steps.list-changed.outputs.changed == 'true'
 
       # We need cert-manager already installed in the cluster because we assume the CRDs exist
       - name: Install cert-manager
+        if: steps.list-changed.outputs.changed == 'true'
         run: |
           helm repo add jetstack https://charts.jetstack.io --force-update
           helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
-        if: steps.list-changed.outputs.changed == 'true'
 
       - name: Run chart-testing (install)
-        run: ct install --config charts/.ci/ct-config.yaml
+        run: |
+          ct install --config charts/.ci/ct-config.yaml

.github/workflows/validate-runners.yaml

@@ -1,4 +1,4 @@
-name: Unit tests for entrypoint
+name: Validate Runners
 
 on:
   pull_request:
@@ -9,13 +9,17 @@ on:
       - 'test/entrypoint/**'
       - '!**.md'
 
+permissions:
+  contents: read
+
 jobs:
-  test:
-    runs-on: ubuntu-latest
+  test-runner-entrypoint:
     name: Test entrypoint
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-    - name: Run unit tests for entrypoint.sh
+
+    - name: Run tests
       run: |
         make acceptance/runner/entrypoint

.github/workflows/wip.yml

@@ -1,51 +0,0 @@
-name: Publish Canary Image
-
-on:
-  push:
-    branches:
-      - master
-    paths-ignore:
-      - .github/workflows/runners.yml
-      - .github/workflows/on-push-lint-charts.yml
-      - .github/workflows/on-push-master-publish-chart.yml
-      - .github/workflows/release.yml
-      - .github/workflows/test-entrypoint.yml
-      - "runner/**"
-      - "**.md"
-      - ".gitignore"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    name: Build and Publish Canary Image
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
-      # Considered unstable builds
-      # See Issue #285, PR #286, and PR #323 for more information
-      - name: Build and Push
-        uses: docker/build-push-action@v2
-        with:
-          file: Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary

Actions-Runner-Controller-Overview.md

@@ -0,0 +1,132 @@
+## Introduction
+This document provides a high level overview of Actions Runner Controller (ARC). ARC enables running Github Actions Runners on Kubernetes (K8s) clusters.
+
+This document provides a background of Github Actions, self-hosted runners and ARC overview. By the end of the doc, the reader should have a foundation with basic scenarios and be capable of reviewing other advanced topics.
+
+## GitHub Actions
+[GitHub Actions](https://github.com/features/actions) is a continuous integration and continuous delivery (CI/CD) platform to automate your build, test, and deployment pipeline. 
+
+You can create workflows that build and test every pull request to your repository, or deploy merged pull requests to production. Your workflow contains one or more jobs which can run in sequential order or in parallel. Each job will run inside its own runner and has one or more steps that either run a script that you define or run an action, which is a reusable extension that can simplify your workflow. To learn more about about Actions - see "[Learn Github Actions](https://docs.github.com/en/actions/learn-github-actions)".
+
+## Runners
+Runners execute the job that is assigned to them by Github Actions workflow. There are two types of Runners:
+
+- [Github-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners) - GitHub provides Linux, Windows, and macOS virtual machines to run your workflows. These virtual machines are hosted in the cloud by Github.
+- [Self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) - you can host your own self-hosted runners in your own data center or cloud infrastructure. ARC deploys self-hosted runners.
+
+## Self hosted runners
+Self-hosted runners offer more control of hardware, operating system, and software tools than GitHub-hosted runners. With self-hosted runners, you can create custom hardware configurations that meet your needs with processing power or memory to run larger jobs, install software available on your local network, and choose an operating system not offered by GitHub-hosted runners.
+
+### Types of Self hosted runners
+Self-hosted runners can be physical, virtual, in a container, on-premises, or in a cloud.
+- Traditional Deployment is having a physical machine, with OS and apps on it. The runner runs on this machine and executes any jobs. It comes with the cost of owning and operating the hardware 24/7 even if it isn't in use that entire time. 
+- Virtualized deployments are simpler to manage. Each runner runs on a virtual machine (VM) that runs on a host. There could be multiple such VMs running on the same host. VMs are complete OS’s and might take time to bring up everytime a clean environment is needed to run workflows.
+- Containerized deployments are similar to VMs, but instead of bringing up entire VM’s, a container gets deployed.Kubernetes (K8s) provides a scalable and reproducible environment for containerized workloads. They are lightweight, loosely coupled, highly efficient and can be managed centrally. There are advantages to using Kubernetes (outlined "[here](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/)."), but it is more complicated and less widely-understood than the other options. A managed provider makes this much simpler to run at scale.
+
+*Actions Runner Controller(ARC) makes it simpler to run self hosted runners on K8s managed containers.*
+
+## Actions Runner Controller (ARC)
+ARC  is a K8s controller to create self-hosted runners on your K8s cluster. With few commands, you can set up self hosted runners that can scale up and down based on demand. And since these could be ephemeral and based on containers, new instances of the runner can be brought up rapidly and cleanly.
+
+### Deploying ARC
+We have a quick start guide that demonstrates how to easily deploy ARC into your K8s environment. For more details, see "[QuickStart Guide](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/QuickStartGuide.md)."
+
+## ARC components
+ARC basically consists of a set of custom resources. An ARC deployment is applying these custom resources onto a K8s cluster. Once applied, it creates a set of Pods, with the Github Actions runner running within them. Github is now able to treat these Pods as self hosted runners and allocate jobs to them.
+
+### Custom resources 
+ARC consists of several custom resource definitions (Runner, Runner Set, Runner Deployment, Runner Replica Set and Horizontal Runner AutoScaler). For more information on CRDs, refer "[Kubernetes Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)."
+
+The helm command (in the QuickStart guide) installs the custom resources into the actions-runner-system namespace.
+```console
+helm install -f custom-values.yaml --wait --namespace actions-runner-system \
+  --create-namespace actions-runner-controller \
+  actions-runner-controller/actions-runner-controller
+ ```
+
+### Runner deployment
+Once the custom resources are installed, another command deploys ARC into your K8s cluster.
+
+![actions-runner-controller architecture](https://user-images.githubusercontent.com/53718047/183928236-ddf72c15-1d11-4304-ad6f-0a0ff251ca55.jpg)
+
+
+
+The `Deployment and Configure ARC` section in the `Quick Start guide` lists the steps to deploy ARC using a `runnerdeployment.yaml` file. Here, we will explain the details
+For more details, see "[QuickStart Guide](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/QuickStartGuide.md)."
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeploy
+spec:
+  replicas: 1
+  template:
+    spec:
+      repository: mumoshu/actions-runner-controller-ci
+```
+
+- `kind: RunnerDeployment`: indicates its a kind of custom resource RunnerDeployment.
+- `replicas: 1` : will deploy one replica. Multiple replicas can also be deployed ( more on that later).
+- `repository: mumoshu/actions-runner-controller-ci` : is the repository to link to when the pod comes up with the Actions runner (Note, this can be configured to link at the Enterprise or Organization level also).
+
+When this configuration is applied with `kubectl apply -f runnerdeployment.yaml` , ARC creates one pod `example-runnerdeploy-[**]` with 2 containers `runner` and `docker`.
+`runner` container has the github runner component installed, `docker` container has docker installed.
+
+
+### The Runner container image
+The GitHub hosted runners include a large amount of pre-installed software packages. For complete list, see "[Runner images](https://github.com/actions/virtual-environments/tree/main/images/linux)."
+
+ARC maintains a few runner images with `latest` aligning with GitHub's Ubuntu version. These images do not contain all of the software installed on the GitHub runners. They contain subset of packages from the GitHub runners: Basic CLI packages, git, docker and build-essentials. To install additional software, it is recommended to use the corresponding setup actions. For instance, `actions/setup-java` for Java or `actions/setup-node` for Node.
+
+## Executing workflows
+Now, all the setup and configuration is done. A workflow can be created in the same repository that could target the self hosted runner created from ARC. The workflow needs to have `runs-on: self-hosted` so it can target the self host pool. For more information on targeting workflows to run on self hosted runners, see "[Using Self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow)."
+
+## Scaling runners - statically with replicas count
+With a small tweak to the replicas count (for eg - `replicas: 2`) in the `runnerdeployment.yaml` file, more runners can be created. Depending on the count of replicas, those many sets of pods would be created. As before, Each pod contains the two containers.
+
+
+## Scaling runners - dynamically with Pull Driven Scaling
+ARC also allows for scaling the runners dynamically. There are two mechanisms for dynamically scaling - (1) Webhook driven scaling and (2) Pull Driven scaling, This document describes the Pull Driven scaling model.
+
+![actions-runner-controller architecture_2](https://user-images.githubusercontent.com/53718047/183928429-7000329d-38eb-4054-9879-41ae44e1ff85.jpg)
+
+
+
+You can enable scaling with 3 steps
+1) Enable `HorizontalRunnerAutoscaler` - Create a `deployment.yaml` file of type `HorizontalRunnerAutoscaler`. The schema for this file is defined below.
+2) Scaling parameters - `minReplicas` and `maxReplicas` indicates the min and max number of replicas to scale to.
+3) Scaling metrics - ARC currently supports `PercentageRunnersBusy` as a metric type. The `PercentageRunnersBusy` will poll GitHub for the number of runners in the `busy` state in the RunnerDeployment's namespace, it will then scale depending on how you have configured the scale factors.
+
+### Pull Driven Scaling Schema
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    # Your RunnerDeployment Here
+    name: example-runnerdeploy
+    kind: RunnerDeployment
+  minReplicas: 1
+  maxReplicas: 5
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'
+    scaleDownThreshold: '0.25'
+    scaleUpFactor: '2'
+    scaleDownFactor: '0.5'
+  ```
+
+For more details - please see "[Pull Driven Scaling](https://github.com/actions-runner-controller/actions-runner-controller#pull-driven-scaling)."
+
+*The period between polls is defined by the controller's `--sync-period` flag. If this flag isn't provided then the controller defaults to a sync period of `1m`, this can be configured in seconds or minutes.*
+
+## Other Configurations
+ARC supports several different advanced configuration. 
+- support for alternate runners : Setting up runner pods with Docker-In-Docker configuration.
+- managing runner groups : Managing a set of running with runner groups thus making it easy to manage different groups within enterprise
+- Webhook driven scaling.
+
+Please refer to the documentation in this repo for further details.

Dockerfile

@@ -1,11 +1,10 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.17 as builder
+FROM --platform=$BUILDPLATFORM golang:1.19.1 as builder
 
 WORKDIR /workspace
 
 # Make it runnable on a distroless image/without libc
 ENV CGO_ENABLED=0
-
 # Copy the Go Modules manifests
 COPY go.mod go.sum ./
 
@@ -25,7 +24,7 @@ RUN go mod download
 # With the above commmand,
 # TARGETOS can be "linux", TARGETARCH can be "amd64", "arm64", and "arm", TARGETVARIANT can be "v7".
 
-ARG TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT
+ARG TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT VERSION=dev
 
 # We intentionally avoid `--mount=type=cache,mode=0777,target=/go/pkg/mod` in the `go mod download` and the `go build` runs
 # to avoid https://github.com/moby/buildkit/issues/2334
@@ -37,7 +36,7 @@ env GOCACHE /build/${TARGETPLATFORM}/root/.cache/go-build
 RUN --mount=target=. \
   --mount=type=cache,mode=0777,target=${GOCACHE} \
   export GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOARM=${TARGETVARIANT#v} && \
-  go build -o /out/manager main.go && \
+  go build -ldflags="-X 'github.com/actions-runner-controller/actions-runner-controller/build.Version=${VERSION}'" -o /out/manager main.go && \
   go build -o /out/github-webhook-server ./cmd/githubwebhookserver
 
 # Use distroless as minimal base image to package the manager binary

Makefile

@@ -4,8 +4,8 @@ else
 	NAME ?= summerwind/actions-runner-controller
 endif
 DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
-VERSION ?= latest
-RUNNER_VERSION ?= 2.290.1
+VERSION ?= dev
+RUNNER_VERSION ?= 2.296.2
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}
@@ -15,7 +15,6 @@ TEST_ORG_REPO ?=
 TEST_EPHEMERAL ?= false
 SYNC_PERIOD ?= 1m
 USE_RUNNERSET ?=
-RUNNER_FEATURE_FLAG_EPHEMERAL ?=
 KUBECONTEXT ?= kind-acceptance
 CLUSTER ?= acceptance
 CERT_MANAGER_VERSION ?= v1.1.1
@@ -57,6 +56,7 @@ GO_TEST_ARGS ?= -short
 # Run tests
 test: generate fmt vet manifests
 	go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
+	go test -fuzz=Fuzz -fuzztime=10s -run=Fuzz* ./controllers
 
 test-with-deps: kube-apiserver etcd kubectl
 	# See https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#pkg-constants
@@ -92,7 +92,7 @@ manifests: manifests-gen-crds chart-crds
 manifests-gen-crds: controller-gen yq
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 	for YAMLFILE in config/crd/bases/actions*.yaml; do \
-		$(YQ) write --inplace "$$YAMLFILE" spec.preserveUnknownFields false; \
+		$(YQ) '.spec.preserveUnknownFields = false' --inplace "$$YAMLFILE" ; \
 	done
 
 chart-crds:
@@ -119,6 +119,7 @@ docker-buildx:
 	docker buildx build --platform ${PLATFORMS} \
 		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		--build-arg VERSION=${VERSION} \
 		-t "${NAME}:${VERSION}" \
 		-f Dockerfile \
 		. ${PUSH_ARG}
@@ -188,7 +189,6 @@ acceptance/deploy:
 	TEST_ORG=${TEST_ORG} TEST_ORG_REPO=${TEST_ORG_REPO} SYNC_PERIOD=${SYNC_PERIOD} \
 	USE_RUNNERSET=${USE_RUNNERSET} \
 	TEST_EPHEMERAL=${TEST_EPHEMERAL} \
-	RUNNER_FEATURE_FLAG_EPHEMERAL=${RUNNER_FEATURE_FLAG_EPHEMERAL} \
 	acceptance/deploy.sh
 
 acceptance/tests:
@@ -223,7 +223,7 @@ ifeq (, $(wildcard $(GOBIN)/controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 ;\
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 endif
@@ -243,7 +243,7 @@ ifeq (, $(wildcard $(GOBIN)/yq))
 	YQ_TMP_DIR=$$(mktemp -d) ;\
 	cd $$YQ_TMP_DIR ;\
 	go mod init tmp ;\
-	go get github.com/mikefarah/yq/v3@3.4.0 ;\
+	go install github.com/mikefarah/yq/v4@v4.25.3 ;\
 	rm -rf $$YQ_TMP_DIR ;\
 	}
 endif

QuickStartGuide.md

@@ -0,0 +1,136 @@
+## Introduction
+
+GitHub Actions can be run in GitHub-hosted cloud or self hosted environments. Self-hosted runners offer more control of hardware, operating system, and software tools than GitHub-hosted runners provide.
+
+With just a few steps, you can set up your kubernetes (K8s) cluster to be a self-hosted environment.
+In this guide, we will setup prerequistes, deploy Actions Runner controller (ARC) and then target that cluster to run GitHub Action workflows.
+
+<p align="center">
+  <img src="https://user-images.githubusercontent.com/53718047/181159115-dbf41416-89a7-408c-b575-bb0d059a1a36.png" />
+</p>
+
+
+
+## Setup your K8s cluster
+
+<details><summary><sub>Create a K8s cluster, if not available.</sub></summary>
+   <sub>
+If you don't have a K8s cluster, you can install a local environment using minikube. For more information, see "[Installing minikube](https://minikube.sigs.k8s.io/docs/start/)."
+     
+     "[Using workflows](/actions/using-workflows)."
+   </sub>
+</details>
+
+:one: Install cert-manager in your cluster. For more information, see "[cert-manager](https://cert-manager.io/docs/installation/)."
+
+```shell
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.2/cert-manager.yaml
+```
+<sub> *note:- This command uses v1.8.2. Please replace with a later version, if available.</sub>
+
+
+>You may also install cert-manager using Helm. For instructions, see "[Installing with Helm](https://cert-manager.io/docs/installation/helm/#installing-with-helm)."
+
+
+:two: Next, Generate a Personal Access Token (PAT) for ARC to authenticate with GitHub.
+   - Login to GitHub account and Navigate to https://github.com/settings/tokens/new.
+   - Select  **repo**.
+   - Click **Generate Token** and then copy the token locally ( we’ll need it later).
+
+
+
+
+## Deploy and Configure ARC
+1️⃣ Deploy  and configure ARC on your K8s cluster. You may use Helm or Kubectl.
+
+
+<details><summary>Helm deployment</summary>
+
+##### Add repository
+```shell
+helm repo add actions-runner-controller https://actions-runner-controller.github.io/actions-runner-controller
+```
+
+##### Install Helm chart
+```shell
+helm upgrade --install --namespace actions-runner-system --create-namespace\
+  --set=authSecret.create=true\
+  --set=authSecret.github_token="REPLACE_YOUR_TOKEN_HERE"\
+  --wait actions-runner-controller actions-runner-controller/actions-runner-controller
+```
+<sub> *note:- Replace REPLACE_YOUR_TOKEN_HERE with your PAT that was generated in Step 1 </sub>
+</details>
+
+<details><summary>Kubectl deployment</summary>
+
+##### Deploy ARC
+```shell
+kubectl apply -f \
+https://github.com/actions-runner-controller/actions-runner-controller/\
+releases/download/v0.22.0/actions-runner-controller.yaml
+```
+<sub> *note:- Replace "v0.22.0" with the version you wish to deploy </sub>
+ 
+
+##### Configure Personal Access Token
+```shell
+kubectl create secret generic controller-manager \
+    -n actions-runner-system \
+    --from-literal=github_token=REPLACE_YOUR_TOKEN_HERE
+````
+<sub> *note:- Replace REPLACE_YOUR_TOKEN_HERE with your PAT that was generated in Step 1. </sub>
+  
+  </details>
+
+2️⃣ Create the GitHub self hosted runners and configure to run against your repository.
+
+Create a `runnerdeployment.yaml` file containing..
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeploy
+spec:
+  replicas: 1
+  template:
+    spec:
+      repository: mumoshu/actions-runner-controller-ci
+````
+<sub> *note:- Replace mumoshu/actions-runner-controller-ci with the full path to your github repository. </sub>
+
+Apply this file to your K8s cluster.
+```shell
+kubectl apply -f runnerdeployment.yaml
+````
+ 
+
+>
+>🎉 We are done - now we should have self hosted runners running in K8s configured to your repository.  🎉
+> 
+> Up Next - lets verify and execute some workflows.
+ 
+## Verify and execute workflows
+:one: Verify your setup is successful with.. 
+```shell
+$ kubectl get runners
+NAME                             REPOSITORY                             STATUS
+example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
+
+$ kubectl get pods
+NAME                           READY   STATUS    RESTARTS   AGE
+example-runnerdeploy2475ht2qbr 2/2     Running   0          1m
+````
+Also, this runner has been registered directly to the specified repository, you can see it in repository settings. For more information, see "[settings](https://docs.github.com/en/actions/hosting-your-own-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner)."
+
+:two: You are ready to execute workflows against this self hosted runner. 
+GitHub documentation lists the steps to target Actions against self hosted runners. For more information, see "[Using self-hosted runners in a workflow - GitHub Docs](https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow#using-self-hosted-runners-in-a-workflow)."
+
+There's also has a quick start guide to get started on Actions, For more information, see "[Quick start Guide to GitHub Actions](https://docs.github.com/en/actions/quickstart)."
+
+## Next steps
+ARC provides several interesting features and capabilities. For more information, see "[readme](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/README.md)."
+
+
+
+ 

SECURITY.md

@@ -0,0 +1,22 @@
+# Security Policy
+
+##  Sponsoring the project
+
+This project is maintained by a small team of two and therefore lacks the resource to provide security fixes in a timely manner.
+
+If you have important business(es) that relies on this project, please consider sponsoring the project so that the maintainer(s) can commit to providing such service.
+
+Please refer to https://github.com/sponsors/actions-runner-controller for available tiers.
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.23.0  | :white_check_mark: |
+| < 0.23.0| :x:                |
+
+## Reporting a Vulnerability
+
+To report a security issue, please email ykuoka+arcsecurity(at)gmail.com with a description of the issue, the steps you took to create the issue, affected versions, and, if known, mitigations for the issue.
+
+A maintainer will try to respond within 5 working days. If the issue is confirmed as a vulnerability, a Security Advisory will be opened. This project tries to follow a 90 day disclosure timeline.

TROUBLESHOOTING.md

@@ -1,10 +1,93 @@
 # Troubleshooting
 
-* [Invalid header field value](#invalid-header-field-value)
-* [Runner coming up before network available](#runner-coming-up-before-network-available)
-* [Deployment fails on GKE due to webhooks](#deployment-fails-on-gke-due-to-webhooks)
+* [Tools](#tools)
+* [Installation](#installation)
+  * [InternalError when calling webhook: context deadline exceeded](#internalerror-when-calling-webhook-context-deadline-exceeded)
+  * [Invalid header field value](#invalid-header-field-value)
+  * [Helm chart install failure: certificate signed by unknown authority](#helm-chart-install-failure-certificate-signed-by-unknown-authority)
+* [Operations](#operations)
+  * [Stuck runner kind or backing pod](#stuck-runner-kind-or-backing-pod)
+  * [Delay in jobs being allocated to runners](#delay-in-jobs-being-allocated-to-runners)
+  * [Runner coming up before network available](#runner-coming-up-before-network-available)
+  * [Outgoing network action hangs indefinitely](#outgoing-network-action-hangs-indefinitely)
+  * [Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns](#unable-to-scale-to-zero-with-totalnumberofqueuedandinprogressworkflowruns)
 
-## Invalid header field value
+## Tools
+
+A list of tools which are helpful for troubleshooting
+
+* https://github.com/rewanthtammana/kubectl-fields Kubernetes resources hierarchy parsing tool
+* https://github.com/stern/stern Multi pod and container log tailing for Kubernetes
+
+## Installation
+
+Troubeshooting runbooks that relate to ARC installation problems
+
+### InternalError when calling webhook: context deadline exceeded
+
+**Problem**
+
+This issue can come up for various reasons like leftovers from previous installations or not being able to access the K8s service's clusterIP associated with the admission webhook server (of ARC).
+
+```
+Internal error occurred: failed calling webhook "mutate.runnerdeployment.actions.summerwind.dev":
+Post "https://actions-runner-controller-webhook.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runnerdeployment?timeout=10s": context deadline exceeded
+```
+
+**Solution**
+
+First we will try the common solution of checking webhook leftovers from previous installations:
+
+1.  ```bash
+    kubectl get validatingwebhookconfiguration -A
+    kubectl get mutatingwebhookconfiguration -A
+    ```
+2.  If you see any webhooks related to actions-runner-controller, delete them:
+    ```bash
+    kubectl delete mutatingwebhookconfiguration actions-runner-controller-mutating-webhook-configuration
+    kubectl delete validatingwebhookconfiguration actions-runner-controller-validating-webhook-configuration
+    ```
+
+If that didn't work then probably your K8s control-plane is somehow unable to access the K8s service's clusterIP associated with the admission webhook server:
+1. You're running apiserver as a binary and you didn't make service cluster IPs available to the host network. 
+2. You're running the apiserver in the pod but your pod network (i.e. CNI plugin installation and config) is not good so your pods(like kube-apiserver) in the K8s control-plane nodes can't access ARC's admission webhook server pod(s) in probably data-plane nodes.
+
+
+Another reason could be due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster: 
+
+To fix this, you may either:
+
+1. Configure the webhook to use another port, such as 443 or 10250, [each of
+   which allow traffic by default](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules).
+
+   ```sh
+   # With helm, you'd set `webhookPort` to the port number of your choice
+   # See https://github.com/actions-runner-controller/actions-runner-controller/pull/1410/files for more information
+   helm upgrade --install --namespace actions-runner-system --create-namespace \
+                --wait actions-runner-controller actions-runner-controller/actions-runner-controller \
+                --set webhookPort=10250
+   ```
+
+2. Set up a firewall rule to allow the master node to connect to the default
+   webhook port. The exact way to do this may vary, but the following script
+   should point you in the right direction:
+
+   ```sh
+   # 1) Retrieve the network tag automatically given to the worker nodes
+   # NOTE: this only works if you have only one cluster in your GCP project. You will have to manually inspect the result of this command to find the tag for the cluster you want to target
+   WORKER_NODES_TAG=$(gcloud compute instances list --format='text(tags.items[0])' --filter='metadata.kubelet-config:*' | grep tags | awk '{print $2}' | sort | uniq)
+
+   # 2) Take note of the VPC network in which you deployed your cluster
+   # NOTE this only works if you have only one network in which you deploy your clusters
+   NETWORK=$(gcloud compute instances list --format='text(networkInterfaces[0].network)' --filter='metadata.kubelet-config:*' | grep networks | awk -F'/' '{print $NF}' | sort | uniq)
+
+   # 3) Get the master source ip block
+   SOURCE=$(gcloud container clusters describe <cluster-name> --region <region> | grep masterIpv4CidrBlock| cut -d ':' -f 2 | tr -d ' ')
+
+   gcloud compute firewall-rules create k8s-cert-manager --source-ranges $SOURCE --target-tags $WORKER_NODES_TAG  --allow TCP:9443 --network $NETWORK
+   ```
+
+### Invalid header field value
 
 **Problem**
 
@@ -23,7 +106,88 @@ Your base64'ed PAT token has a new line at the end, it needs to be created witho
 * `echo -n $TOKEN | base64`
 * Create the secret as described in the docs using the shell and documented flags
 
-## Runner coming up before network available
+### Helm chart install failure: certificate signed by unknown authority
+
+**Problem**
+
+```
+Error: UPGRADE FAILED: failed to create resource: Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority
+```
+
+Apparently, it's failing while `helm` is creating one of resources defined in the ARC chart and the cause was that cert-manager's webhook is not working correctly, due to the missing or the invalid CA certficate.
+
+You'd try to tail logs from the `cert-manager-cainjector` and see it's failing with an error like:
+
+```
+$ kubectl -n cert-manager logs cert-manager-cainjector-7cdbb9c945-g6bt4
+I0703 03:31:55.159339       1 start.go:91] "starting" version="v1.1.1" revision="3ac7418070e22c87fae4b22603a6b952f797ae96"
+I0703 03:31:55.615061       1 leaderelection.go:243] attempting to acquire leader lease  kube-system/cert-manager-cainjector-leader-election...
+I0703 03:32:10.738039       1 leaderelection.go:253] successfully acquired lease kube-system/cert-manager-cainjector-leader-election
+I0703 03:32:10.739941       1 recorder.go:52] cert-manager/controller-runtime/manager/events "msg"="Normal"  "message"="cert-manager-cainjector-7cdbb9c945-g6bt4_88e4bc70-eded-4343-a6fb-0ddd6434eb55 became leader" "object"={"kind":"ConfigMap","namespace":"kube-system","name":"cert-manager-cainjector-leader-election","uid":"942a021e-364c-461a-978c-f54a95723cdc","apiVersion":"v1","resourceVersion":"1576"} "reason"="LeaderElection"
+E0703 03:32:11.192128       1 start.go:119] cert-manager/ca-injector "msg"="manager goroutine exited" "error"=null
+I0703 03:32:12.339197       1 request.go:645] Throttling request took 1.047437675s, request: GET:https://10.96.0.1:443/apis/storage.k8s.io/v1beta1?timeout=32s
+E0703 03:32:13.143790       1 start.go:151] cert-manager/ca-injector "msg"="Error registering certificate based controllers. Retrying after 5 seconds." "error"="no matches for kind \"MutatingWebhookConfiguration\" in version \"admissionregistration.k8s.io/v1beta1\""
+Error: error registering secret controller: no matches for kind "MutatingWebhookConfiguration" in version "admissionregistration.k8s.io/v1beta1"
+```
+
+**Solution**
+
+Your cluster is based on a new enough Kubernetes of version 1.22 or greater which does not support the legacy `admissionregistration.k8s.io/v1beta1` API anymore, and your `cert-manager` is not up-to-date hence it's still trying to use the leagcy Kubernetes API.
+
+In many cases, it's not an option to downgrade Kubernetes. So, just upgrade `cert-manager` to a more recent version that does have have the support for the specific Kubernetes version you're using.
+
+See https://cert-manager.io/docs/installation/supported-releases/ for the list of available cert-manager versions.
+
+## Operations
+
+Troubeshooting runbooks that relate to ARC operational problems
+
+### Stuck runner kind or backing pod
+
+**Problem**
+
+Sometimes either the runner kind (`kubectl get runners`) or it's underlying pod can get stuck in a terminating state for various reasons. You can get the kind unstuck by removing its finaliser using something like this:
+
+**Solution**
+
+Remove the finaliser from the relevent runner kind or pod
+
+```
+# Get all kind runners and remove the finalizer
+$ kubectl get runners --no-headers | awk {'print $1'} | xargs kubectl patch runner --type merge -p '{"metadata":{"finalizers":null}}'
+
+# Get all pods that are stuck terminating and remove the finalizer
+$ kubectl -n get pods | grep Terminating | awk {'print $1'} | xargs kubectl patch pod -p '{"metadata":{"finalizers":null}}'
+```
+
+_Note the code assumes you have already selected the namespace your runners are in and that they 
+are in a namespace not shared with anything else_
+
+### Delay in jobs being allocated to runners
+
+**Problem**
+
+ARC isn't involved in jobs actually getting allocated to a runner. ARC is responsible for orchestrating runners and the runner lifecycle. Why some people see large delays in job allocation is not clear however it has been https://github.com/actions-runner-controller/actions-runner-controller/issues/1387#issuecomment-1122593984 that this is caused from the self-update process somehow.
+
+**Solution**
+
+Disable the self-update process in your runner manifests
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeployment-with-sleep
+spec:
+  template:
+    spec:
+      ...
+      env:
+        - name: DISABLE_RUNNER_UPDATE
+          value: "true"
+```
+
+### Runner coming up before network available
 
 **Problem**
 
@@ -61,40 +225,48 @@ metadata:
 spec:
   template:
     spec:
+      ...
       env:
-        # This runner's entrypoint script will have a 5 seconds delay 
-        # as a first action within the entrypoint script
         - name: STARTUP_DELAY_IN_SECONDS
           value: "5"
 ```
 
-## Deployment fails on GKE due to webhooks
+## Outgoing network action hangs indefinitely
 
 **Problem**
 
-Due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster:
+Some random outgoing network actions hangs indefinitely. This could be because your cluster does not give Docker the standard MTU of 1500, you can check this out by running `ip link` in a pod that encounters the problem and reading the outgoing interface's MTU value. If it is smaller than 1500, then try the following.
 
-```
-Internal error occurred: failed calling webhook "mutate.runner.actions.summerwind.dev": 
-Post https://webhook-service.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runner?timeout=10s: 
-context deadline exceeded
+**Solution**
+
+Add a `dockerMTU` key in your runner's spec with the value you read on the outgoing interface. For instance:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: github-runner
+  namespace: github-system
+spec:
+  replicas: 6
+  template:
+    spec:
+      dockerMTU: 1400
+      repository: $username/$repo
+      env: []
 ```
 
-**Solution**<br />
+There may be more places you need to tweak for MTU.
+Please consult issues like #651 for more information.
 
-To fix this, you need to set up a firewall rule to allow the master node to connect to the webhook port.
-The exact way to do this may wary, but the following script should point you in the right direction:
+## Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns
 
-```
-# 1) Retrieve the network tag automatically given to the worker nodes
-# NOTE: this only works if you have only one cluster in your GCP project. You will have to manually inspect the result of this command to find the tag for the cluster you want to target
-WORKER_NODES_TAG=$(gcloud compute instances list --format='text(tags.items[0])' --filter='metadata.kubelet-config:*' | grep tags | awk '{print $2}' | sort | uniq)
+**Problem**
 
-# 2) Take note of the VPC network in which you deployed your cluster
-# NOTE this only works if you have only one network in which you deploy your clusters
-NETWORK=$(gcloud compute instances list --format='text(networkInterfaces[0].network)' --filter='metadata.kubelet-config:*' | grep networks | awk -F'/' '{print $NF}' | sort | uniq)
+HRA doesn't scale the RunnerDeployment to zero, even though you did configure HRA correctly, to have a pull-based scaling metric `TotalNumberOfQueuedAndInProgressWorkflowRuns`, and set `minReplicas: 0`.
 
-# 3) Get the master source ip block
-SOURCE=$(gcloud container clusters describe <cluster-name> --region <region> | grep masterIpv4CidrBlock| cut -d ':' -f 2 | tr -d ' ')
-gcloud compute firewall-rules create k8s-cert-manager --source-ranges $SOURCE --target-tags $WORKER_NODES_TAG  --allow TCP:9443 --network $NETWORK
-```
+**Solution**
+
+You very likely have some dangling workflow jobs stuck in `queued` or `in_progress` as seen in [#1057](https://github.com/actions-runner-controller/actions-runner-controller/issues/1057#issuecomment-1133439061).
+
+Manually call [the "list workflow runs" API](https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository), and [remove the dangling workflow job(s)](https://docs.github.com/en/rest/actions/workflow-runs#delete-a-workflow-run).

acceptance/argotunnel.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# See https://developers.cloudflare.com/cloudflare-one/tutorials/many-cfd-one-tunnel/
+
+kubectl create ns tunnel || :
+
+kubectl -n tunnel delete secret tunnel-credentials || :
+
+kubectl -n tunnel create secret generic tunnel-credentials \
+  --from-file=credentials.json=$HOME/.cloudflared/${TUNNEL_ID}.json || :
+
+cat <<MANIFEST | kubectl -n tunnel ${OP} -f -
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared
+spec:
+  selector:
+    matchLabels:
+      app: cloudflared
+  replicas: 2 # You could also consider elastic scaling for this deployment
+  template:
+    metadata:
+      labels:
+        app: cloudflared
+    spec:
+      containers:
+      - name: cloudflared
+        image: cloudflare/cloudflared:latest
+        args:
+        - tunnel
+        # Points cloudflared to the config file, which configures what
+        # cloudflared will actually do. This file is created by a ConfigMap
+        # below.
+        - --config
+        - /etc/cloudflared/config/config.yaml
+        - run
+        livenessProbe:
+          httpGet:
+            # Cloudflared has a /ready endpoint which returns 200 if and only if
+            # it has an active connection to the edge.
+            path: /ready
+            port: 2000
+          failureThreshold: 1
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        volumeMounts:
+        - name: config
+          mountPath: /etc/cloudflared/config
+          readOnly: true
+        # Each tunnel has an associated "credentials file" which authorizes machines
+        # to run the tunnel. cloudflared will read this file from its local filesystem,
+        # and it'll be stored in a k8s secret.
+        - name: creds
+          mountPath: /etc/cloudflared/creds
+          readOnly: true
+      volumes:
+      - name: creds
+        secret:
+          secretName: tunnel-credentials
+      # Create a config.yaml file from the ConfigMap below.
+      - name: config
+        configMap:
+          name: cloudflared
+          items:
+          - key: config.yaml
+            path: config.yaml
+---
+# This ConfigMap is just a way to define the cloudflared config.yaml file in k8s.
+# It's useful to define it in k8s, rather than as a stand-alone .yaml file, because
+# this lets you use various k8s templating solutions (e.g. Helm charts) to
+# parameterize your config, instead of just using string literals.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared
+data:
+  config.yaml: |
+    # Name of the tunnel you want to run
+    tunnel: ${TUNNEL_NAME}
+    credentials-file: /etc/cloudflared/creds/credentials.json
+    # Serves the metrics server under /metrics and the readiness server under /ready
+    metrics: 0.0.0.0:2000
+    # Autoupdates applied in a k8s pod will be lost when the pod is removed or restarted, so
+    # autoupdate doesn't make sense in Kubernetes. However, outside of Kubernetes, we strongly
+    # recommend using autoupdate.
+    no-autoupdate: true
+    ingress:
+    # The first rule proxies traffic to the httpbin sample Service defined in app.yaml
+    - hostname: ${TUNNEL_HOSTNAME}
+      service: http://actions-runner-controller-github-webhook-server.actions-runner-system:80
+    # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404.
+    - service: http_status:404
+MANIFEST
+
+kubectl -n tunnel delete po -l app=cloudflared || :

acceptance/deploy.sh

@@ -41,8 +41,23 @@ TEST_ID=${TEST_ID:-default}
 
 if [ "${tool}" == "helm" ]; then
   set -v
+
+  CHART=${CHART:-charts/actions-runner-controller}
+
+  flags=()
+  if [ "${IMAGE_PULL_SECRET}" != "" ]; then
+    flags+=( --set imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set image.actionsRunnerImagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set githubWebhookServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+  fi
+  if [ "${CHART_VERSION}" != "" ]; then
+    flags+=( --version ${CHART_VERSION})
+  fi
+
+  set -vx
+
   helm upgrade --install actions-runner-controller \
-    charts/actions-runner-controller \
+    ${CHART} \
     -n actions-runner-system \
     --create-namespace \
     --set syncPeriod=${SYNC_PERIOD} \
@@ -51,6 +66,7 @@ if [ "${tool}" == "helm" ]; then
     --set image.tag=${VERSION} \
     --set podAnnotations.test-id=${TEST_ID} \
     --set githubWebhookServer.podAnnotations.test-id=${TEST_ID} \
+    ${flags[@]} --set image.imagePullPolicy=${IMAGE_PULL_POLICY} \
     -f ${VALUES_FILE}
   set +v
   # To prevent `CustomResourceDefinition.apiextensions.k8s.io "runners.actions.summerwind.dev" is invalid: metadata.annotations: Too long: must have at most 262144 bytes`
@@ -76,56 +92,3 @@ kubectl -n actions-runner-system wait deploy/actions-runner-controller --for con
 
 # Adhocly wait for some time until actions-runner-controller's admission webhook gets ready
 sleep 20
-
-RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
-
-if [ -n "${TEST_REPO}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl apply -f -
-  else
-    echo 'Deploying runnerdeployment and hra. Set USE_RUNNERSET if you want to deploy runnerset instead.'
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerdeploy envsubst | kubectl apply -f -
-  fi
-else
-  echo 'Skipped deploying runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy.'
-fi
-
-if [ -n "${TEST_ORG}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerset envsubst | kubectl apply -f -
-  else
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerdeploy envsubst | kubectl apply -f -
-  fi
-
-  if [ -n "${TEST_ORG_GROUP}" ]; then
-    if [ "${USE_RUNNERSET}" != "false" ]; then
-      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orgroupg-runnerset envsubst | kubectl apply -f -
-    else
-      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl apply -f -
-    fi
-  else
-    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ORG_GROUP to deploy.'
-  fi
-else
-  echo 'Skipped deploying organizational runnerdeployment. Set TEST_ORG to deploy.'
-fi
-
-if [ -n "${TEST_ENTERPRISE}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerset envsubst | kubectl apply -f -
-  else
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerdeploy envsubst | kubectl apply -f -
-  fi
-
-  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
-    if [ "${USE_RUNNERSET}" != "false" ]; then
-      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerset envsubst | kubectl apply -f -
-    else
-      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl apply -f -
-    fi
-  else
-    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to deploy.'
-  fi
-else
-  echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE to deploy.'
-fi

acceptance/deploy_runners.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -e
+
+OP=${OP:-apply}
+
+RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
+
+cat acceptance/testdata/kubernetes_container_mode.envsubst.yaml  | NAMESPACE=${RUNNER_NAMESPACE} envsubst  | kubectl apply -f -
+
+if [ -n "${TEST_REPO}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl ${OP} -f -
+  else
+    echo "Running ${OP} runnerdeployment and hra. Set USE_RUNNERSET if you want to deploy runnerset instead."
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+else
+  echo "Skipped ${OP} for runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy."
+fi
+
+if [ -n "${TEST_ORG}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerset envsubst | kubectl ${OP} -f -
+  else
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+
+  if [ -n "${TEST_ORG_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerset envsubst | kubectl ${OP} -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl ${OP} -f -
+    fi
+  else
+    echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ORG_GROUP to ${OP}."
+  fi
+else
+  echo "Skipped ${OP} on organizational runnerdeployment. Set TEST_ORG to ${OP}."
+fi
+
+if [ -n "${TEST_ENTERPRISE}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerset envsubst | kubectl ${OP} -f -
+  else
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+
+  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerset envsubst | kubectl ${OP} -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl ${OP} -f -
+    fi
+  else
+    echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to ${OP}."
+  fi
+else
+  echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ENTERPRISE to ${OP}."
+fi

acceptance/testdata/kubernetes_container_mode.envsubst.yaml

@@ -0,0 +1,86 @@
+# USAGE:
+#   cat acceptance/testdata/kubernetes_container_mode.envsubst.yaml  | NAMESPACE=default envsubst  | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: k8s-mode-runner
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["get", "create"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get", "list", "watch",]
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "create", "delete"]
+# Needed to report test success by crating a cm from within workflow job step
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: runner-status-updater
+rules:
+- apiGroups: ["actions.summerwind.dev"]
+  resources: ["runners/status"]
+  verbs: ["get", "update", "patch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+---
+# To verify it's working, try:
+#   kubectl auth can-i --as system:serviceaccount:default:runner get pod
+# If incomplete, workflows and jobs would fail with an error message like:
+#   Error: Error: The Service account needs the following permissions [{"group":"","verbs":["get","list","create","delete"],"resource":"pods","subresource":""},{"group":"","verbs":["get","create"],"resource":"pods","subresource":"exec"},{"group":"","verbs":["get","list","watch"],"resource":"pods","subresource":"log"},{"group":"batch","verbs":["get","list","create","delete"],"resource":"jobs","subresource":""},{"group":"","verbs":["create","delete","get","list"],"resource":"secrets","subresource":""}] on the pod resource in the 'default' namespace. Please contact your self hosted runner administrator.
+#   Error: Process completed with exit code 1.
+apiVersion: rbac.authorization.k8s.io/v1
+# This role binding allows "jane" to read pods in the "default" namespace.
+# You need to already have a Role named "pod-reader" in that namespace.
+kind: RoleBinding
+metadata:
+  name: runner-k8s-mode-runner
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: k8s-mode-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: runner-runner-stat-supdater
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: runner-status-updater
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: org-runnerdeploy-runner-work-dir
+  labels:
+    content: org-runnerdeploy-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

acceptance/testdata/runnerdeploy.envsubst.yaml

@@ -1,3 +1,13 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-runner-work-dir
+  labels:
+    content: ${NAME}-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
@@ -19,11 +29,6 @@ spec:
 
       ephemeral: ${TEST_EPHEMERAL}
 
-      # Whether to pass --ephemeral (true) or --once (false, deprecated)
-      env:
-      - name: RUNNER_FEATURE_FLAG_EPHEMERAL
-        value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
-
       #
       # dockerd within runner container
       #
@@ -44,10 +49,26 @@ spec:
       labels:
       - "${RUNNER_LABEL}"
 
+      env:
+      - name: ROLLING_UPDATE_PHASE
+        value: "${ROLLING_UPDATE_PHASE}"
+
       #
       # Non-standard working directory
       #
       # workDir: "/"
+
+      # # Uncomment the below to enable the kubernetes container mode
+      # # See https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs
+      containerMode: ${RUNNER_CONTAINER_MODE}
+      workVolumeClaimTemplate:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: "${NAME}-runner-work-dir"
+        resources:
+          requests:
+            storage: 10Gi
+      serviceAccountName: ${RUNNER_SERVICE_ACCOUNT_NAME}
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: HorizontalRunnerAutoscaler
@@ -57,7 +78,8 @@ spec:
   scaleTargetRef:
     name: ${NAME}
   scaleUpTriggers:
-  - githubEvent: {}
+  - githubEvent:
+      workflowJob: {}
     amount: 1
     duration: "10m"
   minReplicas: ${RUNNER_MIN_REPLICAS}

acceptance/testdata/runnerset.envsubst.yaml

@@ -1,3 +1,59 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-runner-work-dir
+  labels:
+    content: ${NAME}-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}
+# In kind environments, the provider writes:
+#   /var/lib/docker/volumes/KIND_NODE_CONTAINER_VOL_ID/_data/local-path-provisioner/PV_NAME
+# It can be hundreds of gigabytes depending on what you cache in the test workflow. Beware to not encounter `no space left on device` errors!
+# If you did encounter no space errorrs try:
+#   docker system prune
+#   docker buildx prune #=> frees up /var/lib/docker/volumes/buildx_buildkit_container-builder0_state
+#   sudo rm -rf /var/lib/docker/volumes/KIND_NODE_CONTAINER_VOL_ID/_data/local-path-provisioner #=> frees up local-path-provisioner's data
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-var-lib-docker
+  labels:
+    content: ${NAME}-var-lib-docker
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-cache
+  labels:
+    content: ${NAME}-cache
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-runner-tool-cache
+  labels:
+    content: ${NAME}-runner-tool-cache
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerSet
 metadata:
@@ -56,14 +112,134 @@ spec:
       labels:
         app: ${NAME}
     spec:
+      serviceAccountName: ${RUNNER_SERVICE_ACCOUNT_NAME}
       containers:
       - name: runner
         imagePullPolicy: IfNotPresent
         env:
         - name: RUNNER_FEATURE_FLAG_EPHEMERAL
           value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
-      #- name: docker
-      #  #image: mumoshu/actions-runner-dind:dev
+        - name: GOMODCACHE
+          value: "/home/runner/.cache/go-mod"
+        - name: ROLLING_UPDATE_PHASE
+          value: "${ROLLING_UPDATE_PHASE}"
+        # PV-backed runner work dir
+        volumeMounts:
+        # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
+        # The volume and mount with the same names will be created by workVolumeClaimTemplate and the kubernetes container mode support.
+        # - name: work
+        #   mountPath: /runner/_work
+        # Cache docker image layers, in case dockerdWithinRunnerContainer=true
+        - name: var-lib-docker
+          mountPath: /var/lib/docker
+        # Cache go modules and builds
+        # - name: gocache
+        #   # Run `goenv | grep GOCACHE` to verify the path is correct for your env
+        #   mountPath: /home/runner/.cache/go-build
+        # - name: gomodcache
+        #   # Run `goenv | grep GOMODCACHE` to verify the path is correct for your env
+        #   # mountPath: /home/runner/go/pkg/mod
+        - name: cache
+          # go: could not create module cache: stat /home/runner/.cache/go-mod: permission denied
+          mountPath: "/home/runner/.cache"
+        - name: runner-tool-cache
+          # This corresponds to our runner image's default setting of RUNNER_TOOL_CACHE=/opt/hostedtoolcache.
+          #
+          # In case you customize the envvar in both runner and docker containers of the runner pod spec,
+          # You'd need to change this mountPath accordingly.
+          #
+          # The tool cache directory is defined in actions/toolkit's tool-cache module:
+          # https://github.com/actions/toolkit/blob/2f164000dcd42fb08287824a3bc3030dbed33687/packages/tool-cache/src/tool-cache.ts#L621-L638
+          #
+          # Many setup-* actions like setup-go utilizes the tool-cache module to download and cache installed binaries:
+          # https://github.com/actions/setup-go/blob/56a61c9834b4a4950dbbf4740af0b8a98c73b768/src/installer.ts#L144
+          mountPath: "/opt/hostedtoolcache"
+      # Valid only when dockerdWithinRunnerContainer=false
+      # - name: docker
+      #   # PV-backed runner work dir
+      #   volumeMounts:
+      #   - name: work
+      #     mountPath: /runner/_work
+      #   # Cache docker image layers, in case dockerdWithinRunnerContainer=false
+      #   - name: var-lib-docker
+      #     mountPath: /var/lib/docker
+      #   # image: mumoshu/actions-runner-dind:dev
+
+      #   # For buildx cache
+      #   - name: cache
+      #     mountPath: "/home/runner/.cache"
+        # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
+      # volumes:
+      # - name: work
+      #   ephemeral:
+      #     volumeClaimTemplate:
+      #       spec:
+      #         accessModes:
+      #         - ReadWriteOnce
+      #         storageClassName: "${NAME}-runner-work-dir"
+      #         resources:
+      #           requests:
+      #             storage: 10Gi
+  volumeClaimTemplates:
+  - metadata:
+      name: vol1
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}
+      ## Dunno which provider supports auto-provisioning with selector.
+      ## At least the rancher local path provider stopped with:
+      ##  waiting for a volume to be created, either by external provisioner "rancher.io/local-path" or manually created by system administrator
+      # selector:
+      #    matchLabels:
+      #      runnerset-volume-id: ${NAME}-vol1
+  - metadata:
+      name: vol2
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}
+      # selector:
+      #    matchLabels:
+      #      runnerset-volume-id: ${NAME}-vol2
+  - metadata:
+      name: var-lib-docker
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}-var-lib-docker
+  - metadata:
+      name: cache
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}-cache
+  - metadata:
+      name: runner-tool-cache
+      # It turns out labels doesn't distinguish PVs across PVCs and the
+      # end result is PVs are reused by wrong PVCs.
+      # The correct way seems to be to differentiate storage class per pvc template.
+      # labels:
+      #   id: runner-tool-cache
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}-runner-tool-cache
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: HorizontalRunnerAutoscaler
@@ -74,9 +250,17 @@ spec:
     kind: RunnerSet
     name: ${NAME}
   scaleUpTriggers:
-  - githubEvent: {}
+  - githubEvent:
+      workflowJob: {}
     amount: 1
     duration: "10m"
   minReplicas: ${RUNNER_MIN_REPLICAS}
   maxReplicas: 10
   scaleDownDelaySecondsAfterScaleOut: ${RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT}
+  # Comment out the whole metrics if you'd like to solely test webhook-based scaling
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'
+    scaleDownThreshold: '0.25'
+    scaleUpFactor: '2'
+    scaleDownFactor: '0.5'

acceptance/values.yaml

@@ -1,6 +1,18 @@
 # Set actions-runner-controller settings for testing
 logLevel: "-4"
+imagePullSecrets: []
+image:
+  # This needs to be an empty array rather than a single-item array with empty name.
+  # Otherwise you end up with the following error on helm-upgrade:
+  #   Error: UPGRADE FAILED: failed to create patch: map: map[] does not contain declared merge key: name && failed to create patch: map: map[] does not contain declared merge key: name
+  actionsRunnerImagePullSecrets: []
+runner:
+  statusUpdateHook:
+    enabled: true
+rbac:
+  allowGrantingKubernetesContainerModePermissions: true
 githubWebhookServer:
+  imagePullSecrets: []
   logLevel: "-4"
   enabled: true
   labels: {}

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -60,6 +60,9 @@ type HorizontalRunnerAutoscalerSpec struct {
 	// The earlier a scheduled override is, the higher it is prioritized.
 	// +optional
 	ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
+
+	// +optional
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
 }
 
 type ScaleUpTrigger struct {
@@ -130,7 +133,7 @@ type ScaleTargetRef struct {
 
 type MetricSpec struct {
 	// Type is the type of metric to be used for autoscaling.
-	// The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+	// It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
 	Type string `json:"type,omitempty"`
 
 	// RepositoryNames is the list of repository names to be used for calculating the metric.
@@ -170,7 +173,7 @@ type MetricSpec struct {
 }
 
 // ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
-// A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+// A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
 type ScheduledOverride struct {
 	// StartTime is the time at which the first override starts.
 	StartTime metav1.Time `json:"startTime"`

api/v1alpha1/runner_types.go

@@ -18,8 +18,10 @@ package v1alpha1
 
 import (
 	"errors"
+	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -71,6 +73,19 @@ type RunnerConfig struct {
 	VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"`
 	// +optional
 	VolumeStorageMedium *string `json:"volumeStorageMedium,omitempty"`
+
+	// +optional
+	ContainerMode string `json:"containerMode,omitempty"`
+
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
+}
+
+type GitHubAPICredentialsFrom struct {
+	SecretRef SecretReference `json:"secretRef,omitempty"`
+}
+
+type SecretReference struct {
+	Name string `json:"name"`
 }
 
 // RunnerPodSpec defines the desired pod spec fields of the runner pod
@@ -135,6 +150,9 @@ type RunnerPodSpec struct {
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 
+	// +optional
+	PriorityClassName string `json:"priorityClassName,omitempty"`
+
 	// +optional
 	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
 
@@ -154,10 +172,32 @@ type RunnerPodSpec struct {
 
 	// +optional
 	DnsConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"`
+
+	// +optional
+	WorkVolumeClaimTemplate *WorkVolumeClaimTemplate `json:"workVolumeClaimTemplate,omitempty"`
+}
+
+func (rs *RunnerSpec) Validate(rootPath *field.Path) field.ErrorList {
+	var (
+		errList field.ErrorList
+		err     error
+	)
+
+	err = rs.validateRepository()
+	if err != nil {
+		errList = append(errList, field.Invalid(rootPath.Child("repository"), rs.Repository, err.Error()))
+	}
+
+	err = rs.validateWorkVolumeClaimTemplate()
+	if err != nil {
+		errList = append(errList, field.Invalid(rootPath.Child("workVolumeClaimTemplate"), rs.WorkVolumeClaimTemplate, err.Error()))
+	}
+
+	return errList
 }
 
 // ValidateRepository validates repository field.
-func (rs *RunnerSpec) ValidateRepository() error {
+func (rs *RunnerSpec) validateRepository() error {
 	// Enterprise, Organization and repository are both exclusive.
 	foundCount := 0
 	if len(rs.Organization) > 0 {
@@ -179,6 +219,18 @@ func (rs *RunnerSpec) ValidateRepository() error {
 	return nil
 }
 
+func (rs *RunnerSpec) validateWorkVolumeClaimTemplate() error {
+	if rs.ContainerMode != "kubernetes" {
+		return nil
+	}
+
+	if rs.WorkVolumeClaimTemplate == nil {
+		return errors.New("Spec.ContainerMode: kubernetes must have workVolumeClaimTemplate field specified")
+	}
+
+	return rs.WorkVolumeClaimTemplate.validate()
+}
+
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
 	// Turns true only if the runner pod is ready.
@@ -207,13 +259,60 @@ type RunnerStatusRegistration struct {
 	ExpiresAt    metav1.Time `json:"expiresAt"`
 }
 
+type WorkVolumeClaimTemplate struct {
+	StorageClassName string                              `json:"storageClassName"`
+	AccessModes      []corev1.PersistentVolumeAccessMode `json:"accessModes"`
+	Resources        corev1.ResourceRequirements         `json:"resources"`
+}
+
+func (w *WorkVolumeClaimTemplate) validate() error {
+	if w.AccessModes == nil || len(w.AccessModes) == 0 {
+		return errors.New("Access mode should have at least one mode specified")
+	}
+
+	for _, accessMode := range w.AccessModes {
+		switch accessMode {
+		case corev1.ReadWriteOnce, corev1.ReadWriteMany:
+		default:
+			return fmt.Errorf("Access mode %v is not supported", accessMode)
+		}
+	}
+	return nil
+}
+
+func (w *WorkVolumeClaimTemplate) V1Volume() corev1.Volume {
+	return corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes:      w.AccessModes,
+						StorageClassName: &w.StorageClassName,
+						Resources:        w.Resources,
+					},
+				},
+			},
+		},
+	}
+}
+
+func (w *WorkVolumeClaimTemplate) V1VolumeMount(mountPath string) corev1.VolumeMount {
+	return corev1.VolumeMount{
+		MountPath: mountPath,
+		Name:      "work",
+	}
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:JSONPath=".spec.enterprise",name=Enterprise,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.organization",name=Organization,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.repository",name=Repository,type=string
+// +kubebuilder:printcolumn:JSONPath=".spec.group",name=Group,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.labels",name=Labels,type=string
 // +kubebuilder:printcolumn:JSONPath=".status.phase",name=Status,type=string
+// +kubebuilder:printcolumn:JSONPath=".status.message",name=Message,type=string
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
 // Runner is the Schema for the runners API

api/v1alpha1/runner_webhook.go

@@ -66,15 +66,7 @@ func (r *Runner) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *Runner) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "repository"), r.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Validate(field.NewPath("spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

api/v1alpha1/runnerdeployment_types.go

@@ -33,7 +33,7 @@ type RunnerDeploymentSpec struct {
 
 	// EffectiveTime is the time the upstream controller requested to sync Replicas.
 	// It is usually populated by the webhook-based autoscaler via HRA.
-	// The value is inherited to RunnerRepicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
+	// The value is inherited to RunnerReplicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
 	//
 	// +optional
 	// +nullable

api/v1alpha1/runnerdeployment_webhook.go

@@ -66,15 +66,7 @@ func (r *RunnerDeployment) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *RunnerDeployment) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.Template.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

api/v1alpha1/runnerreplicaset_webhook.go

@@ -66,15 +66,7 @@ func (r *RunnerReplicaSet) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *RunnerReplicaSet) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.Template.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

api/v1alpha1/runnerset_types.go

@@ -33,6 +33,12 @@ type RunnerSetSpec struct {
 	// +nullable
 	EffectiveTime *metav1.Time `json:"effectiveTime,omitempty"`
 
+	// +optional
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
+	// +optional
+	WorkVolumeClaimTemplate *WorkVolumeClaimTemplate `json:"workVolumeClaimTemplate,omitempty"`
+
 	appsv1.StatefulSetSpec `json:",inline"`
 }
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -90,6 +90,22 @@ func (in *CheckRunSpec) DeepCopy() *CheckRunSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GitHubAPICredentialsFrom) DeepCopyInto(out *GitHubAPICredentialsFrom) {
+	*out = *in
+	out.SecretRef = in.SecretRef
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubAPICredentialsFrom.
+func (in *GitHubAPICredentialsFrom) DeepCopy() *GitHubAPICredentialsFrom {
+	if in == nil {
+		return nil
+	}
+	out := new(GitHubAPICredentialsFrom)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTriggerSpec) {
 	*out = *in
@@ -231,6 +247,11 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
@@ -425,6 +446,11 @@ func (in *RunnerConfig) DeepCopyInto(out *RunnerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerConfig.
@@ -741,6 +767,11 @@ func (in *RunnerPodSpec) DeepCopyInto(out *RunnerPodSpec) {
 		*out = new(v1.PodDNSConfig)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.WorkVolumeClaimTemplate != nil {
+		in, out := &in.WorkVolumeClaimTemplate, &out.WorkVolumeClaimTemplate
+		*out = new(WorkVolumeClaimTemplate)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerPodSpec.
@@ -939,6 +970,11 @@ func (in *RunnerSetSpec) DeepCopyInto(out *RunnerSetSpec) {
 		in, out := &in.EffectiveTime, &out.EffectiveTime
 		*out = (*in).DeepCopy()
 	}
+	if in.WorkVolumeClaimTemplate != nil {
+		in, out := &in.WorkVolumeClaimTemplate, &out.WorkVolumeClaimTemplate
+		*out = new(WorkVolumeClaimTemplate)
+		(*in).DeepCopyInto(*out)
+	}
 	in.StatefulSetSpec.DeepCopyInto(&out.StatefulSetSpec)
 }
 
@@ -1126,6 +1162,42 @@ func (in *ScheduledOverride) DeepCopy() *ScheduledOverride {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretReference)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WorkVolumeClaimTemplate) DeepCopyInto(out *WorkVolumeClaimTemplate) {
+	*out = *in
+	if in.AccessModes != nil {
+		in, out := &in.AccessModes, &out.AccessModes
+		*out = make([]v1.PersistentVolumeAccessMode, len(*in))
+		copy(*out, *in)
+	}
+	in.Resources.DeepCopyInto(&out.Resources)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkVolumeClaimTemplate.
+func (in *WorkVolumeClaimTemplate) DeepCopy() *WorkVolumeClaimTemplate {
+	if in == nil {
+		return nil
+	}
+	out := new(WorkVolumeClaimTemplate)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WorkflowJobSpec) DeepCopyInto(out *WorkflowJobSpec) {
 	*out = *in

build/version.go

@@ -0,0 +1,4 @@
+package build
+
+// This is overridden at build-time using go-build ldflags. dev is the fallback value
+var Version = "NA"

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.17.3
+version: 0.21.0
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.22.3
+appVersion: 0.26.0
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/README.md

@@ -8,103 +8,105 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 
 > _Default values are the defaults set in the charts `values.yaml`, some properties have default configurations in the code for when the property is omitted or invalid_
 
-| Key                                                      | Description                                                                                                                | Default                                                              |
-|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------|
-| `labels`                                                 | Set labels to apply to all resources in the chart                                                                          |                                                                      |
-| `replicaCount`                                           | Set the number of controller pods                                                                                          | 1                                                                    |
-| `syncPeriod`                                             | Set the period in which the controler reconciles the desired runners count                                                 | 10m                                                                  |
-| `enableLeaderElection`                                   | Enable election configuration                                                                                              | true                                                                 |
-| `leaderElectionId`                                       | Set the election ID for the controller group                                                                               |                                                                      |
-| `githubEnterpriseServerURL`                              | Set the URL for a self-hosted GitHub Enterprise Server                                                                     |                                                                      |
-| `githubURL`                                              | Override GitHub URL to be used for GitHub API calls                                                                        |                                                                      |
-| `githubUploadURL`                                        | Override GitHub Upload URL to be used for GitHub API calls                                                                 |                                                                      |
-| `runnerGithubURL`                                        | Override GitHub URL to be used by runners during registration                                                              |                                                                      |
-| `logLevel`                                               | Set the log level of the controller container                                                                              |                                                                      |
-| `additionalVolumes`                                      | Set additional volumes to add to the manager container                                                                     |                                                                      |
-| `additionalVolumeMounts`                                 | Set additional volume mounts to add to the manager container                                                               |                                                                      |
-| `authSecret.create`                                      | Deploy the controller auth secret                                                                                          | false                                                                |
-| `authSecret.name`                                        | Set the name of the auth secret                                                                                            | controller-manager                                                   |
-| `authSecret.annotations`                                 | Set annotations for the auth Secret                                                                                        |                                                                      |
-| `authSecret.github_app_id`                               | The ID of your GitHub App. **This can't be set at the same time as `authSecret.github_token`**                             |                                                                      |
-| `authSecret.github_app_installation_id`                  | The ID of your GitHub App installation. **This can't be set at the same time as `authSecret.github_token`**                |                                                                      |
-| `authSecret.github_app_private_key`                      | The multiline string of your GitHub App's private key. **This can't be set at the same time as `authSecret.github_token`** |                                                                      |
-| `authSecret.github_token`                                | Your chosen GitHub PAT token. **This can't be set at the same time as the `authSecret.github_app_*`**                      |                                                                      |
-| `authSecret.github_basicauth_username`                     | Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                      |
-| `authSecret.github_basicauth_password`                     | Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                      |
-| `dockerRegistryMirror`                                   | The default Docker Registry Mirror used by runners.                                                                        |                                                                      |
-| `hostNetwork`                                            | The "hostNetwork" of the controller container                                                                              | false                                                                |
-| `image.repository`                                       | The "repository/image" of the controller container                                                                         | summerwind/actions-runner-controller                                 |
-| `image.tag`                                              | The tag of the controller container                                                                                        |                                                                      |
-| `image.actionsRunnerRepositoryAndTag`                    | The "repository/image" of the actions runner container                                                                     | summerwind/actions-runner:latest                                     |
-| `image.actionsRunnerImagePullSecrets`                    | Optional image pull secrets to be included in the runner pod's ImagePullSecrets                                            |                                                                      |
-| `image.dindSidecarRepositoryAndTag`                      | The "repository/image" of the dind sidecar container                                                                       | docker:dind                                                          |
-| `image.pullPolicy`                                       | The pull policy of the controller image                                                                                    | IfNotPresent                                                         |
-| `metrics.serviceMonitor`                                 | Deploy serviceMonitor kind for for use with prometheus-operator CRDs                                                       | false                                                                |
-| `metrics.serviceAnnotations`                             | Set annotations for the provisioned metrics service resource                                                               |                                                                      |
-| `metrics.port`                                           | Set port of metrics service                                                                                                | 8443                                                                 |
-| `metrics.proxy.enabled`                                  | Deploy kube-rbac-proxy container in controller pod                                                                         | true                                                                 |
-| `metrics.proxy.image.repository`                         | The "repository/image" of the kube-proxy container                                                                         | quay.io/brancz/kube-rbac-proxy                                       |
-| `metrics.proxy.image.tag`                                | The tag of the kube-proxy image to use when pulling the container                                                          | v0.10.0                                                              |
-| `metrics.serviceMonitorLabels`                           | Set labels to apply to ServiceMonitor resources                                                                            |                                                                      |
-| `imagePullSecrets`                                       | Specifies the secret to be used when pulling the controller pod containers                                                 |                                                                      |
-| `fullnameOverride`                                       | Override the full resource names	                                                                                        |                                                                      |
-| `nameOverride`                                           | Override the resource name prefix	                                                                                        |                                                                      |
-| `serviceAccount.annotations`                              | Set annotations to the service account                                                                                     |                                                                      |
-| `serviceAccount.create`                                  | Deploy the controller pod under a service account                                                                          | true                                                                 |
-| `podAnnotations`                                         | Set annotations for the controller pod                                                                                     |                                                                      |
-| `podLabels`                                              | Set labels for the controller pod                                                                                          |                                                                      |
-| `serviceAccount.name`                                    | Set the name of the service account                                                                                        |                                                                      |
-| `securityContext`                                        | Set the security context for each container in the controller pod                                                          |                                                                      |
-| `podSecurityContext`                                     | Set the security context to controller pod                                                                                 |                                                                      |
-| `service.annotations`                                    | Set annotations for the provisioned webhook service resource                                                               |                                                                      |
-| `service.port`                                           | Set controller service ports                                                                                                |                                                                      |
-| `service.type`                                           | Set controller service type                                                                                               |                                                                      |
-| `topologySpreadConstraints`                              | Set the controller pod topologySpreadConstraints                                                                           |                                                                      |
-| `nodeSelector`                                           | Set the controller pod nodeSelector                                                                                        |                                                                      |
-| `resources`                                              | Set the controller pod resources                                                                                           |                                                                      |
-| `affinity`                                               | Set the controller pod affinity rules                                                                                      |                                                                      |
-| `podDisruptionBudget.enabled`                            | Enables a PDB to ensure HA of controller pods                                                                              |      false                                                           |
-| `podDisruptionBudget.minAvailable`                       | Minimum number of pods that must be available after eviction                                                               |                                                                      |
-| `podDisruptionBudget.maxUnavailable`                     | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                      |
-| `tolerations`                                            | Set the controller pod tolerations                                                                                         |                                                                      |
-| `env`                                                    | Set environment variables for the controller container                                                                     |                                                                      |
-| `priorityClassName`                                      | Set the controller pod priorityClassName                                                                                   |                                                                      |
-| `scope.watchNamespace`                                   | Tells the controller and the github webhook server which namespace to watch if `scope.singleNamespace` is true             | `Release.Namespace` (the default namespace of the helm chart).       |
-| `scope.singleNamespace`                                  | Limit the controller to watch a single namespace                                                                           | false                                                                |
-| `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                   | true                                                                 |
-| `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                  |                                                                      |
-| `githubWebhookServer.logLevel`                           | Set the log level of the githubWebhookServer container                                                                     |                                                                      |
-| `githubWebhookServer.replicaCount`                       | Set the number of webhook server pods                                                                                      | 1                                                                    |
-| `githubWebhookServer.useRunnerGroupsVisibility`          | Enable supporting runner groups with custom visibility. This will incur in extra API calls and may blow up your budget. Currently, you also need to set `githubWebhookServer.secret.enabled` to enable this feature. | false                                                                |
-| `githubWebhookServer.syncPeriod`                         | Set the period in which the controller reconciles the resources                                                            | 10m                                                                  |
-| `githubWebhookServer.enabled`                            | Deploy the webhook server pod                                                                                              | false                                                                |
-| `githubWebhookServer.secret.enabled`                      | Passes the webhook hook secret to the github-webhook-server                                                                             | false                                                                |
-| `githubWebhookServer.secret.create`                      | Deploy the webhook hook secret                                                                                             | false                                                                |
-| `githubWebhookServer.secret.name`                        | Set the name of the webhook hook secret                                                                                    | github-webhook-server                                                |
-| `githubWebhookServer.secret.github_webhook_secret_token` | Set the webhook secret token value                                                                                         |                                                                      |
-| `githubWebhookServer.imagePullSecrets`                   | Specifies the secret to be used when pulling the githubWebhookServer pod containers                                        |                                                                      |
-| `githubWebhookServer.nameOverride`                        | Override the resource name prefix	                                                                                        |                                                                      |
-| `githubWebhookServer.fullnameOverride`                    | Override the full resource names	                                                                                        |                                                                      |
-| `githubWebhookServer.serviceAccount.create`              | Deploy the githubWebhookServer under a service account                                                                     | true                                                                 |
-| `githubWebhookServer.serviceAccount.annotations`         | Set annotations for the service account                                                                                    |                                                                      |
-| `githubWebhookServer.serviceAccount.name`                | Set the service account name                                                                                               |                                                                      |
-| `githubWebhookServer.podAnnotations`                     | Set annotations for the githubWebhookServer pod                                                                            |                                                                      |
-| `githubWebhookServer.podLabels`                          | Set labels for the githubWebhookServer pod                                                                                 |                                                                      |
-| `githubWebhookServer.podSecurityContext`                 | Set the security context to githubWebhookServer pod                                                                        |                                                                      |
-| `githubWebhookServer.securityContext`                    | Set the security context for each container in the githubWebhookServer pod                                                 |                                                                      |
-| `githubWebhookServer.resources`                          | Set the githubWebhookServer pod resources                                                                                  |                                                                      |
-| `githubWebhookServer.topologySpreadConstraints`          | Set the githubWebhookServer pod topologySpreadConstraints                                                                  |                                                                      |
-| `githubWebhookServer.nodeSelector`                       | Set the githubWebhookServer pod nodeSelector                                                                               |                                                                      |
-| `githubWebhookServer.tolerations`                        | Set the githubWebhookServer pod tolerations                                                                                |                                                                      |
-| `githubWebhookServer.affinity`                           | Set the githubWebhookServer pod affinity rules                                                                             |                                                                      |
-| `githubWebhookServer.priorityClassName`                  | Set the githubWebhookServer pod priorityClassName                                                                          |                                                                      |
-| `githubWebhookServer.service.type`                       | Set githubWebhookServer service type                                                                                       |                                                                      |
-| `githubWebhookServer.service.ports`                      | Set githubWebhookServer service ports                                                                                      | `[{"port":80, "targetPort:"http", "protocol":"TCP", "name":"http"}]` |
-| `githubWebhookServer.ingress.enabled`                    | Deploy an ingress kind for the githubWebhookServer                                                                         | false                                                                |
-| `githubWebhookServer.ingress.annotations`                | Set annotations for the ingress kind                                                                                       |                                                                      |
-| `githubWebhookServer.ingress.hosts`                      | Set hosts configuration for ingress                                                                                        | `[{"host": "chart-example.local", "paths": []}]`                     |
-| `githubWebhookServer.ingress.tls`                        | Set tls configuration for ingress                                                                                          |                                                                      |
-| `githubWebhookServer.ingress.ingressClassName`           | Set ingress class name                                                                                                     |                                                                      |
-| `githubWebhookServer.podDisruptionBudget.enabled`        | Enables a PDB to ensure HA of githubwebhook pods                                                                           |      false                                                           |
-| `githubWebhookServer.podDisruptionBudget.minAvailable`   | Minimum number of pods that must be available after eviction                                                               |                                                                      |
-| `githubWebhookServer.podDisruptionBudget.maxUnavailable` | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                      |
+| Key                                                      | Description                                                                                                                | Default                                                                                         |
+|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|
+| `labels`                                                 | Set labels to apply to all resources in the chart                                                                          |                                                                                                 |
+| `replicaCount`                                           | Set the number of controller pods                                                                                          | 1                                                                                               |
+| `webhookPort`                                            | Set the containerPort for the webhook Pod                                                                                  | 9443                                                                                            |
+| `syncPeriod`                                             | Set the period in which the controler reconciles the desired runners count                                                 | 10m                                                                                             |
+| `enableLeaderElection`                                   | Enable election configuration                                                                                              | true                                                                                            |
+| `leaderElectionId`                                       | Set the election ID for the controller group                                                                               |                                                                                                 |
+| `githubEnterpriseServerURL`                              | Set the URL for a self-hosted GitHub Enterprise Server                                                                     |                                                                                                 |
+| `githubURL`                                              | Override GitHub URL to be used for GitHub API calls                                                                        |                                                                                                 |
+| `githubUploadURL`                                        | Override GitHub Upload URL to be used for GitHub API calls                                                                 |                                                                                                 |
+| `runnerGithubURL`                                        | Override GitHub URL to be used by runners during registration                                                              |                                                                                                 |
+| `logLevel`                                               | Set the log level of the controller container                                                                              |                                                                                                 |
+| `additionalVolumes`                                      | Set additional volumes to add to the manager container                                                                     |                                                                                                 |
+| `additionalVolumeMounts`                                 | Set additional volume mounts to add to the manager container                                                               |                                                                                                 |
+| `authSecret.create`                                      | Deploy the controller auth secret                                                                                          | false                                                                                           |
+| `authSecret.name`                                        | Set the name of the auth secret                                                                                            | controller-manager                                                                              |
+| `authSecret.annotations`                                 | Set annotations for the auth Secret                                                                                        |                                                                                                 |
+| `authSecret.github_app_id`                               | The ID of your GitHub App. **This can't be set at the same time as `authSecret.github_token`**                             |                                                                                                 |
+| `authSecret.github_app_installation_id`                  | The ID of your GitHub App installation. **This can't be set at the same time as `authSecret.github_token`**                |                                                                                                 |
+| `authSecret.github_app_private_key`                      | The multiline string of your GitHub App's private key. **This can't be set at the same time as `authSecret.github_token`** |                                                                                                 |
+| `authSecret.github_token`                                | Your chosen GitHub PAT token. **This can't be set at the same time as the `authSecret.github_app_*`**                      |                                                                                                 |
+| `authSecret.github_basicauth_username`                   | Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                                                 |
+| `authSecret.github_basicauth_password`                   | Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                                                 |
+| `dockerRegistryMirror`                                   | The default Docker Registry Mirror used by runners.                                                                        |                                                                                                 |
+| `hostNetwork`                                            | The "hostNetwork" of the controller container                                                                              | false                                                                                           |
+| `image.repository`                                       | The "repository/image" of the controller container                                                                         | summerwind/actions-runner-controller                                                            |
+| `image.tag`                                              | The tag of the controller container                                                                                        |                                                                                                 |
+| `image.actionsRunnerRepositoryAndTag`                    | The "repository/image" of the actions runner container                                                                     | summerwind/actions-runner:latest                                                                |
+| `image.actionsRunnerImagePullSecrets`                    | Optional image pull secrets to be included in the runner pod's ImagePullSecrets                                            |                                                                                                 |
+| `image.dindSidecarRepositoryAndTag`                      | The "repository/image" of the dind sidecar container                                                                       | docker:dind                                                                                     |
+| `image.pullPolicy`                                       | The pull policy of the controller image                                                                                    | IfNotPresent                                                                                    |
+| `metrics.serviceMonitor`                                 | Deploy serviceMonitor kind for for use with prometheus-operator CRDs                                                       | false                                                                                           |
+| `metrics.serviceAnnotations`                             | Set annotations for the provisioned metrics service resource                                                               |                                                                                                 |
+| `metrics.port`                                           | Set port of metrics service                                                                                                | 8443                                                                                            |
+| `metrics.proxy.enabled`                                  | Deploy kube-rbac-proxy container in controller pod                                                                         | true                                                                                            |
+| `metrics.proxy.image.repository`                         | The "repository/image" of the kube-proxy container                                                                         | quay.io/brancz/kube-rbac-proxy                                                                  |
+| `metrics.proxy.image.tag`                                | The tag of the kube-proxy image to use when pulling the container                                                          | v0.10.0                                                                                         |
+| `metrics.serviceMonitorLabels`                           | Set labels to apply to ServiceMonitor resources                                                                            |                                                                                                 |
+| `imagePullSecrets`                                       | Specifies the secret to be used when pulling the controller pod containers                                                 |                                                                                                 |
+| `fullnameOverride`                                       | Override the full resource names	                                                                                        |                                                                                                 |
+| `nameOverride`                                           | Override the resource name prefix	                                                                                        |                                                                                                 |
+| `serviceAccount.annotations`                             | Set annotations to the service account                                                                                     |                                                                                                 |
+| `serviceAccount.create`                                  | Deploy the controller pod under a service account                                                                          | true                                                                                            |
+| `podAnnotations`                                         | Set annotations for the controller pod                                                                                     |                                                                                                 |
+| `podLabels`                                              | Set labels for the controller pod                                                                                          |                                                                                                 |
+| `serviceAccount.name`                                    | Set the name of the service account                                                                                        |                                                                                                 |
+| `securityContext`                                        | Set the security context for each container in the controller pod                                                          |                                                                                                 |
+| `podSecurityContext`                                     | Set the security context to controller pod                                                                                 |                                                                                                 |
+| `service.annotations`                                    | Set annotations for the provisioned webhook service resource                                                               |                                                                                                 |
+| `service.port`                                           | Set controller service ports                                                                                               |                                                                                                 |
+| `service.type`                                           | Set controller service type                                                                                                |                                                                                                 |
+| `topologySpreadConstraints`                              | Set the controller pod topologySpreadConstraints                                                                           |                                                                                                 |
+| `nodeSelector`                                           | Set the controller pod nodeSelector                                                                                        |                                                                                                 |
+| `resources`                                              | Set the controller pod resources                                                                                           |                                                                                                 |
+| `affinity`                                               | Set the controller pod affinity rules                                                                                      |                                                                                                 |
+| `podDisruptionBudget.enabled`                            | Enables a PDB to ensure HA of controller pods                                                                              |      false                                                                                      |
+| `podDisruptionBudget.minAvailable`                       | Minimum number of pods that must be available after eviction                                                               |                                                                                                 |
+| `podDisruptionBudget.maxUnavailable`                     | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                                                 |
+| `tolerations`                                            | Set the controller pod tolerations                                                                                         |                                                                                                 |
+| `env`                                                    | Set environment variables for the controller container                                                                     |                                                                                                 |
+| `priorityClassName`                                      | Set the controller pod priorityClassName                                                                                   |                                                                                                 |
+| `scope.watchNamespace`                                   | Tells the controller and the github webhook server which namespace to watch if `scope.singleNamespace` is true             | `Release.Namespace` (the default namespace of the helm chart).                                  |
+| `scope.singleNamespace`                                  | Limit the controller to watch a single namespace                                                                           | false                                                                                           |
+| `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                   | true                                                                                            |
+| `runner.statusUpdateHook.enabled`                        | Use custom RBAC for runners (role, role binding and service account), this will enable reporting runner statuses           | false                                                                                           |
+| `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                  |                                                                                                 |
+| `githubWebhookServer.logLevel`                           | Set the log level of the githubWebhookServer container                                                                     |                                                                                                 |
+| `githubWebhookServer.replicaCount`                       | Set the number of webhook server pods                                                                                      | 1                                                                                               |
+| `githubWebhookServer.useRunnerGroupsVisibility`          | Enable supporting runner groups with custom visibility. This will incur in extra API calls and may blow up your budget. Currently, you also need to set `githubWebhookServer.secret.enabled` to enable this feature. | false |
+| `githubWebhookServer.enabled`                            | Deploy the webhook server pod                                                                                              | false                                                                                           |
+| `githubWebhookServer.queueLimit`                         | Set the queue size limit in the githubWebhookServer                                                                        |                                                                                                 |
+| `githubWebhookServer.secret.enabled`                     | Passes the webhook hook secret to the github-webhook-server                                                                | false                                                                                           |
+| `githubWebhookServer.secret.create`                      | Deploy the webhook hook secret                                                                                             | false                                                                                           |
+| `githubWebhookServer.secret.name`                        | Set the name of the webhook hook secret                                                                                    | github-webhook-server                                                                           |
+| `githubWebhookServer.secret.github_webhook_secret_token` | Set the webhook secret token value                                                                                         |                                                                                                 |
+| `githubWebhookServer.imagePullSecrets`                   | Specifies the secret to be used when pulling the githubWebhookServer pod containers                                        |                                                                                                 |
+| `githubWebhookServer.nameOverride`                       | Override the resource name prefix	                                                                                        |                                                                                                 |
+| `githubWebhookServer.fullnameOverride`                   | Override the full resource names	                                                                                        |                                                                                                 |
+| `githubWebhookServer.serviceAccount.create`              | Deploy the githubWebhookServer under a service account                                                                     | true                                                                                            |
+| `githubWebhookServer.serviceAccount.annotations`         | Set annotations for the service account                                                                                    |                                                                                                 |
+| `githubWebhookServer.serviceAccount.name`                | Set the service account name                                                                                               |                                                                                                 |
+| `githubWebhookServer.podAnnotations`                     | Set annotations for the githubWebhookServer pod                                                                            |                                                                                                 |
+| `githubWebhookServer.podLabels`                          | Set labels for the githubWebhookServer pod                                                                                 |                                                                                                 |
+| `githubWebhookServer.podSecurityContext`                 | Set the security context to githubWebhookServer pod                                                                        |                                                                                                 |
+| `githubWebhookServer.securityContext`                    | Set the security context for each container in the githubWebhookServer pod                                                 |                                                                                                 |
+| `githubWebhookServer.resources`                          | Set the githubWebhookServer pod resources                                                                                  |                                                                                                 |
+| `githubWebhookServer.topologySpreadConstraints`          | Set the githubWebhookServer pod topologySpreadConstraints                                                                  |                                                                                                 |
+| `githubWebhookServer.nodeSelector`                       | Set the githubWebhookServer pod nodeSelector                                                                               |                                                                                                 |
+| `githubWebhookServer.tolerations`                        | Set the githubWebhookServer pod tolerations                                                                                |                                                                                                 |
+| `githubWebhookServer.affinity`                           | Set the githubWebhookServer pod affinity rules                                                                             |                                                                                                 |
+| `githubWebhookServer.priorityClassName`                  | Set the githubWebhookServer pod priorityClassName                                                                          |                                                                                                 |
+| `githubWebhookServer.service.type`                       | Set githubWebhookServer service type                                                                                       |                                                                                                 |
+| `githubWebhookServer.service.ports`                      | Set githubWebhookServer service ports                                                                                      | `[{"port":80, "targetPort:"http", "protocol":"TCP", "name":"http"}]`                            |
+| `githubWebhookServer.ingress.enabled`                    | Deploy an ingress kind for the githubWebhookServer                                                                         | false                                                                                           |
+| `githubWebhookServer.ingress.annotations`                | Set annotations for the ingress kind                                                                                       |                                                                                                 |
+| `githubWebhookServer.ingress.hosts`                      | Set hosts configuration for ingress                                                                                        | `[{"host": "chart-example.local", "paths": []}]`                                                |
+| `githubWebhookServer.ingress.tls`                        | Set tls configuration for ingress                                                                                          |                                                                                                 |
+| `githubWebhookServer.ingress.ingressClassName`           | Set ingress class name                                                                                                     |                                                                                                 |
+| `githubWebhookServer.podDisruptionBudget.enabled`        | Enables a PDB to ensure HA of githubwebhook pods                                                                           | false                                                                                           |
+| `githubWebhookServer.podDisruptionBudget.minAvailable`   | Minimum number of pods that must be available after eviction                                                               |                                                                                                 |
+| `githubWebhookServer.podDisruptionBudget.maxUnavailable` | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                                                 |

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer
@@ -92,7 +102,7 @@ spec:
                         description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up.
                         type: string
                       type:
-                        description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                        description: Type is the type of metric to be used for autoscaling. It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
                         type: string
                     type: object
                   type: array
@@ -170,7 +180,7 @@ spec:
                 scheduledOverrides:
                   description: ScheduledOverrides is the list of ScheduledOverride. It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. The earlier a scheduled override is, the higher it is prioritized.
                   items:
-                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
                     properties:
                       endTime:
                         description: EndTime is the time at which the first override ends.

charts/actions-runner-controller/templates/controller.metrics.serviceMonitor.yaml

@@ -8,6 +8,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   name: {{ include "actions-runner-controller.serviceMonitorName" . }}
+  namespace: {{ .Release.Namespace }}
 spec:
   endpoints:
     - path: /metrics

charts/actions-runner-controller/templates/deployment.yaml

@@ -44,6 +44,7 @@ spec:
         {{- if .Values.leaderElectionId }}
         - "--leader-election-id={{ .Values.leaderElectionId }}"
         {{- end }}
+        - "--port={{ .Values.webhookPort }}"
         - "--sync-period={{ .Values.syncPeriod }}"
         - "--default-scale-down-delay={{ .Values.defaultScaleDownDelay }}"
         - "--docker-image={{ .Values.image.dindSidecarRepositoryAndTag }}"
@@ -57,15 +58,15 @@ spec:
         {{- if .Values.scope.singleNamespace }}
         - "--watch-namespace={{ default .Release.Namespace .Values.scope.watchNamespace }}"
         {{- end }}
-        {{- if .Values.githubAPICacheDuration }}
-        - "--github-api-cache-duration={{ .Values.githubAPICacheDuration }}"
-        {{- end }}
         {{- if .Values.logLevel }}
         - "--log-level={{ .Values.logLevel }}"
         {{- end }}
         {{- if .Values.runnerGithubURL  }}
         - "--runner-github-url={{ .Values.runnerGithubURL }}"
         {{- end }}
+        {{- if .Values.runner.statusUpdateHook.enabled }}
+        - "--runner-status-update-hook"
+        {{- end }}
         command:
         - "/manager"
         env:
@@ -117,15 +118,19 @@ spec:
               name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         {{- end }}
+        {{- if kindIs "slice" .Values.env }}
+        {{- toYaml .Values.env | nindent 8 }}
+        {{- else }}
         {{- range $key, $val := .Values.env }}
         - name: {{ $key }}
           value: {{ $val | quote }}
         {{- end }}
+        {{- end }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
         name: manager
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         ports:
-        - containerPort: 9443
+        - containerPort: {{ .Values.webhookPort }}
           name: webhook-server
           protocol: TCP
         {{- if not .Values.metrics.proxy.enabled }}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -39,7 +39,6 @@ spec:
         {{- $metricsHost := .Values.metrics.proxy.enabled | ternary "127.0.0.1" "0.0.0.0" }}
         {{- $metricsPort := .Values.metrics.proxy.enabled | ternary "8080" .Values.metrics.port }}
         - "--metrics-addr={{ $metricsHost }}:{{ $metricsPort }}"
-        - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}"
         {{- if .Values.githubWebhookServer.logLevel }}
         - "--log-level={{ .Values.githubWebhookServer.logLevel }}"
         {{- end }}
@@ -49,6 +48,9 @@ spec:
         {{- if .Values.runnerGithubURL  }}
         - "--runner-github-url={{ .Values.runnerGithubURL }}"
         {{- end }}
+        {{- if .Values.githubWebhookServer.queueLimit }}
+        - "--queue-limit={{ .Values.githubWebhookServer.queueLimit }}"
+        {{- end }}
         command:
         - "/github-webhook-server"
         env:

charts/actions-runner-controller/templates/githubwebhook.ingress.yaml

@@ -1,16 +1,11 @@
 {{- if .Values.githubWebhookServer.ingress.enabled -}}
 {{- $fullName := include "actions-runner-controller-github-webhook-server.fullname" . -}}
 {{- $svcPort := (index .Values.githubWebhookServer.service.ports 0).port -}}
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
 apiVersion: networking.k8s.io/v1
-{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
-apiVersion: networking.k8s.io/v1beta1
-{{- else if .Capabilities.APIVersions.Has "extensions/v1beta1" }}
-apiVersion: extensions/v1beta1
-{{- end }}
 kind: Ingress
 metadata:
   name: {{ $fullName }}
+  namespace: {{ .Release.Namespace }}
   labels:
     {{- include "actions-runner-controller.labels" . | nindent 4 }}
   {{- with .Values.githubWebhookServer.ingress.annotations }}
@@ -36,21 +31,17 @@ spec:
     - host: {{ .host | quote }}
       http:
         paths:
+          {{- if .extraPaths }}
+          {{- toYaml .extraPaths | nindent 10 }}
+          {{- end }}
           {{- range .paths }}
           - path: {{ .path }}
-            {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
             pathType: {{ .pathType }}
-            {{- end }}
             backend:
-              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
               service:
                name: {{ $fullName }}
                port:
                  number: {{ $svcPort }}
-              {{- else }}
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-              {{- end }}
           {{- end }}
     {{- end }}
   {{- end }}

charts/actions-runner-controller/templates/githubwebhook.secrets.yaml

@@ -12,5 +12,17 @@ data:
 {{- if .Values.githubWebhookServer.secret.github_webhook_secret_token }}
   github_webhook_secret_token: {{ .Values.githubWebhookServer.secret.github_webhook_secret_token | toString | b64enc }}
 {{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_id }}
+  github_app_id: {{ .Values.githubWebhookServer.secret.github_app_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_installation_id }}
+  github_app_installation_id: {{ .Values.githubWebhookServer.secret.github_app_installation_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_private_key }}
+  github_app_private_key: {{ .Values.githubWebhookServer.secret.github_app_private_key | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_token }}
+  github_token: {{ .Values.githubWebhookServer.secret.github_token | toString | b64enc }}
+{{- end }}
 {{- end }}
 {{- end }}

charts/actions-runner-controller/templates/githubwebhook.serviceMonitor.yaml

@@ -8,6 +8,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   name: {{ include "actions-runner-controller-github-webhook-server.serviceMonitorName" . }}
+  namespace: {{ .Release.Namespace }}
 spec:
   endpoints:
     - path: /metrics

charts/actions-runner-controller/templates/manager_role.yaml

@@ -195,6 +195,28 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - coordination.k8s.io
   resources:
@@ -228,3 +250,72 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+{{- if .Values.runner.statusUpdateHook.enabled }}
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get
+{{- end }}
+{{- if .Values.rbac.allowGrantingKubernetesContainerModePermissions }}
+{{/* These permissions are required by ARC to create RBAC resources for the runner pod to use the kubernetes container mode. */}}
+{{/* See https://github.com/actions-runner-controller/actions-runner-controller/pull/1268/files#r917331632 */}}
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/log
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "batch"
+  resources:
+  - jobs
+  verbs:
+  - get
+  - list
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+{{- end }}

charts/actions-runner-controller/templates/webhook_service.yaml

@@ -13,7 +13,7 @@ spec:
   type: {{ .Values.service.type }}
   ports:
     - port: 443
-      targetPort: 9443
+      targetPort: {{ .Values.webhookPort }}
       protocol: TCP
       name: https
   selector:

charts/actions-runner-controller/values.yaml

@@ -6,6 +6,7 @@ labels: {}
 
 replicaCount: 1
 
+webhookPort: 9443
 syncPeriod: 1m
 defaultScaleDownDelay: 10m
 
@@ -14,12 +15,6 @@ enableLeaderElection: true
 # Must be unique if more than one controller installed onto the same namespace.
 #leaderElectionId: "actions-runner-controller"
 
-# DEPRECATED: This has been removed as unnecessary in #1192
-# The controller tries its best not to repeat the duplicate GitHub API call
-# within this duration.
-# Defaults to syncPeriod - 10s.
-#githubAPICacheDuration: 30s
-
 # The URL of your GitHub Enterprise server, if you're using one.
 #githubEnterpriseServerURL: https://github.example.com
 
@@ -66,6 +61,18 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+runner:
+  statusUpdateHook:
+    enabled: false
+
+rbac:
+  {}
+  # # This allows ARC to dynamically create a ServiceAccount and a Role for each Runner pod that uses "kubernetes" container mode,
+  # # by extending ARC's manager role to have the same permissions required by the pod runs the runner agent in "kubernetes" container mode.
+  # # Without this, Kubernetes blocks ARC to create the role to prevent a priviledge escalation.
+  # # See https://github.com/actions-runner-controller/actions-runner-controller/pull/1268/files#r917327010
+  # allowGrantingKubernetesContainerModePermissions: true
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true
@@ -108,7 +115,7 @@ metrics:
     enabled: true
     image:
       repository: quay.io/brancz/kube-rbac-proxy
-      tag: v0.11.0
+      tag: v0.13.0
 
 resources:
   {}
@@ -142,10 +149,20 @@ priorityClassName: ""
 
 env:
   {}
+  # specify additional environment variables for the controller pod.
+  # It's possible to specify either key vale pairs e.g.:
   # http_proxy: "proxy.com:8080"
   # https_proxy: "proxy.com:8080"
   # no_proxy: ""
 
+  # or a list of complete environment variable definitions e.g.:
+  # - name: GITHUB_APP_INSTALLATION_ID
+  #   valueFrom:
+  #     secretKeyRef:
+  #       key: some_key_in_the_secret
+  #       name: some-secret-name
+  #       optional: true
+
 ## specify additional volumes to mount in the manager container, this can be used
 ## to specify additional storage of material or to inject files from ConfigMaps
 ## into the running container
@@ -174,7 +191,6 @@ admissionWebHooks:
 githubWebhookServer:
   enabled: false
   replicaCount: 1
-  syncPeriod: 10m
   useRunnerGroupsVisibility: false
   secret:
     enabled: false
@@ -182,6 +198,13 @@ githubWebhookServer:
     name: "github-webhook-server"
     ### GitHub Webhook Configuration
     github_webhook_secret_token: ""
+    ### GitHub Apps Configuration
+    ## NOTE: IDs MUST be strings, use quotes
+    #github_app_id: ""
+    #github_app_installation_id: ""
+    #github_app_private_key: |
+    ### GitHub PAT Configuration
+    #github_token: ""
   imagePullSecrets: []
   nameOverride: ""
   fullnameOverride: ""
@@ -223,6 +246,20 @@ githubWebhookServer:
         paths: []
         #  - path: /*
         #    pathType: ImplementationSpecific
+        # Extra paths that are not automatically connected to the server. This is useful when working with annotation based services.
+        extraPaths: []
+        # - path: /*
+        #   backend:
+        #     serviceName: ssl-redirect
+        #     servicePort: use-annotation
+        ## for Kubernetes >=1.19 (when "networking.k8s.io/v1" is used)
+        # - path: /*
+        #   pathType: Prefix
+        #   backend:
+        #     service:
+        #       name: ssl-redirect
+        #       port:
+        #         name: use-annotation
     tls: []
     #  - secretName: chart-example-tls
     #    hosts:
@@ -233,3 +270,4 @@ githubWebhookServer:
     enabled: false
     # minAvailable: 1
     # maxUnavailable: 3
+  # queueLimit: 100

cmd/githubwebhookserver/main.go

@@ -69,9 +69,8 @@ func main() {
 
 		watchNamespace string
 
-		enableLeaderElection bool
-		syncPeriod           time.Duration
-		logLevel             string
+		logLevel   string
+		queueLimit int
 
 		ghClient *github.Client
 	)
@@ -88,10 +87,8 @@ func main() {
 	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&watchNamespace, "watch-namespace", "", "The namespace to watch for HorizontalRunnerAutoscaler's to scale on Webhook. Set to empty for letting it watch for all namespaces.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
-		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
-	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
+	flag.IntVar(&queueLimit, "queue-limit", controllers.DefaultQueueLimit, `The maximum length of the scale operation queue. The scale opration is enqueued per every matching webhook event, and the server returns a 500 HTTP status when the queue was already full on enqueue attempt.`)
 	flag.StringVar(&webhookSecretToken, "github-webhook-secret-token", "", "The personal access token of GitHub.")
 	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
 	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
@@ -142,10 +139,10 @@ func main() {
 		setupLog.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls")
 	}
 
+	syncPeriod := 10 * time.Minute
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		SyncPeriod:         &syncPeriod,
-		LeaderElection:     enableLeaderElection,
 		Namespace:          watchNamespace,
 		MetricsBindAddress: metricsAddr,
 		Port:               9443,
@@ -164,6 +161,7 @@ func main() {
 		SecretKeyBytes: []byte(webhookSecretToken),
 		Namespace:      watchNamespace,
 		GitHubClient:   ghClient,
+		QueueLimit:     queueLimit,
 	}
 
 	if err = hraGitHubWebhook.SetupWithManager(mgr); err != nil {

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer
@@ -92,7 +102,7 @@ spec:
                         description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up.
                         type: string
                       type:
-                        description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                        description: Type is the type of metric to be used for autoscaling. It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
                         type: string
                     type: object
                   type: array
@@ -170,7 +180,7 @@ spec:
                 scheduledOverrides:
                   description: ScheduledOverrides is the list of ScheduledOverride. It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. The earlier a scheduled override is, the higher it is prioritized.
                   items:
-                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
                     properties:
                       endTime:
                         description: EndTime is the time at which the first override ends.

config/default/kustomization.yaml

@@ -22,8 +22,6 @@ bases:
 - ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
-# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
-#- ../github-webhook-server
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
@@ -46,10 +44,6 @@ patchesStrategicMerge:
 # 'CERTMANAGER' needs to be enabled to use ca injection
 - webhookcainjection_patch.yaml
 
-# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
-# Protect the GitHub webhook server metrics endpoint by putting it behind auth.
-# - gh-webhook-server-auth-proxy-patch.yaml
-
 # the following config is for teaching kustomize how to do var substitution
 vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.

config/github-webhook-server/kustomization.yaml

@@ -2,11 +2,14 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 images:
-  - name: controller
-    newName: summerwind/actions-runner-controller
-    newTag: latest
+- name: controller
+  newName: summerwind/actions-runner-controller
+  newTag: latest
 
 resources:
-  - deployment.yaml
-  - rbac.yaml
-  - service.yaml
+- deployment.yaml
+- rbac.yaml
+- service.yaml
+
+patchesStrategicMerge:
+- gh-webhook-server-auth-proxy-patch.yaml

config/rbac/role.yaml

@@ -202,6 +202,29 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -226,3 +249,36 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get

controllers/autoscaling.go

@@ -9,7 +9,11 @@ import (
 	"strings"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v39/github"
+	prometheus_metrics "github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
+	arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/google/go-github/v47/github"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -19,7 +23,7 @@ const (
 	defaultScaleDownFactor    = 0.7
 )
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	if hra.Spec.MinReplicas == nil {
 		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
 	} else if hra.Spec.MaxReplicas == nil {
@@ -46,9 +50,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 
 	switch primaryMetricType {
 	case v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns:
-		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &primaryMetric)
+		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &primaryMetric)
 	case v1alpha1.AutoscalingMetricTypePercentageRunnersBusy:
-		suggested, err = r.suggestReplicasByPercentageRunnersBusy(st, hra, primaryMetric)
+		suggested, err = r.suggestReplicasByPercentageRunnersBusy(ghc, st, hra, primaryMetric)
 	default:
 		return nil, fmt.Errorf("validating autoscaling metrics: unsupported metric type %q", primaryMetric)
 	}
@@ -81,11 +85,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 		)
 	}
 
-	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &fallbackMetric)
+	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &fallbackMetric)
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
-
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
 	var repos [][]string
 	repoID := st.repo
 	if repoID == "" {
@@ -124,7 +127,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 		opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}}
 		var allJobs []*github.WorkflowJob
 		for {
-			jobs, resp, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
+			jobs, resp, err := ghc.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
 			if err != nil {
 				r.Log.Error(err, "Error listing workflow jobs")
 				return //err
@@ -182,7 +185,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
-		workflowRuns, err := r.GitHubClient.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
+		workflowRuns, err := ghc.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
 		if err != nil {
 			return nil, err
 		}
@@ -209,6 +212,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	necessaryReplicas := queued + inProgress
 
+	prometheus_metrics.SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		necessaryReplicas,
+		completed,
+		inProgress,
+		queued,
+		unknown,
+	)
+
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas),
 		"workflow_runs_completed", completed,
@@ -224,7 +241,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 	return &necessaryReplicas, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
 	ctx := context.Background()
 	scaleUpThreshold := defaultScaleUpThreshold
 	scaleDownThreshold := defaultScaleDownThreshold
@@ -293,7 +310,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	)
 
 	// ListRunners will return all runners managed by GitHub - not restricted to ns
-	runners, err := r.GitHubClient.ListRunners(
+	runners, err := ghc.ListRunners(
 		ctx,
 		enterprise,
 		organization,
@@ -314,22 +331,52 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 		numRunners           int
 		numRunnersRegistered int
 		numRunnersBusy       int
+		numTerminatingBusy   int
 	)
 
 	numRunners = len(runnerMap)
 
+	busyTerminatingRunnerPods := map[string]struct{}{}
+
+	kindLabel := LabelKeyRunnerDeploymentName
+	if hra.Spec.ScaleTargetRef.Kind == "RunnerSet" {
+		kindLabel = LabelKeyRunnerSetName
+	}
+
+	var runnerPodList corev1.PodList
+	if err := r.Client.List(ctx, &runnerPodList, client.InNamespace(hra.Namespace), client.MatchingLabels(map[string]string{
+		kindLabel: hra.Spec.ScaleTargetRef.Name,
+	})); err != nil {
+		return nil, err
+	}
+
+	for _, p := range runnerPodList.Items {
+		if p.Annotations[AnnotationKeyUnregistrationFailureMessage] != "" {
+			busyTerminatingRunnerPods[p.Name] = struct{}{}
+		}
+	}
+
 	for _, runner := range runners {
 		if _, ok := runnerMap[*runner.Name]; ok {
 			numRunnersRegistered++
 
 			if runner.GetBusy() {
 				numRunnersBusy++
+			} else if _, ok := busyTerminatingRunnerPods[*runner.Name]; ok {
+				numTerminatingBusy++
 			}
+
+			delete(busyTerminatingRunnerPods, *runner.Name)
 		}
 	}
 
+	// Remaining busyTerminatingRunnerPods are runners that were not on the ListRunners API response yet
+	for range busyTerminatingRunnerPods {
+		numTerminatingBusy++
+	}
+
 	var desiredReplicas int
-	fractionBusy := float64(numRunnersBusy) / float64(desiredReplicasBefore)
+	fractionBusy := float64(numRunnersBusy+numTerminatingBusy) / float64(desiredReplicasBefore)
 	if fractionBusy >= scaleUpThreshold {
 		if scaleUpAdjustment > 0 {
 			desiredReplicas = desiredReplicasBefore + scaleUpAdjustment
@@ -350,6 +397,19 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	//
 	// - num_runners can be as twice as large as replicas_desired_before while
 	//   the runnerdeployment controller is replacing RunnerReplicaSet for runner update.
+	prometheus_metrics.SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		desiredReplicas,
+		numRunners,
+		numRunnersRegistered,
+		numRunnersBusy,
+		numTerminatingBusy,
+	)
 
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas),
@@ -358,6 +418,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 		"num_runners", numRunners,
 		"num_runners_registered", numRunnersRegistered,
 		"num_runners_busy", numRunnersBusy,
+		"num_terminating_busy", numTerminatingBusy,
 		"namespace", hra.Namespace,
 		"kind", st.kind,
 		"name", st.st,

controllers/autoscaling_test.go

@@ -330,7 +330,6 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
-				GitHubClient:          client,
 				Scheme:                scheme,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
@@ -379,7 +378,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)
@@ -720,7 +719,6 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
 				Scheme:                scheme,
-				GitHubClient:          client,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
 
@@ -781,7 +779,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)

controllers/constants.go

@@ -4,17 +4,22 @@ import "time"
 
 const (
 	LabelKeyRunnerSetName = "runnerset-name"
+	LabelKeyRunner        = "actions-runner"
 )
 
 const (
 	// This names requires at least one slash to work.
 	// See https://github.com/google/knative-gcp/issues/378
-	runnerPodFinalizerName = "actions.summerwind.dev/runner-pod"
+	runnerPodFinalizerName             = "actions.summerwind.dev/runner-pod"
+	runnerLinkedResourcesFinalizerName = "actions.summerwind.dev/linked-resources"
 
 	annotationKeyPrefix = "actions-runner/"
 
 	AnnotationKeyLastRegistrationCheckTime = "actions-runner-controller/last-registration-check-time"
 
+	// AnnotationKeyUnregistrationFailureMessage is the annotation that is added onto the pod once it failed to be unregistered from GitHub due to e.g. 422 error
+	AnnotationKeyUnregistrationFailureMessage = annotationKeyPrefix + "unregistration-failure-message"
+
 	// AnnotationKeyUnregistrationCompleteTimestamp is the annotation that is added onto the pod once the previously started unregistration process has been completed.
 	AnnotationKeyUnregistrationCompleteTimestamp = annotationKeyPrefix + "unregistration-complete-timestamp"
 
@@ -47,8 +52,6 @@ const (
 	// A pod that is timed out can be terminated if needed.
 	registrationTimeout = 10 * time.Minute
 
-	defaultRegistrationCheckInterval = time.Minute
-
 	// DefaultRunnerPodRecreationDelayAfterWebhookScale is the delay until syncing the runners with the desired replicas
 	// after a webhook-based scale up.
 	// This is used to prevent ARC from recreating completed runner pods that are deleted soon without being used at all.
@@ -63,4 +66,7 @@ const (
 
 	EnvVarRunnerName  = "RUNNER_NAME"
 	EnvVarRunnerToken = "RUNNER_TOKEN"
+
+	// defaultHookPath is path to the hook script used when the "containerMode: kubernetes" is specified
+	defaultRunnerHookPath = "/runner/k8s/index.js"
 )

controllers/horizontal_runner_autoscaler_batch_scale.go

@@ -0,0 +1,207 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/go-logr/logr"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type batchScaler struct {
+	Ctx      context.Context
+	Client   client.Client
+	Log      logr.Logger
+	interval time.Duration
+
+	queue       chan *ScaleTarget
+	workerStart sync.Once
+}
+
+func newBatchScaler(ctx context.Context, client client.Client, log logr.Logger) *batchScaler {
+	return &batchScaler{
+		Ctx:      ctx,
+		Client:   client,
+		Log:      log,
+		interval: 3 * time.Second,
+	}
+}
+
+type batchScaleOperation struct {
+	namespacedName types.NamespacedName
+	scaleOps       []scaleOperation
+}
+
+type scaleOperation struct {
+	trigger v1alpha1.ScaleUpTrigger
+	log     logr.Logger
+}
+
+// Add the scale target to the unbounded queue, blocking until the target is successfully added to the queue.
+// All the targets in the queue are dequeued every 3 seconds, grouped by the HRA, and applied.
+// In a happy path, batchScaler update each HRA only once, even though the HRA had two or more associated webhook events in the 3 seconds interval,
+// which results in less K8s API calls and less HRA update conflicts in case your ARC installation receives a lot of webhook events
+func (s *batchScaler) Add(st *ScaleTarget) {
+	if st == nil {
+		return
+	}
+
+	s.workerStart.Do(func() {
+		var expBackoff = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second}
+
+		s.queue = make(chan *ScaleTarget)
+
+		log := s.Log
+
+		go func() {
+			log.Info("Starting batch worker")
+			defer log.Info("Stopped batch worker")
+
+			for {
+				select {
+				case <-s.Ctx.Done():
+					return
+				default:
+				}
+
+				log.V(2).Info("Batch worker is dequeueing operations")
+
+				batches := map[types.NamespacedName]batchScaleOperation{}
+				after := time.After(s.interval)
+				var ops uint
+
+			batch:
+				for {
+					select {
+					case <-after:
+						after = nil
+						break batch
+					case st := <-s.queue:
+						nsName := types.NamespacedName{
+							Namespace: st.HorizontalRunnerAutoscaler.Namespace,
+							Name:      st.HorizontalRunnerAutoscaler.Name,
+						}
+						b, ok := batches[nsName]
+						if !ok {
+							b = batchScaleOperation{
+								namespacedName: nsName,
+							}
+						}
+						b.scaleOps = append(b.scaleOps, scaleOperation{
+							log:     *st.log,
+							trigger: st.ScaleUpTrigger,
+						})
+						batches[nsName] = b
+						ops++
+					}
+				}
+
+				log.V(2).Info("Batch worker dequeued operations", "ops", ops, "batches", len(batches))
+
+			retry:
+				for i := 0; ; i++ {
+					failed := map[types.NamespacedName]batchScaleOperation{}
+
+					for nsName, b := range batches {
+						b := b
+						if err := s.batchScale(context.Background(), b); err != nil {
+							log.V(2).Info("Failed to scale due to error", "error", err)
+							failed[nsName] = b
+						} else {
+							log.V(2).Info("Successfully ran batch scale", "hra", b.namespacedName)
+						}
+					}
+
+					if len(failed) == 0 {
+						break retry
+					}
+
+					batches = failed
+
+					delay := 16 * time.Second
+					if i < len(expBackoff) {
+						delay = expBackoff[i]
+					}
+					time.Sleep(delay)
+				}
+			}
+		}()
+	})
+
+	s.queue <- st
+}
+
+func (s *batchScaler) batchScale(ctx context.Context, batch batchScaleOperation) error {
+	var hra v1alpha1.HorizontalRunnerAutoscaler
+
+	if err := s.Client.Get(ctx, batch.namespacedName, &hra); err != nil {
+		return err
+	}
+
+	copy := hra.DeepCopy()
+
+	copy.Spec.CapacityReservations = getValidCapacityReservations(copy)
+
+	var added, completed int
+
+	for _, scale := range batch.scaleOps {
+		amount := 1
+
+		if scale.trigger.Amount != 0 {
+			amount = scale.trigger.Amount
+		}
+
+		scale.log.V(2).Info("Adding capacity reservation", "amount", amount)
+
+		if amount > 0 {
+			now := time.Now()
+			copy.Spec.CapacityReservations = append(copy.Spec.CapacityReservations, v1alpha1.CapacityReservation{
+				EffectiveTime:  metav1.Time{Time: now},
+				ExpirationTime: metav1.Time{Time: now.Add(scale.trigger.Duration.Duration)},
+				Replicas:       amount,
+			})
+
+			added += amount
+		} else if amount < 0 {
+			var reservations []v1alpha1.CapacityReservation
+
+			var found bool
+
+			for _, r := range copy.Spec.CapacityReservations {
+				if !found && r.Replicas+amount == 0 {
+					found = true
+				} else {
+					reservations = append(reservations, r)
+				}
+			}
+
+			copy.Spec.CapacityReservations = reservations
+
+			completed += amount
+		}
+	}
+
+	before := len(hra.Spec.CapacityReservations)
+	expired := before - len(copy.Spec.CapacityReservations)
+	after := len(copy.Spec.CapacityReservations)
+
+	s.Log.V(1).Info(
+		fmt.Sprintf("Updating hra %s for capacityReservations update", hra.Name),
+		"before", before,
+		"expired", expired,
+		"added", added,
+		"completed", completed,
+		"after", after,
+	)
+
+	if err := s.Client.Update(ctx, copy); err != nil {
+		return fmt.Errorf("updating horizontalrunnerautoscaler to add capacity reservation: %w", err)
+	}
+
+	return nil
+}

controllers/horizontal_runner_autoscaler_webhook.go

@@ -23,14 +23,14 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/go-logr/logr"
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v47/github"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -46,6 +46,8 @@ const (
 
 	keyPrefixEnterprise = "enterprises/"
 	keyRunnerGroup      = "/group/"
+
+	DefaultQueueLimit = 100
 )
 
 // HorizontalRunnerAutoscalerGitHubWebhook autoscales a HorizontalRunnerAutoscaler and the RunnerDeployment on each
@@ -68,6 +70,15 @@ type HorizontalRunnerAutoscalerGitHubWebhook struct {
 	// Set to empty for letting it watch for all namespaces.
 	Namespace string
 	Name      string
+
+	// QueueLimit is the maximum length of the bounded queue of scale targets and their associated operations
+	// A scale target is enqueued on each retrieval of each eligible webhook event, so that it is processed asynchronously.
+	QueueLimit int
+
+	worker      *worker
+	workerInit  sync.Once
+	workerStart sync.Once
+	batchCh     chan *ScaleTarget
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) {
@@ -312,9 +323,19 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.Respons
 		return
 	}
 
-	if err := autoscaler.tryScale(context.TODO(), target); err != nil {
-		log.Error(err, "could not scale up")
+	autoscaler.workerInit.Do(func() {
+		batchScaler := newBatchScaler(context.Background(), autoscaler.Client, autoscaler.Log)
 
+		queueLimit := autoscaler.QueueLimit
+		if queueLimit == 0 {
+			queueLimit = DefaultQueueLimit
+		}
+		autoscaler.worker = newWorker(context.Background(), queueLimit, batchScaler.Add)
+	})
+
+	target.log = &log
+	if ok := autoscaler.worker.Add(target); !ok {
+		log.Error(err, "Could not scale up due to queue full")
 		return
 	}
 
@@ -383,6 +404,8 @@ func matchTriggerConditionAgainstEvent(types []string, eventAction *string) bool
 type ScaleTarget struct {
 	v1alpha1.HorizontalRunnerAutoscaler
 	v1alpha1.ScaleUpTrigger
+
+	log *logr.Logger
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) searchScaleTargets(hras []v1alpha1.HorizontalRunnerAutoscaler, f func(v1alpha1.ScaleUpTrigger) bool) []ScaleTarget {
@@ -501,6 +524,7 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTargetWithF
 	if autoscaler.GitHubClient != nil {
 		simu := &simulator.Simulator{
 			Client: autoscaler.GitHubClient,
+			Log:    log,
 		}
 		// Get available organization runner groups and enterprise runner groups for a repository
 		// These are the sum of runner groups with repository access = All repositories and runner groups
@@ -770,63 +794,6 @@ HRA:
 	return nil, nil
 }
 
-func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScale(ctx context.Context, target *ScaleTarget) error {
-	if target == nil {
-		return nil
-	}
-
-	copy := target.HorizontalRunnerAutoscaler.DeepCopy()
-
-	amount := 1
-
-	if target.ScaleUpTrigger.Amount != 0 {
-		amount = target.ScaleUpTrigger.Amount
-	}
-
-	capacityReservations := getValidCapacityReservations(copy)
-
-	if amount > 0 {
-		now := time.Now()
-		copy.Spec.CapacityReservations = append(capacityReservations, v1alpha1.CapacityReservation{
-			EffectiveTime:  metav1.Time{Time: now},
-			ExpirationTime: metav1.Time{Time: now.Add(target.ScaleUpTrigger.Duration.Duration)},
-			Replicas:       amount,
-		})
-	} else if amount < 0 {
-		var reservations []v1alpha1.CapacityReservation
-
-		var found bool
-
-		for _, r := range capacityReservations {
-			if !found && r.Replicas+amount == 0 {
-				found = true
-			} else {
-				reservations = append(reservations, r)
-			}
-		}
-
-		copy.Spec.CapacityReservations = reservations
-	}
-
-	before := len(target.HorizontalRunnerAutoscaler.Spec.CapacityReservations)
-	expired := before - len(capacityReservations)
-	after := len(copy.Spec.CapacityReservations)
-
-	autoscaler.Log.V(1).Info(
-		fmt.Sprintf("Patching hra %s for capacityReservations update", target.HorizontalRunnerAutoscaler.Name),
-		"before", before,
-		"expired", expired,
-		"amount", amount,
-		"after", after,
-	)
-
-	if err := autoscaler.Client.Patch(ctx, copy, client.MergeFrom(&target.HorizontalRunnerAutoscaler)); err != nil {
-		return fmt.Errorf("patching horizontalrunnerautoscaler to add capacity reservation: %w", err)
-	}
-
-	return nil
-}
-
 func getValidCapacityReservations(autoscaler *v1alpha1.HorizontalRunnerAutoscaler) []v1alpha1.CapacityReservation {
 	var capacityReservations []v1alpha1.CapacityReservation
 

controllers/horizontal_runner_autoscaler_webhook_on_check_run.go

@@ -3,7 +3,7 @@ package controllers
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/pkg/actionsglob"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v47/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchCheckRunEvent(event *github.CheckRunEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v47/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPullRequestEvent(event *github.PullRequestEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_push.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v47/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event *github.PushEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
@@ -15,10 +15,6 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event
 
 		push := g.Push
 
-		if push == nil {
-			return false
-		}
-
-		return true
+		return push != nil
 	}
 }

controllers/horizontal_runner_autoscaler_webhook_test.go

@@ -15,7 +15,7 @@ import (
 
 	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v47/github"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"

controllers/horizontal_runner_autoscaler_webhook_worker.go

@@ -0,0 +1,55 @@
+package controllers
+
+import (
+	"context"
+)
+
+// worker is a worker that has a non-blocking bounded queue of scale targets, dequeues scale target and executes the scale operation one by one.
+type worker struct {
+	scaleTargetQueue chan *ScaleTarget
+	work             func(*ScaleTarget)
+	done             chan struct{}
+}
+
+func newWorker(ctx context.Context, queueLimit int, work func(*ScaleTarget)) *worker {
+	w := &worker{
+		scaleTargetQueue: make(chan *ScaleTarget, queueLimit),
+		work:             work,
+		done:             make(chan struct{}),
+	}
+
+	go func() {
+		defer close(w.done)
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case t := <-w.scaleTargetQueue:
+				work(t)
+			}
+		}
+	}()
+
+	return w
+}
+
+// Add the scale target to the bounded queue, returning the result as a bool value. It returns true on successful enqueue, and returns false otherwise.
+// When returned false, the queue is already full so the enqueue operation must be retried later.
+// If the enqueue was triggered by an external source and there's no intermediate queue that we can use,
+// you must instruct the source to resend the original request later.
+// In case you're building a webhook server around this worker, this means that you must return a http error to the webhook server,
+// so that (hopefully) the sender can resend the webhook event later, or at least the human operator can notice or be notified about the
+// webhook develiery failure so that a manual retry can be done later.
+func (w *worker) Add(st *ScaleTarget) bool {
+	select {
+	case w.scaleTargetQueue <- st:
+		return true
+	default:
+		return false
+	}
+}
+
+func (w *worker) Done() chan struct{} {
+	return w.done
+}

controllers/horizontal_runner_autoscaler_webhook_worker_test.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWorker_Add(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	w := newWorker(ctx, 2, func(st *ScaleTarget) {})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+}
+
+func TestWorker_Work(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var count int
+
+	w := newWorker(ctx, 1, func(st *ScaleTarget) {
+		count++
+		cancel()
+	})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+
+	<-w.Done()
+
+	require.Equal(t, count, 1)
+}

controllers/horizontalrunnerautoscaler_controller.go

@@ -24,7 +24,6 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -38,6 +37,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
+	arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 const (
@@ -47,11 +47,10 @@ const (
 // HorizontalRunnerAutoscalerReconciler reconciles a HorizontalRunnerAutoscaler object
 type HorizontalRunnerAutoscalerReconciler struct {
 	client.Client
-	GitHubClient          *github.Client
+	GitHubClient          *MultiGitHubClient
 	Log                   logr.Logger
 	Recorder              record.EventRecorder
 	Scheme                *runtime.Scheme
-	CacheDuration         time.Duration
 	DefaultScaleDownDelay time.Duration
 	Name                  string
 }
@@ -73,6 +72,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(ctx context.Context, re
 	}
 
 	if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForHRA(&hra)
+
 		return ctrl.Result{}, nil
 	}
 
@@ -310,7 +311,12 @@ func (r *HorizontalRunnerAutoscalerReconciler) reconcile(ctx context.Context, re
 		return ctrl.Result{}, err
 	}
 
-	newDesiredReplicas, err := r.computeReplicasWithCache(log, now, st, hra, minReplicas)
+	ghc, err := r.GitHubClient.InitForHRA(context.Background(), &hra)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	newDesiredReplicas, err := r.computeReplicasWithCache(ghc, log, now, st, hra, minReplicas)
 	if err != nil {
 		r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -461,10 +467,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) getMinReplicas(log logr.Logger, n
 	return minReplicas, active, upcoming, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(ghc *arcgithub.Client, log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
 	var suggestedReplicas int
 
-	v, err := r.suggestDesiredReplicas(st, hra)
+	v, err := r.suggestDesiredReplicas(ghc, st, hra)
 	if err != nil {
 		return 0, err
 	}

controllers/integration_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	github2 "github.com/actions-runner-controller/actions-runner-controller/github"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v47/github"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
 
@@ -99,12 +99,14 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 			return fmt.Sprintf("%s%s", ns.Name, name)
 		}
 
+		multiClient := NewMultiGitHubClient(mgr.GetClient(), env.ghClient)
+
 		runnerController := &RunnerReconciler{
 			Client:                      mgr.GetClient(),
 			Scheme:                      scheme.Scheme,
 			Log:                         logf.Log,
 			Recorder:                    mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient:                env.ghClient,
+			GitHubClient:                multiClient,
 			RunnerImage:                 "example/runner:test",
 			DockerImage:                 "example/docker:test",
 			Name:                        controllerName("runner"),
@@ -116,12 +118,11 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 		Expect(err).NotTo(HaveOccurred(), "failed to setup runner controller")
 
 		replicasetController := &RunnerReplicaSetReconciler{
-			Client:       mgr.GetClient(),
-			Scheme:       scheme.Scheme,
-			Log:          logf.Log,
-			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient: env.ghClient,
-			Name:         controllerName("runnerreplicaset"),
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     controllerName("runnerreplicaset"),
 		}
 		err = replicasetController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup runnerreplicaset controller")
@@ -137,13 +138,12 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 		Expect(err).NotTo(HaveOccurred(), "failed to setup runnerdeployment controller")
 
 		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
-			Client:        mgr.GetClient(),
-			Scheme:        scheme.Scheme,
-			Log:           logf.Log,
-			GitHubClient:  env.ghClient,
-			Recorder:      mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
-			CacheDuration: 1 * time.Second,
-			Name:          controllerName("horizontalrunnerautoscaler"),
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			GitHubClient: multiClient,
+			Recorder:     mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+			Name:         controllerName("horizontalrunnerautoscaler"),
 		}
 		err = autoscalerController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler controller")
@@ -1367,7 +1367,7 @@ func (env *testEnvironment) ExpectRegisteredNumberCountEventuallyEquals(want int
 
 			return len(rs)
 		},
-		time.Second*5, time.Millisecond*500).Should(Equal(want), optionalDescriptions...)
+		time.Second*10, time.Millisecond*500).Should(Equal(want), optionalDescriptions...)
 }
 
 func (env *testEnvironment) SendOrgPullRequestEvent(org, repo, branch, action string) {

controllers/metrics/horizontalrunnerautoscaler.go

@@ -7,8 +7,13 @@ import (
 )
 
 const (
-	hraName      = "horizontalrunnerautoscaler"
-	hraNamespace = "namespace"
+	hraName        = "horizontalrunnerautoscaler"
+	hraNamespace   = "namespace"
+	stEnterprise   = "enterprise"
+	stOrganization = "organization"
+	stRepository   = "repository"
+	stKind         = "kind"
+	stName         = "name"
 )
 
 var (
@@ -16,6 +21,16 @@ var (
 		horizontalRunnerAutoscalerMinReplicas,
 		horizontalRunnerAutoscalerMaxReplicas,
 		horizontalRunnerAutoscalerDesiredReplicas,
+		horizontalRunnerAutoscalerReplicasDesired,
+		horizontalRunnerAutoscalerRunners,
+		horizontalRunnerAutoscalerRunnersRegistered,
+		horizontalRunnerAutoscalerRunnersBusy,
+		horizontalRunnerAutoscalerTerminatingBusy,
+		horizontalRunnerAutoscalerNecessaryReplicas,
+		horizontalRunnerAutoscalerWorkflowRunsCompleted,
+		horizontalRunnerAutoscalerWorkflowRunsInProgress,
+		horizontalRunnerAutoscalerWorkflowRunsQueued,
+		horizontalRunnerAutoscalerWorkflowRunsUnknown,
 	}
 )
 
@@ -41,6 +56,78 @@ var (
 		},
 		[]string{hraName, hraNamespace},
 	)
+	// PercentageRunnersBusy
+	horizontalRunnerAutoscalerReplicasDesired = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_replicas_desired",
+			Help: "replicas_desired of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunners = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners",
+			Help: "num_runners of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersRegistered = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_registered",
+			Help: "num_runners_registered of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_busy",
+			Help: "num_runners_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerTerminatingBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_terminating_busy",
+			Help: "num_terminating_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	// QueuedAndInProgressWorkflowRuns
+	horizontalRunnerAutoscalerNecessaryReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_necessary_replicas",
+			Help: "necessary_replicas of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsCompleted = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_completed",
+			Help: "workflow_runs_completed of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsInProgress = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_in_progress",
+			Help: "workflow_runs_in_progress of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsQueued = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_queued",
+			Help: "workflow_runs_queued of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsUnknown = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_unknown",
+			Help: "workflow_runs_unknown of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
 )
 
 func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) {
@@ -65,3 +152,61 @@ func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.Ho
 		horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas))
 	}
 }
+
+func SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	desiredReplicas int,
+	numRunners int,
+	numRunnersRegistered int,
+	numRunnersBusy int,
+	numTerminatingBusy int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerReplicasDesired.With(labels).Set(float64(desiredReplicas))
+	horizontalRunnerAutoscalerRunners.With(labels).Set(float64(numRunners))
+	horizontalRunnerAutoscalerRunnersRegistered.With(labels).Set(float64(numRunnersRegistered))
+	horizontalRunnerAutoscalerRunnersBusy.With(labels).Set(float64(numRunnersBusy))
+	horizontalRunnerAutoscalerTerminatingBusy.With(labels).Set(float64(numTerminatingBusy))
+}
+
+func SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	necessaryReplicas int,
+	workflowRunsCompleted int,
+	workflowRunsInProgress int,
+	workflowRunsQueued int,
+	workflowRunsUnknown int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerNecessaryReplicas.With(labels).Set(float64(necessaryReplicas))
+	horizontalRunnerAutoscalerWorkflowRunsCompleted.With(labels).Set(float64(workflowRunsCompleted))
+	horizontalRunnerAutoscalerWorkflowRunsInProgress.With(labels).Set(float64(workflowRunsInProgress))
+	horizontalRunnerAutoscalerWorkflowRunsQueued.With(labels).Set(float64(workflowRunsQueued))
+	horizontalRunnerAutoscalerWorkflowRunsUnknown.With(labels).Set(float64(workflowRunsUnknown))
+}

controllers/multi_githubclient.go

@@ -0,0 +1,358 @@
+package controllers
+
+import (
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"strconv"
+	"sync"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// The api creds scret annotation is added by the runner controller or the runnerset controller according to runner.spec.githubAPICredentialsFrom.secretRef.name,
+	// so that the runner pod controller can share the same GitHub API credentials and the instance of the GitHub API client with the upstream controllers.
+	annotationKeyGitHubAPICredsSecret = annotationKeyPrefix + "github-api-creds-secret"
+)
+
+type runnerOwnerRef struct {
+	// kind is either StatefulSet or Runner, and populated via the owner reference in the runner pod controller or via the reconcilation target's kind in
+	// runnerset and runner controllers.
+	kind     string
+	ns, name string
+}
+
+type secretRef struct {
+	ns, name string
+}
+
+// savedClient is the each cache entry that contains the client for the specific set of credentials,
+// like a PAT or a pair of key and cert.
+// the `hash` is a part of the savedClient not the key because we are going to keep only the client for the latest creds
+// in case the operator updated the k8s secret containing the credentials.
+type savedClient struct {
+	hash string
+
+	// refs is the map of all the objects that references this client, used for reference counting to gc
+	// the client if unneeded.
+	refs map[runnerOwnerRef]struct{}
+
+	*github.Client
+}
+
+type resourceReader interface {
+	Get(context.Context, types.NamespacedName, client.Object) error
+}
+
+type MultiGitHubClient struct {
+	mu sync.Mutex
+
+	client resourceReader
+
+	githubClient *github.Client
+
+	// The saved client is freed once all its dependents disappear, or the contents of the secret changed.
+	// We track dependents via a golang map embedded within the savedClient struct. Each dependent is checked on their respective Kubernetes finalizer,
+	// so that we won't miss any dependent's termination.
+	// The change is the secret is determined using the hash of its contents.
+	clients map[secretRef]savedClient
+}
+
+func NewMultiGitHubClient(client resourceReader, githubClient *github.Client) *MultiGitHubClient {
+	return &MultiGitHubClient{
+		client:       client,
+		githubClient: githubClient,
+		clients:      map[secretRef]savedClient{},
+	}
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerPod(ctx context.Context, pod *corev1.Pod) (*github.Client, error) {
+	// These 3 default values are used only when the user created the pod directly, not via Runner, RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunnerPod(pod)
+	secretName := pod.Annotations[annotationKeyGitHubAPICredsSecret]
+
+	// kind can be any of Pod, Runner, RunnerReplicaSet, RunnerDeployment, or RunnerSet depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, pod.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunner(ctx context.Context, r *v1alpha1.Runner) (*github.Client, error) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	// These 3 default values are used only when the user created the runner resource directly, not via RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunner(r)
+	if ref.ns != r.Namespace {
+		return nil, fmt.Errorf("referencing github api creds secret from owner in another namespace is not supported yet")
+	}
+
+	// kind can be any of Runner, RunnerReplicaSet, or RunnerDeployment depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, r.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerSet(ctx context.Context, rs *v1alpha1.RunnerSet) (*github.Client, error) {
+	ref := refFromRunnerSet(rs)
+
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, rs.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForHRA(ctx context.Context, hra *v1alpha1.HorizontalRunnerAutoscaler) (*github.Client, error) {
+	ref := refFromHorizontalRunnerAutoscaler(hra)
+
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, hra.Namespace, secretName, ref)
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerPod(p *corev1.Pod) {
+	secretName := p.Annotations[annotationKeyGitHubAPICredsSecret]
+	c.derefClient(p.Namespace, secretName, refFromRunnerPod(p))
+}
+
+func (c *MultiGitHubClient) DeinitForRunner(r *v1alpha1.Runner) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(r.Namespace, secretName, refFromRunner(r))
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerSet(rs *v1alpha1.RunnerSet) {
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(rs.Namespace, secretName, refFromRunnerSet(rs))
+}
+
+func (c *MultiGitHubClient) deinitClientForRunnerReplicaSet(rs *v1alpha1.RunnerReplicaSet) {
+	c.derefClient(rs.Namespace, rs.Spec.Template.Spec.GitHubAPICredentialsFrom.SecretRef.Name, refFromRunnerReplicaSet(rs))
+}
+
+func (c *MultiGitHubClient) deinitClientForRunnerDeployment(rd *v1alpha1.RunnerDeployment) {
+	c.derefClient(rd.Namespace, rd.Spec.Template.Spec.GitHubAPICredentialsFrom.SecretRef.Name, refFromRunnerDeployment(rd))
+}
+
+func (c *MultiGitHubClient) DeinitForHRA(hra *v1alpha1.HorizontalRunnerAutoscaler) {
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(hra.Namespace, secretName, refFromHorizontalRunnerAutoscaler(hra))
+}
+
+func (c *MultiGitHubClient) initClientForSecret(secret *corev1.Secret, dependent *runnerOwnerRef) (*savedClient, error) {
+	secRef := secretRef{
+		ns:   secret.Namespace,
+		name: secret.Name,
+	}
+
+	cliRef := c.clients[secRef]
+
+	var ks []string
+
+	for k := range secret.Data {
+		ks = append(ks, k)
+	}
+
+	sort.SliceStable(ks, func(i, j int) bool { return ks[i] < ks[j] })
+
+	hash := sha1.New()
+	for _, k := range ks {
+		hash.Write(secret.Data[k])
+	}
+	hashStr := hex.EncodeToString(hash.Sum(nil))
+
+	if cliRef.hash != hashStr {
+		delete(c.clients, secRef)
+
+		conf, err := secretDataToGitHubClientConfig(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		// Fallback to the controller-wide setting if EnterpriseURL is not set and the original client is an enterprise client.
+		if conf.EnterpriseURL == "" && c.githubClient.IsEnterprise {
+			conf.EnterpriseURL = c.githubClient.GithubBaseURL
+		}
+
+		cli, err := conf.NewClient()
+		if err != nil {
+			return nil, err
+		}
+
+		cliRef = savedClient{
+			hash:   hashStr,
+			refs:   map[runnerOwnerRef]struct{}{},
+			Client: cli,
+		}
+
+		c.clients[secRef] = cliRef
+	}
+
+	if dependent != nil {
+		c.clients[secRef].refs[*dependent] = struct{}{}
+	}
+
+	return &cliRef, nil
+}
+
+func (c *MultiGitHubClient) initClientWithSecretName(ctx context.Context, ns, secretName string, runRef *runnerOwnerRef) (*github.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if secretName == "" {
+		return c.githubClient, nil
+	}
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if _, ok := c.clients[secRef]; !ok {
+		c.clients[secRef] = savedClient{}
+	}
+
+	var sec corev1.Secret
+	if err := c.client.Get(ctx, types.NamespacedName{Namespace: ns, Name: secretName}, &sec); err != nil {
+		return nil, err
+	}
+
+	savedClient, err := c.initClientForSecret(&sec, runRef)
+	if err != nil {
+		return nil, err
+	}
+
+	return savedClient.Client, nil
+}
+
+func (c *MultiGitHubClient) derefClient(ns, secretName string, dependent *runnerOwnerRef) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if dependent != nil {
+		delete(c.clients[secRef].refs, *dependent)
+	}
+
+	cliRef := c.clients[secRef]
+
+	if dependent == nil || len(cliRef.refs) == 0 {
+		delete(c.clients, secRef)
+	}
+}
+
+func secretDataToGitHubClientConfig(data map[string][]byte) (*github.Config, error) {
+	var (
+		conf github.Config
+
+		err error
+	)
+
+	conf.URL = string(data["github_url"])
+
+	conf.UploadURL = string(data["github_upload_url"])
+
+	conf.EnterpriseURL = string(data["github_enterprise_url"])
+
+	conf.RunnerGitHubURL = string(data["github_runner_url"])
+
+	conf.Token = string(data["github_token"])
+
+	appID := string(data["github_app_id"])
+
+	conf.AppID, err = strconv.ParseInt(appID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	instID := string(data["github_app_installation_id"])
+
+	conf.AppInstallationID, err = strconv.ParseInt(instID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	conf.AppPrivateKey = string(data["github_app_private_key"])
+
+	return &conf, nil
+}
+
+func refFromRunnerDeployment(rd *v1alpha1.RunnerDeployment) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rd.Kind,
+		ns:   rd.Namespace,
+		name: rd.Name,
+	}
+}
+
+func refFromRunnerReplicaSet(rs *v1alpha1.RunnerReplicaSet) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rs.Kind,
+		ns:   rs.Namespace,
+		name: rs.Name,
+	}
+}
+
+func refFromRunner(r *v1alpha1.Runner) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: r.Kind,
+		ns:   r.Namespace,
+		name: r.Name,
+	}
+}
+
+func refFromRunnerPod(po *corev1.Pod) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: po.Kind,
+		ns:   po.Namespace,
+		name: po.Name,
+	}
+}
+func refFromRunnerSet(rs *v1alpha1.RunnerSet) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rs.Kind,
+		ns:   rs.Namespace,
+		name: rs.Name,
+	}
+}
+
+func refFromHorizontalRunnerAutoscaler(hra *v1alpha1.HorizontalRunnerAutoscaler) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: hra.Kind,
+		ns:   hra.Namespace,
+		name: hra.Name,
+	}
+}

controllers/new_runner_pod_test.go

@@ -7,12 +7,45 @@ import (
 	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+func newWorkGenericEphemeralVolume(t *testing.T, storageReq string) corev1.Volume {
+	GBs, err := resource.ParseQuantity(storageReq)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	return corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes: []corev1.PersistentVolumeAccessMode{
+							corev1.ReadWriteOnce,
+						},
+						StorageClassName: strPtr("runner-work-dir"),
+						Resources: corev1.ResourceRequirements{
+							Requests: corev1.ResourceList{
+								corev1.ResourceStorage: GBs,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func TestNewRunnerPod(t *testing.T) {
+	workGenericEphemeralVolume := newWorkGenericEphemeralVolume(t, "10Gi")
+
 	type testcase struct {
 		description string
 
@@ -25,7 +58,7 @@ func TestNewRunnerPod(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
-				"runnerset-name": "runner",
+				"actions-runner": "",
 			},
 		},
 		Spec: corev1.PodSpec{
@@ -94,6 +127,10 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "DOCKER_HOST",
 							Value: "tcp://localhost:2376",
@@ -106,10 +143,6 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "DOCKER_CERT_PATH",
 							Value: "/certs/client",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -159,7 +192,7 @@ func TestNewRunnerPod(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}
 
@@ -171,7 +204,7 @@ func TestNewRunnerPod(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
-				"runnerset-name": "runner",
+				"actions-runner": "",
 			},
 		},
 		Spec: corev1.PodSpec{
@@ -229,8 +262,8 @@ func TestNewRunnerPod(t *testing.T) {
 							Value: "true",
 						},
 						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
 						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
@@ -245,7 +278,7 @@ func TestNewRunnerPod(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}
 
@@ -253,7 +286,7 @@ func TestNewRunnerPod(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
-				"runnerset-name": "runner",
+				"actions-runner": "",
 			},
 		},
 		Spec: corev1.PodSpec{
@@ -311,8 +344,8 @@ func TestNewRunnerPod(t *testing.T) {
 							Value: "true",
 						},
 						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
 						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
@@ -327,7 +360,7 @@ func TestNewRunnerPod(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}
 
@@ -400,8 +433,87 @@ func TestNewRunnerPod(t *testing.T) {
 				DockerEnabled: boolPtr(false),
 			},
 			want: newTestPod(dockerDisabled, func(p *corev1.Pod) {
-				// TODO
-				// p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+				p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (with explicit volumeMount)",
+			template: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "runner",
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "work",
+									MountPath: "/runner/_work",
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						workGenericEphemeralVolume,
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					workGenericEphemeralVolume,
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+				}
+				p.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+					{
+						Name:      "work",
+						MountPath: "/runner/_work",
+					},
+					{
+						Name:      "runner",
+						MountPath: "/runner",
+					},
+					{
+						Name:      "certs-client",
+						MountPath: "/certs/client",
+						ReadOnly:  true,
+					},
+				}
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (without explicit volumeMount)",
+			template: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Volumes: []corev1.Volume{
+						workGenericEphemeralVolume,
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					workGenericEphemeralVolume,
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+				}
 			}),
 		},
 	}
@@ -417,14 +529,20 @@ func TestNewRunnerPod(t *testing.T) {
 	for i := range testcases {
 		tc := testcases[i]
 		t.Run(tc.description, func(t *testing.T) {
-			got, err := newRunnerPod("runner", tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, false)
+			got, err := newRunnerPod(tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, false)
 			require.NoError(t, err)
 			require.Equal(t, tc.want, got)
 		})
 	}
 }
 
+func strPtr(s string) *string {
+	return &s
+}
+
 func TestNewRunnerPodFromRunnerController(t *testing.T) {
+	workGenericEphemeralVolume := newWorkGenericEphemeralVolume(t, "10Gi")
+
 	type testcase struct {
 		description string
 
@@ -442,7 +560,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
 				"pod-template-hash": "8857b86c7",
-				"runnerset-name":    "runner",
+				"actions-runner":    "",
 			},
 			OwnerReferences: []metav1.OwnerReference{
 				{
@@ -520,6 +638,10 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "DOCKER_HOST",
 							Value: "tcp://localhost:2376",
@@ -532,10 +654,6 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "DOCKER_CERT_PATH",
 							Value: "/certs/client",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -593,7 +711,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}
 
@@ -603,7 +721,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
 				"pod-template-hash": "8857b86c7",
-				"runnerset-name":    "runner",
+				"actions-runner":    "",
 			},
 			OwnerReferences: []metav1.OwnerReference{
 				{
@@ -670,8 +788,8 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Value: "true",
 						},
 						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
 						},
 						{
 							Name:  "RUNNER_NAME",
@@ -694,7 +812,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}
 
@@ -704,7 +822,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
 				"pod-template-hash": "8857b86c7",
-				"runnerset-name":    "runner",
+				"actions-runner":    "",
 			},
 			OwnerReferences: []metav1.OwnerReference{
 				{
@@ -771,8 +889,8 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Value: "true",
 						},
 						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
 						},
 						{
 							Name:  "RUNNER_NAME",
@@ -795,7 +913,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}
 
@@ -904,7 +1022,97 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			},
 
 			want: newTestPod(dockerDisabled, func(p *corev1.Pod) {
-				// p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+				p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (with explicit volumeMount)",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerPodSpec: arcv1alpha1.RunnerPodSpec{
+						Containers: []corev1.Container{
+							{
+								Name: "runner",
+								VolumeMounts: []corev1.VolumeMount{
+									{
+										Name:      "work",
+										MountPath: "/runner/_work",
+									},
+								},
+							},
+						},
+						Volumes: []corev1.Volume{
+							workGenericEphemeralVolume,
+						},
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					workGenericEphemeralVolume,
+				}
+				p.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+					{
+						Name:      "work",
+						MountPath: "/runner/_work",
+					},
+					{
+						Name:      "runner",
+						MountPath: "/runner",
+					},
+					{
+						Name:      "certs-client",
+						MountPath: "/certs/client",
+						ReadOnly:  true,
+					},
+				}
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (without explicit volumeMount)",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerPodSpec: arcv1alpha1.RunnerPodSpec{
+						Volumes: []corev1.Volume{
+							workGenericEphemeralVolume,
+						},
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					workGenericEphemeralVolume,
+				}
 			}),
 		},
 	}
@@ -923,13 +1131,20 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 
 	for i := range testcases {
 		tc := testcases[i]
+
+		rr := &testResourceReader{
+			objects: map[types.NamespacedName]client.Object{},
+		}
+
+		multiClient := NewMultiGitHubClient(rr, &github.Client{GithubBaseURL: githubBaseURL})
+
 		t.Run(tc.description, func(t *testing.T) {
 			r := &RunnerReconciler{
 				RunnerImage:            defaultRunnerImage,
 				RunnerImagePullSecrets: defaultRunnerImagePullSecrets,
 				DockerImage:            defaultDockerImage,
 				DockerRegistryMirror:   defaultDockerRegistryMirror,
-				GitHubClient:           &github.Client{GithubBaseURL: githubBaseURL},
+				GitHubClient:           multiClient,
 				Scheme:                 scheme,
 			}
 			got, err := r.newPod(tc.runner)

controllers/persistent_volume_claim_controller.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2022 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// RunnerPersistentVolumeClaimReconciler reconciles a PersistentVolume object
+type RunnerPersistentVolumeClaimReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
+}
+
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerPersistentVolumeClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("pvc", req.NamespacedName)
+
+	var pvc corev1.PersistentVolumeClaim
+	if err := r.Get(ctx, req.NamespacedName, &pvc); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	res, err := syncPVC(ctx, r.Client, log, req.Namespace, &pvc)
+
+	if res == nil {
+		res = &ctrl.Result{}
+	}
+
+	return *res, err
+}
+
+func (r *RunnerPersistentVolumeClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerpersistentvolumeclaim-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.PersistentVolumeClaim{}).
+		Named(name).
+		Complete(r)
+}

controllers/persistent_volume_controller.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2022 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// RunnerPersistentVolumeReconciler reconciles a PersistentVolume object
+type RunnerPersistentVolumeReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
+}
+
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerPersistentVolumeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("pv", req.NamespacedName)
+
+	var pv corev1.PersistentVolume
+	if err := r.Get(ctx, req.NamespacedName, &pv); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	res, err := syncPV(ctx, r.Client, log, req.Namespace, &pv)
+	if res == nil {
+		res = &ctrl.Result{}
+	}
+
+	return *res, err
+}
+
+func (r *RunnerPersistentVolumeReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerpersistentvolume-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.PersistentVolume{}).
+		Named(name).
+		Complete(r)
+}

controllers/pod_runner_token_injector.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
 	"gomodules.xyz/jsonpatch/v2"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -29,7 +28,7 @@ type PodRunnerTokenInjector struct {
 	Name         string
 	Log          logr.Logger
 	Recorder     record.EventRecorder
-	GitHubClient *github.Client
+	GitHubClient *MultiGitHubClient
 	decoder      *admission.Decoder
 }
 
@@ -66,7 +65,12 @@ func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Reque
 		return newEmptyResponse()
 	}
 
-	rt, err := t.GitHubClient.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
+	ghc, err := t.GitHubClient.InitForRunnerPod(ctx, &pod)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	rt, err := ghc.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
 	if err != nil {
 		t.Log.Error(err, "Failed to get new registration token")
 		return admission.Errored(http.StatusInternalServerError, err)
@@ -78,9 +82,7 @@ func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Reque
 
 	updated.Annotations[AnnotationKeyTokenExpirationDate] = ts
 
-	if pod.Spec.RestartPolicy != corev1.RestartPolicyOnFailure {
-		updated.Spec.RestartPolicy = corev1.RestartPolicyOnFailure
-	}
+	forceRunnerPodRestartPolicyNever(updated)
 
 	buf, err := json.Marshal(updated)
 	if err != nil {

controllers/runner_controller.go

@@ -18,7 +18,10 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
@@ -33,10 +36,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 const (
@@ -47,15 +50,13 @@ const (
 
 	retryDelayOnGitHubAPIRateLimitError = 30 * time.Second
 
-	// This is an annotation internal to actions-runner-controller and can change in backward-incompatible ways
-	annotationKeyRegistrationOnly = "actions-runner-controller/registration-only"
-
-	EnvVarOrg                        = "RUNNER_ORG"
-	EnvVarRepo                       = "RUNNER_REPO"
-	EnvVarEnterprise                 = "RUNNER_ENTERPRISE"
-	EnvVarEphemeral                  = "RUNNER_EPHEMERAL"
-	EnvVarRunnerFeatureFlagEphemeral = "RUNNER_FEATURE_FLAG_EPHEMERAL"
-	EnvVarTrue                       = "true"
+	EnvVarOrg        = "RUNNER_ORG"
+	EnvVarRepo       = "RUNNER_REPO"
+	EnvVarGroup      = "RUNNER_GROUP"
+	EnvVarLabels     = "RUNNER_LABELS"
+	EnvVarEnterprise = "RUNNER_ENTERPRISE"
+	EnvVarEphemeral  = "RUNNER_EPHEMERAL"
+	EnvVarTrue       = "true"
 )
 
 // RunnerReconciler reconciles a Runner object
@@ -64,7 +65,7 @@ type RunnerReconciler struct {
 	Log                         logr.Logger
 	Recorder                    record.EventRecorder
 	Scheme                      *runtime.Scheme
-	GitHubClient                *github.Client
+	GitHubClient                *MultiGitHubClient
 	RunnerImage                 string
 	RunnerImagePullSecrets      []string
 	DockerImage                 string
@@ -72,16 +73,20 @@ type RunnerReconciler struct {
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
-
-	UnregistrationRetryDelay time.Duration
+	UseRunnerStatusUpdateHook   bool
+	UnregistrationRetryDelay    time.Duration
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;delete
 // +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=create;delete;get
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;delete;get
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;delete;get
 
 func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("runner", req.NamespacedName)
@@ -116,6 +121,9 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			// Pod was not found
 			return r.processRunnerDeletion(runner, ctx, log, nil)
 		}
+
+		r.GitHubClient.DeinitForRunner(&runner)
+
 		return r.processRunnerDeletion(runner, ctx, log, &pod)
 	}
 
@@ -135,7 +143,7 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 	ready := runnerPodReady(&pod)
 
-	if runner.Status.Phase != phase || runner.Status.Ready != ready {
+	if (runner.Status.Phase != phase || runner.Status.Ready != ready) && !r.UseRunnerStatusUpdateHook || runner.Status.Phase == "" && r.UseRunnerStatusUpdateHook {
 		if pod.Status.Phase == corev1.PodRunning {
 			// Seeing this message, you can expect the runner to become `Running` soon.
 			log.V(1).Info(
@@ -207,6 +215,24 @@ func runnerPodOrContainerIsStopped(pod *corev1.Pod) bool {
 	return stopped
 }
 
+func ephemeralRunnerContainerStatus(pod *corev1.Pod) *corev1.ContainerStatus {
+	if getRunnerEnv(pod, "RUNNER_EPHEMERAL") != "true" {
+		return nil
+	}
+
+	for _, status := range pod.Status.ContainerStatuses {
+		if status.Name != containerName {
+			continue
+		}
+
+		status := status
+
+		return &status
+	}
+
+	return nil
+}
+
 func (r *RunnerReconciler) processRunnerDeletion(runner v1alpha1.Runner, ctx context.Context, log logr.Logger, pod *corev1.Pod) (reconcile.Result, error) {
 	finalizers, removed := removeFinalizer(runner.ObjectMeta.Finalizers, finalizerName)
 
@@ -238,6 +264,96 @@ func (r *RunnerReconciler) processRunnerCreation(ctx context.Context, runner v1a
 		return ctrl.Result{}, err
 	}
 
+	needsServiceAccount := runner.Spec.ServiceAccountName == "" && (r.UseRunnerStatusUpdateHook || runner.Spec.ContainerMode == "kubernetes")
+	if needsServiceAccount {
+		serviceAccount := &corev1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+		}
+		if res := r.createObject(ctx, serviceAccount, serviceAccount.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+
+		rules := []rbacv1.PolicyRule{}
+
+		if r.UseRunnerStatusUpdateHook {
+			rules = append(rules, []rbacv1.PolicyRule{
+				{
+					APIGroups:     []string{"actions.summerwind.dev"},
+					Resources:     []string{"runners/status"},
+					Verbs:         []string{"get", "update", "patch"},
+					ResourceNames: []string{runner.ObjectMeta.Name},
+				},
+			}...)
+		}
+
+		if runner.Spec.ContainerMode == "kubernetes" {
+			// Permissions based on https://github.com/actions/runner-container-hooks/blob/main/packages/k8s/README.md
+			rules = append(rules, []rbacv1.PolicyRule{
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods/exec"},
+					Verbs:     []string{"get", "create"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods/log"},
+					Verbs:     []string{"get", "list", "watch"},
+				},
+				{
+					APIGroups: []string{"batch"},
+					Resources: []string{"jobs"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"secrets"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+			}...)
+		}
+
+		role := &rbacv1.Role{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+			Rules: rules,
+		}
+		if res := r.createObject(ctx, role, role.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+
+		roleBinding := &rbacv1.RoleBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+			RoleRef: rbacv1.RoleRef{
+				APIGroup: "rbac.authorization.k8s.io",
+				Kind:     "Role",
+				Name:     runner.ObjectMeta.Name,
+			},
+			Subjects: []rbacv1.Subject{
+				{
+					Kind:      "ServiceAccount",
+					Name:      runner.ObjectMeta.Name,
+					Namespace: runner.ObjectMeta.Namespace,
+				},
+			},
+		}
+		if res := r.createObject(ctx, roleBinding, roleBinding.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+	}
+
 	if err := r.Create(ctx, &newPod); err != nil {
 		if kerrors.IsAlreadyExists(err) {
 			// Gracefully handle pod-already-exists errors due to informer cache delay.
@@ -260,6 +376,27 @@ func (r *RunnerReconciler) processRunnerCreation(ctx context.Context, runner v1a
 	return ctrl.Result{}, nil
 }
 
+func (r *RunnerReconciler) createObject(ctx context.Context, obj client.Object, meta metav1.ObjectMeta, runner *v1alpha1.Runner, log logr.Logger) *ctrl.Result {
+	kind := strings.Split(reflect.TypeOf(obj).String(), ".")[1]
+	if err := ctrl.SetControllerReference(runner, obj, r.Scheme); err != nil {
+		log.Error(err, fmt.Sprintf("Could not add owner reference to %s %s. %s", kind, meta.Name, err.Error()))
+		return &ctrl.Result{Requeue: true}
+	}
+	if err := r.Create(ctx, obj); err != nil {
+		if kerrors.IsAlreadyExists(err) {
+			log.Info(fmt.Sprintf("Failed to create %s %s as it already exists. Reusing existing %s", kind, meta.Name, kind))
+			r.Recorder.Event(runner, corev1.EventTypeNormal, fmt.Sprintf("%sReused", kind), fmt.Sprintf("Reused %s '%s'", kind, meta.Name))
+			return nil
+		}
+
+		log.Error(err, fmt.Sprintf("Retrying as failed to create %s %s resource", kind, meta.Name))
+		return &ctrl.Result{Requeue: true}
+	}
+	r.Recorder.Event(runner, corev1.EventTypeNormal, fmt.Sprintf("%sCreated", kind), fmt.Sprintf("Created %s '%s'", kind, meta.Name))
+	log.Info(fmt.Sprintf("Created %s", kind), "name", meta.Name)
+	return nil
+}
+
 func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v1alpha1.Runner) (bool, error) {
 	if runner.IsRegisterable() {
 		return false, nil
@@ -267,7 +404,12 @@ func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v
 
 	log := r.Log.WithValues("runner", runner.Name)
 
-	rt, err := r.GitHubClient.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+	ghc, err := r.GitHubClient.InitForRunner(ctx, &runner)
+	if err != nil {
+		return false, err
+	}
+
+	rt, err := ghc.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 	if err != nil {
 		// An error can be a permanent, permission issue like the below:
 		//    POST https://api.github.com/enterprises/YOUR_ENTERPRISE/actions/runners/registration-token: 403 Resource not accessible by integration []
@@ -307,6 +449,11 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		labels[k] = v
 	}
 
+	ghc, err := r.GitHubClient.InitForRunner(context.Background(), &runner)
+	if err != nil {
+		return corev1.Pod{}, err
+	}
+
 	// This implies that...
 	//
 	// (1) We recreate the runner pod whenever the runner has changes in:
@@ -330,7 +477,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		filterLabels(runner.ObjectMeta.Labels, LabelKeyRunnerTemplateHash),
 		runner.ObjectMeta.Annotations,
 		runner.Spec,
-		r.GitHubClient.GithubBaseURL,
+		ghc.GithubBaseURL,
 		// Token change should trigger replacement.
 		// We need to include this explicitly here because
 		// runner.Spec does not contain the possibly updated token stored in the
@@ -349,31 +496,66 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 
 	if len(runner.Spec.Containers) == 0 {
 		template.Spec.Containers = append(template.Spec.Containers, corev1.Container{
-			Name:            "runner",
-			ImagePullPolicy: runner.Spec.ImagePullPolicy,
-			EnvFrom:         runner.Spec.EnvFrom,
-			Env:             runner.Spec.Env,
-			Resources:       runner.Spec.Resources,
+			Name: "runner",
 		})
 
 		if (runner.Spec.DockerEnabled == nil || *runner.Spec.DockerEnabled) && (runner.Spec.DockerdWithinRunnerContainer == nil || !*runner.Spec.DockerdWithinRunnerContainer) {
 			template.Spec.Containers = append(template.Spec.Containers, corev1.Container{
-				Name:         "docker",
-				VolumeMounts: runner.Spec.DockerVolumeMounts,
-				Resources:    runner.Spec.DockerdContainerResources,
-				Env:          runner.Spec.DockerEnv,
+				Name: "docker",
 			})
 		}
 	} else {
 		template.Spec.Containers = runner.Spec.Containers
 	}
 
+	for i, c := range template.Spec.Containers {
+		switch c.Name {
+		case "runner":
+			if c.ImagePullPolicy == "" {
+				template.Spec.Containers[i].ImagePullPolicy = runner.Spec.ImagePullPolicy
+			}
+			if len(c.EnvFrom) == 0 {
+				template.Spec.Containers[i].EnvFrom = runner.Spec.EnvFrom
+			}
+			if len(c.Env) == 0 {
+				template.Spec.Containers[i].Env = runner.Spec.Env
+			}
+			if len(c.Resources.Requests) == 0 {
+				template.Spec.Containers[i].Resources.Requests = runner.Spec.Resources.Requests
+			}
+			if len(c.Resources.Limits) == 0 {
+				template.Spec.Containers[i].Resources.Limits = runner.Spec.Resources.Limits
+			}
+		case "docker":
+			if len(c.VolumeMounts) == 0 {
+				template.Spec.Containers[i].VolumeMounts = runner.Spec.DockerVolumeMounts
+			}
+			if len(c.Resources.Limits) == 0 {
+				template.Spec.Containers[i].Resources.Limits = runner.Spec.DockerdContainerResources.Limits
+			}
+			if len(c.Resources.Requests) == 0 {
+				template.Spec.Containers[i].Resources.Requests = runner.Spec.DockerdContainerResources.Requests
+			}
+			if len(c.Env) == 0 {
+				template.Spec.Containers[i].Env = runner.Spec.DockerEnv
+			}
+		}
+	}
+
 	template.Spec.SecurityContext = runner.Spec.SecurityContext
 	template.Spec.EnableServiceLinks = runner.Spec.EnableServiceLinks
 
-	registrationOnly := metav1.HasAnnotation(runner.ObjectMeta, annotationKeyRegistrationOnly)
+	if runner.Spec.ContainerMode == "kubernetes" {
+		workDir := runner.Spec.WorkDir
+		if workDir == "" {
+			workDir = "/runner/_work"
+		}
+		if err := applyWorkVolumeClaimTemplateToPod(&template, runner.Spec.WorkVolumeClaimTemplate, workDir); err != nil {
+			return corev1.Pod{}, err
+		}
+	}
 
-	pod, err := newRunnerPod(runner.Name, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL, registrationOnly)
+	pod, err := newRunnerPodWithContainerMode(runner.Spec.ContainerMode, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, ghc.GithubBaseURL, r.UseRunnerStatusUpdateHook)
 	if err != nil {
 		return pod, err
 	}
@@ -385,6 +567,9 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		// if operater provides a work volume mount, use that
 		isPresent, _ := workVolumeMountPresent(runnerSpec.VolumeMounts)
 		if isPresent {
+			if runnerSpec.ContainerMode == "kubernetes" {
+				return pod, errors.New("volume mount \"work\" should be specified by workVolumeClaimTemplate in container mode kubernetes")
+			}
 			// remove work volume since it will be provided from runnerSpec.Volumes
 			// if we don't remove it here we would get a duplicate key error, i.e. two volumes named work
 			_, index := workVolumeMountPresent(pod.Spec.Containers[0].VolumeMounts)
@@ -398,6 +583,9 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		// if operator provides a work volume. use that
 		isPresent, _ := workVolumePresent(runnerSpec.Volumes)
 		if isPresent {
+			if runnerSpec.ContainerMode == "kubernetes" {
+				return pod, errors.New("volume \"work\" should be specified by workVolumeClaimTemplate in container mode kubernetes")
+			}
 			_, index := workVolumePresent(pod.Spec.Volumes)
 
 			// remove work volume since it will be provided from runnerSpec.Volumes
@@ -407,6 +595,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 
 		pod.Spec.Volumes = append(pod.Spec.Volumes, runnerSpec.Volumes...)
 	}
+
 	if len(runnerSpec.InitContainers) != 0 {
 		pod.Spec.InitContainers = append(pod.Spec.InitContainers, runnerSpec.InitContainers...)
 	}
@@ -414,9 +603,13 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	if runnerSpec.NodeSelector != nil {
 		pod.Spec.NodeSelector = runnerSpec.NodeSelector
 	}
+
 	if runnerSpec.ServiceAccountName != "" {
 		pod.Spec.ServiceAccountName = runnerSpec.ServiceAccountName
+	} else if r.UseRunnerStatusUpdateHook || runner.Spec.ContainerMode == "kubernetes" {
+		pod.Spec.ServiceAccountName = runner.ObjectMeta.Name
 	}
+
 	if runnerSpec.AutomountServiceAccountToken != nil {
 		pod.Spec.AutomountServiceAccountToken = runnerSpec.AutomountServiceAccountToken
 	}
@@ -437,6 +630,10 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		pod.Spec.Tolerations = runnerSpec.Tolerations
 	}
 
+	if runnerSpec.PriorityClassName != "" {
+		pod.Spec.PriorityClassName = runnerSpec.PriorityClassName
+	}
+
 	if len(runnerSpec.TopologySpreadConstraints) != 0 {
 		pod.Spec.TopologySpreadConstraints = runnerSpec.TopologySpreadConstraints
 	}
@@ -487,7 +684,45 @@ func mutatePod(pod *corev1.Pod, token string) *corev1.Pod {
 	return updated
 }
 
-func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, registrationOnly bool) (corev1.Pod, error) {
+func runnerHookEnvs(pod *corev1.Pod) ([]corev1.EnvVar, error) {
+	isRequireSameNode, err := isRequireSameNode(pod)
+	if err != nil {
+		return nil, err
+	}
+
+	return []corev1.EnvVar{
+		{
+			Name:  "ACTIONS_RUNNER_CONTAINER_HOOKS",
+			Value: defaultRunnerHookPath,
+		},
+		{
+			Name:  "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER",
+			Value: "true",
+		},
+		{
+			Name: "ACTIONS_RUNNER_POD_NAME",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "metadata.name",
+				},
+			},
+		},
+		{
+			Name: "ACTIONS_RUNNER_JOB_NAMESPACE",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "metadata.namespace",
+				},
+			},
+		},
+		corev1.EnvVar{
+			Name:  "ACTIONS_RUNNER_REQUIRE_SAME_NODE",
+			Value: strconv.FormatBool(isRequireSameNode),
+		},
+	}, nil
+}
+
+func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, useRunnerStatusUpdateHook bool) (corev1.Pod, error) {
 	var (
 		privileged                bool = true
 		dockerdInRunner           bool = runnerSpec.DockerdWithinRunnerContainer != nil && *runnerSpec.DockerdWithinRunnerContainer
@@ -496,11 +731,20 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 		dockerdInRunnerPrivileged bool = dockerdInRunner
 	)
 
+	if containerMode == "kubernetes" {
+		dockerdInRunner = false
+		dockerEnabled = false
+		dockerdInRunnerPrivileged = false
+	}
+
 	template = *template.DeepCopy()
 
 	// This label selector is used by default when rd.Spec.Selector is empty.
-	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerName)
+	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunner, "")
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyPodMutation, LabelValuePodMutation)
+	if runnerSpec.GitHubAPICredentialsFrom != nil {
+		template.ObjectMeta.Annotations = CloneAndAddLabel(template.ObjectMeta.Annotations, annotationKeyGitHubAPICredsSecret, runnerSpec.GitHubAPICredentialsFrom.SecretRef.Name)
+	}
 
 	workDir := runnerSpec.WorkDir
 	if workDir == "" {
@@ -530,11 +774,11 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 			Value: runnerSpec.Enterprise,
 		},
 		{
-			Name:  "RUNNER_LABELS",
+			Name:  EnvVarLabels,
 			Value: strings.Join(runnerSpec.Labels, ","),
 		},
 		{
-			Name:  "RUNNER_GROUP",
+			Name:  EnvVarGroup,
 			Value: runnerSpec.Group,
 		},
 		{
@@ -557,14 +801,10 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 			Name:  EnvVarEphemeral,
 			Value: fmt.Sprintf("%v", ephemeral),
 		},
-	}
-
-	if registrationOnly {
-		env = append(env, corev1.EnvVar{
-			Name:  "RUNNER_REGISTRATION_ONLY",
-			Value: "true",
+		{
+			Name:  "RUNNER_STATUS_UPDATE_HOOK",
+			Value: fmt.Sprintf("%v", useRunnerStatusUpdateHook),
 		},
-		)
 	}
 
 	var seLinuxOptions *corev1.SELinuxOptions
@@ -590,6 +830,17 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 		}
 	}
 
+	if containerMode == "kubernetes" {
+		if dockerdContainer != nil {
+			template.Spec.Containers = append(template.Spec.Containers[:dockerdContainerIndex], template.Spec.Containers[dockerdContainerIndex+1:]...)
+		}
+		if runnerContainerIndex < runnerContainerIndex {
+			runnerContainerIndex--
+		}
+		dockerdContainer = nil
+		dockerdContainerIndex = -1
+	}
+
 	if runnerContainer == nil {
 		runnerContainerIndex = -1
 		runnerContainer = &corev1.Container{
@@ -620,18 +871,26 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 	}
 
 	runnerContainer.Env = append(runnerContainer.Env, env...)
+	if containerMode == "kubernetes" {
+		hookEnvs, err := runnerHookEnvs(&template)
+		if err != nil {
+			return corev1.Pod{}, err
+		}
+		runnerContainer.Env = append(runnerContainer.Env, hookEnvs...)
+	}
 
 	if runnerContainer.SecurityContext == nil {
 		runnerContainer.SecurityContext = &corev1.SecurityContext{}
 	}
-	// Runner need to run privileged if it contains DinD
-	runnerContainer.SecurityContext.Privileged = &dockerdInRunnerPrivileged
+
+	if runnerContainer.SecurityContext.Privileged == nil {
+		// Runner need to run privileged if it contains DinD
+		runnerContainer.SecurityContext.Privileged = &dockerdInRunnerPrivileged
+	}
 
 	pod := template.DeepCopy()
 
-	if pod.Spec.RestartPolicy == "" {
-		pod.Spec.RestartPolicy = "OnFailure"
-	}
+	forceRunnerPodRestartPolicyNever(pod)
 
 	if mtu := runnerSpec.DockerMTU; mtu != nil && dockerdInRunner {
 		runnerContainer.Env = append(runnerContainer.Env, []corev1.EnvVar{
@@ -709,13 +968,18 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 			)
 		}
 
-		pod.Spec.Volumes = append(pod.Spec.Volumes,
-			corev1.Volume{
-				Name: "work",
-				VolumeSource: corev1.VolumeSource{
-					EmptyDir: &corev1.EmptyDirVolumeSource{},
+		if ok, _ := workVolumePresent(pod.Spec.Volumes); !ok {
+			pod.Spec.Volumes = append(pod.Spec.Volumes,
+				corev1.Volume{
+					Name: "work",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
 				},
-			},
+			)
+		}
+
+		pod.Spec.Volumes = append(pod.Spec.Volumes,
 			corev1.Volume{
 				Name: "certs-client",
 				VolumeSource: corev1.VolumeSource{
@@ -724,11 +988,16 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 			},
 		)
 
+		if ok, _ := workVolumeMountPresent(runnerContainer.VolumeMounts); !ok {
+			runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts,
+				corev1.VolumeMount{
+					Name:      "work",
+					MountPath: workDir,
+				},
+			)
+		}
+
 		runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts,
-			corev1.VolumeMount{
-				Name:      "work",
-				MountPath: workDir,
-			},
 			corev1.VolumeMount{
 				Name:      "certs-client",
 				MountPath: "/certs/client",
@@ -830,15 +1099,13 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 		}
 	}
 
-	// TODO Remove this once we remove RUNNER_FEATURE_FLAG_EPHEMERAL from runner's entrypoint.sh
-	// and make --ephemeral the default option.
-	if getRunnerEnv(pod, EnvVarRunnerFeatureFlagEphemeral) == "" {
-		setRunnerEnv(pod, EnvVarRunnerFeatureFlagEphemeral, EnvVarTrue)
-	}
-
 	return *pod, nil
 }
 
+func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, useRunnerStatusUpdateHookEphemeralRole bool) (corev1.Pod, error) {
+	return newRunnerPodWithContainerMode("", template, runnerSpec, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, useRunnerStatusUpdateHookEphemeralRole)
+}
+
 func (r *RunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	name := "runner-controller"
 	if r.Name != "" {
@@ -901,3 +1168,71 @@ func workVolumeMountPresent(items []corev1.VolumeMount) (bool, int) {
 	}
 	return false, 0
 }
+
+func applyWorkVolumeClaimTemplateToPod(pod *corev1.Pod, workVolumeClaimTemplate *v1alpha1.WorkVolumeClaimTemplate, workDir string) error {
+	if workVolumeClaimTemplate == nil {
+		return errors.New("work volume claim template must be specified in container mode kubernetes")
+	}
+	for i := range pod.Spec.Volumes {
+		if pod.Spec.Volumes[i].Name == "work" {
+			return fmt.Errorf("Work volume should not be specified in container mode kubernetes. workVolumeClaimTemplate field should be used instead.")
+		}
+	}
+	pod.Spec.Volumes = append(pod.Spec.Volumes, workVolumeClaimTemplate.V1Volume())
+
+	var runnerContainer *corev1.Container
+	for i := range pod.Spec.Containers {
+		if pod.Spec.Containers[i].Name == "runner" {
+			runnerContainer = &pod.Spec.Containers[i]
+			break
+		}
+	}
+
+	if runnerContainer == nil {
+		return fmt.Errorf("runner container is not present when applying work volume claim template")
+	}
+
+	if isPresent, _ := workVolumeMountPresent(runnerContainer.VolumeMounts); isPresent {
+		return fmt.Errorf("volume mount \"work\" should not be present on the runner container in container mode kubernetes")
+	}
+
+	runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts, workVolumeClaimTemplate.V1VolumeMount(workDir))
+
+	return nil
+}
+
+// isRequireSameNode specifies for the runner in kubernetes mode wether it should
+// schedule jobs to the same node where the runner is
+//
+// This function should only be called in containerMode: kubernetes
+func isRequireSameNode(pod *corev1.Pod) (bool, error) {
+	isPresent, index := workVolumePresent(pod.Spec.Volumes)
+	if !isPresent {
+		return true, errors.New("internal error: work volume mount must exist in containerMode: kubernetes")
+	}
+
+	if pod.Spec.Volumes[index].Ephemeral == nil || pod.Spec.Volumes[index].Ephemeral.VolumeClaimTemplate == nil {
+		return true, errors.New("containerMode: kubernetes should have pod.Spec.Volumes[].Ephemeral.VolumeClaimTemplate set")
+	}
+
+	for _, accessMode := range pod.Spec.Volumes[index].Ephemeral.VolumeClaimTemplate.Spec.AccessModes {
+		switch accessMode {
+		case corev1.ReadWriteOnce:
+			return true, nil
+		case corev1.ReadWriteMany:
+		default:
+			return true, errors.New("actions-runner-controller supports ReadWriteOnce and ReadWriteMany modes only")
+		}
+	}
+	return false, nil
+}
+
+func overwriteRunnerEnv(runner *v1alpha1.Runner, key string, value string) {
+	for i := range runner.Spec.Env {
+		if runner.Spec.Env[i].Name == key {
+			runner.Spec.Env[i].Value = value
+			return
+		}
+	}
+	runner.Spec.Env = append(runner.Spec.Env, corev1.EnvVar{Name: key, Value: value})
+}

controllers/runner_graceful_stop.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v47/github"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -113,10 +113,28 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 		// Happens e.g. when dind is in runner and run completes
 		log.Info("Runner pod has been stopped with a successful status.")
 	} else if pod != nil && pod.Annotations[AnnotationKeyRunnerCompletionWaitStartTimestamp] != "" {
-		log.Info("Runner pod is annotated to wait for completion")
+		ct := ephemeralRunnerContainerStatus(pod)
+		if ct == nil {
+			log.Info("Runner pod is annotated to wait for completion, and the runner container is not ephemeral")
 
-		return &ctrl.Result{RequeueAfter: retryDelay}, nil
-	} else if ok, err := unregisterRunner(ctx, ghClient, enterprise, organization, repository, runner, *runnerID); err != nil {
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		lts := ct.LastTerminationState.Terminated
+		if lts == nil {
+			log.Info("Runner pod is annotated to wait for completion, and the runner container is not restarting")
+
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		// Prevent runner pod from stucking in Terminating.
+		// See https://github.com/actions-runner-controller/actions-runner-controller/issues/1369
+		log.Info("Deleting runner pod anyway because it has stopped prematurely. This may leave a dangling runner resource in GitHub Actions",
+			"lastState.exitCode", lts.ExitCode,
+			"lastState.message", lts.Message,
+			"pod.phase", pod.Status.Phase,
+		)
+	} else if ok, err := unregisterRunner(ctx, ghClient, enterprise, organization, repository, *runnerID); err != nil {
 		if errors.Is(err, &gogithub.RateLimitError{}) {
 			// We log the underlying error when we failed calling GitHub API to list or unregisters,
 			// or the runner is still busy.
@@ -133,7 +151,10 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 
 		log.V(1).Info("Failed to unregister runner before deleting the pod.", "error", err)
 
-		var runnerBusy bool
+		var (
+			runnerBusy                         bool
+			runnerUnregistrationFailureMessage string
+		)
 
 		errRes := &gogithub.ErrorResponse{}
 		if errors.As(err, &errRes) {
@@ -155,6 +176,7 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 			}
 
 			runnerBusy = errRes.Response.StatusCode == 422
+			runnerUnregistrationFailureMessage = errRes.Message
 
 			if runnerBusy && code != nil {
 				log.V(2).Info("Runner container has already stopped but the unregistration attempt failed. "+
@@ -169,13 +191,18 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 		}
 
 		if runnerBusy {
+			_, err := annotatePodOnce(ctx, c, log, pod, AnnotationKeyUnregistrationFailureMessage, runnerUnregistrationFailureMessage)
+			if err != nil {
+				return &ctrl.Result{}, err
+			}
+
 			// We want to prevent spamming the deletion attemps but returning ctrl.Result with RequeueAfter doesn't
 			// work as the reconcilation can happen earlier due to pod status update.
 			// For ephemeral runners, we can expect it to stop and unregister itself on completion.
 			// So we can just wait for the completion without actively retrying unregistration.
 			ephemeral := getRunnerEnv(pod, EnvVarEphemeral)
 			if ephemeral == "true" {
-				pod, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerCompletionWaitStartTimestamp, time.Now().Format(time.RFC3339))
+				_, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerCompletionWaitStartTimestamp, time.Now().Format(time.RFC3339))
 				if err != nil {
 					return &ctrl.Result{}, err
 				}
@@ -352,7 +379,7 @@ func setRunnerEnv(pod *corev1.Pod, key, value string) {
 // There isn't a single right grace period that works for everyone.
 // The longer the grace period is, the earlier a cluster resource shortage can occur due to throttoled runner pod deletions,
 // while the shorter the grace period is, the more likely you may encounter the race issue.
-func unregisterRunner(ctx context.Context, client *github.Client, enterprise, org, repo, name string, id int64) (bool, error) {
+func unregisterRunner(ctx context.Context, client *github.Client, enterprise, org, repo string, id int64) (bool, error) {
 	// For the record, historically ARC did not try to call RemoveRunner on a busy runner, but it's no longer true.
 	// The reason ARC did so was to let a runner running a job to not stop prematurely.
 	//

controllers/runner_pod.go

@@ -0,0 +1,22 @@
+package controllers
+
+import corev1 "k8s.io/api/core/v1"
+
+// Force the runner pod managed by either RunnerDeployment and RunnerSet to have restartPolicy=Never.
+// See https://github.com/actions-runner-controller/actions-runner-controller/issues/1369 for more context.
+//
+// This is to prevent runner pods from stucking in Terminating when a K8s node disappeared along with the runnr pod and the runner container within it.
+//
+// Previously, we used restartPolicy of OnFailure, it turned wrong later, and therefore we now set Never.
+//
+// When the restartPolicy is OnFailure and the node disappeared, runner pods on the node seem to stuck in state.terminated==nil, state.waiting!=nil, and state.lastTerminationState!=nil,
+// and will ever become Running.
+// It's probably due to that the node onto which the pods have been scheduled will ever come back, hence the container restart attempt swill ever succeed,
+// the pods stuck waiting for successful restarts forever.
+//
+// By forcing runner pods to never restart, we hope there will be no chances of pods being stuck waiting.
+func forceRunnerPodRestartPolicyNever(pod *corev1.Pod) {
+	if pod.Spec.RestartPolicy != corev1.RestartPolicyNever {
+		pod.Spec.RestartPolicy = corev1.RestartPolicyNever
+	}
+}

controllers/runner_pod_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -31,8 +32,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	corev1 "k8s.io/api/core/v1"
-
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 // RunnerPodReconciler reconciles a Runner object
@@ -41,7 +40,7 @@ type RunnerPodReconciler struct {
 	Log                         logr.Logger
 	Recorder                    record.EventRecorder
 	Scheme                      *runtime.Scheme
-	GitHubClient                *github.Client
+	GitHubClient                *MultiGitHubClient
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
@@ -50,6 +49,7 @@ type RunnerPodReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -60,8 +60,11 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	_, isRunnerPod := runnerPod.Labels[LabelKeyRunnerSetName]
-	if !isRunnerPod {
+	_, isRunnerPod := runnerPod.Labels[LabelKeyRunner]
+	_, isRunnerSetPod := runnerPod.Labels[LabelKeyRunnerSetName]
+	_, isRunnerDeploymentPod := runnerPod.Labels[LabelKeyRunnerDeploymentName]
+
+	if !isRunnerPod && !isRunnerSetPod && !isRunnerDeploymentPod {
 		return ctrl.Result{}, nil
 	}
 
@@ -77,6 +80,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	var enterprise, org, repo string
+	var isContainerMode bool
 
 	for _, e := range envvars {
 		switch e.Name {
@@ -86,13 +90,25 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			org = e.Value
 		case EnvVarRepo:
 			repo = e.Value
+		case "ACTIONS_RUNNER_CONTAINER_HOOKS":
+			isContainerMode = true
 		}
 	}
 
+	ghc, err := r.GitHubClient.InitForRunnerPod(ctx, &runnerPod)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
 	if runnerPod.ObjectMeta.DeletionTimestamp.IsZero() {
 		finalizers, added := addFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
 
-		if added {
+		var cleanupFinalizersAdded bool
+		if isContainerMode {
+			finalizers, cleanupFinalizersAdded = addFinalizer(finalizers, runnerLinkedResourcesFinalizerName)
+		}
+
+		if added || cleanupFinalizersAdded {
 			newRunner := runnerPod.DeepCopy()
 			newRunner.ObjectMeta.Finalizers = finalizers
 
@@ -108,13 +124,34 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	} else {
 		log.V(2).Info("Seen deletion-timestamp is already set")
 
+		if finalizers, removed := removeFinalizer(runnerPod.ObjectMeta.Finalizers, runnerLinkedResourcesFinalizerName); removed {
+			if err := r.cleanupRunnerLinkedPods(ctx, &runnerPod, log); err != nil {
+				log.Info("Runner-linked pods clean up that has failed due to an error. If this persists, please manually remove the runner-linked pods to unblock ARC", "err", err.Error())
+				return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+			}
+			if err := r.cleanupRunnerLinkedSecrets(ctx, &runnerPod, log); err != nil {
+				log.Info("Runner-linked secrets clean up that has failed due to an error. If this persists, please manually remove the runner-linked secrets to unblock ARC", "err", err.Error())
+				return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+			}
+			patchedPod := runnerPod.DeepCopy()
+			patchedPod.ObjectMeta.Finalizers = finalizers
+
+			if err := r.Patch(ctx, patchedPod, client.MergeFrom(&runnerPod)); err != nil {
+				log.Error(err, "Failed to update runner for finalizer linked resources removal")
+				return ctrl.Result{}, err
+			}
+
+			// Otherwise the subsequent patch request can revive the removed finalizer and it will trigger a unnecessary reconcilation
+			runnerPod = *patchedPod
+		}
+
 		finalizers, removed := removeFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
 
 		if removed {
 			// In a standard scenario, the upstream controller, like runnerset-controller, ensures this runner to be gracefully stopped before the deletion timestamp is set.
 			// But for the case that the user manually deleted it for whatever reason,
 			// we have to ensure it to gracefully stop now.
-			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 			if res != nil {
 				return *res, err
 			}
@@ -130,6 +167,8 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 			log.V(2).Info("Removed finalizer")
 
+			r.GitHubClient.DeinitForRunnerPod(updatedPod)
+
 			return ctrl.Result{}, nil
 		}
 
@@ -168,7 +207,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
-	po, res, err := ensureRunnerPodRegistered(ctx, log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+	po, res, err := ensureRunnerPodRegistered(ctx, log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 	if res != nil {
 		return *res, err
 	}
@@ -182,7 +221,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		//
 		// In a standard scenario, ARC starts the unregistration process before marking the pod for deletion at all,
 		// so that it isn't subject to terminationGracePeriod and can safely take hours to finish it's work.
-		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 		if res != nil {
 			return *res, err
 		}
@@ -222,3 +261,93 @@ func (r *RunnerPodReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Named(name).
 		Complete(r)
 }
+
+func (r *RunnerPodReconciler) cleanupRunnerLinkedPods(ctx context.Context, pod *corev1.Pod, log logr.Logger) error {
+	var runnerLinkedPodList corev1.PodList
+	if err := r.List(ctx, &runnerLinkedPodList, client.InNamespace(pod.Namespace), client.MatchingLabels(
+		map[string]string{
+			"runner-pod": pod.ObjectMeta.Name,
+		},
+	)); err != nil {
+		return fmt.Errorf("failed to list runner-linked pods: %w", err)
+	}
+
+	var (
+		wg   sync.WaitGroup
+		errs []error
+	)
+	for _, p := range runnerLinkedPodList.Items {
+		if !p.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		p := p
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := r.Delete(ctx, &p); err != nil {
+				if kerrors.IsNotFound(err) || kerrors.IsGone(err) {
+					return
+				}
+				errs = append(errs, fmt.Errorf("delete pod %q error: %v", p.ObjectMeta.Name, err))
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			log.Error(err, "failed to remove runner-linked pod")
+		}
+		return errors.New("failed to remove some runner linked pods")
+	}
+
+	return nil
+}
+
+func (r *RunnerPodReconciler) cleanupRunnerLinkedSecrets(ctx context.Context, pod *corev1.Pod, log logr.Logger) error {
+	log.V(2).Info("Listing runner-linked secrets to be deleted", "ns", pod.Namespace)
+
+	var runnerLinkedSecretList corev1.SecretList
+	if err := r.List(ctx, &runnerLinkedSecretList, client.InNamespace(pod.Namespace), client.MatchingLabels(
+		map[string]string{
+			"runner-pod": pod.ObjectMeta.Name,
+		},
+	)); err != nil {
+		return fmt.Errorf("failed to list runner-linked secrets: %w", err)
+	}
+
+	var (
+		wg   sync.WaitGroup
+		errs []error
+	)
+	for _, s := range runnerLinkedSecretList.Items {
+		if !s.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		s := s
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := r.Delete(ctx, &s); err != nil {
+				if kerrors.IsNotFound(err) || kerrors.IsGone(err) {
+					return
+				}
+				errs = append(errs, fmt.Errorf("delete secret %q error: %v", s.ObjectMeta.Name, err))
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			log.Error(err, "failed to remove runner-linked secret")
+		}
+		return errors.New("failed to remove some runner linked secrets")
+	}
+
+	return nil
+}

controllers/runnerdeployment_controller.go

@@ -165,6 +165,8 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, err
 		}
 
+		log.V(1).Info("Updated runnerreplicaset due to selector change")
+
 		// At this point, we are already sure that there's no need to create a new replicaset
 		// as the runner template hash is not changed.
 		//
@@ -179,7 +181,17 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	newDesiredReplicas := getIntOrDefault(desiredRS.Spec.Replicas, defaultReplicas)
 
 	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
-	if currentDesiredReplicas != newDesiredReplicas {
+	//
+	// If we missed taking the EffectiveTime diff into account, you might end up experiencing scale-ups being delayed scale-down.
+	// See https://github.com/actions-runner-controller/actions-runner-controller/pull/1477#issuecomment-1164154496
+	var et1, et2 time.Time
+	if newestSet.Spec.EffectiveTime != nil {
+		et1 = newestSet.Spec.EffectiveTime.Time
+	}
+	if rd.Spec.EffectiveTime != nil {
+		et2 = rd.Spec.EffectiveTime.Time
+	}
+	if currentDesiredReplicas != newDesiredReplicas || et1 != et2 {
 		newestSet.Spec.Replicas = &newDesiredReplicas
 		newestSet.Spec.EffectiveTime = rd.Spec.EffectiveTime
 
@@ -189,6 +201,13 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, err
 		}
 
+		log.V(1).Info("Updated runnerreplicaset due to spec change",
+			"currentDesiredReplicas", currentDesiredReplicas,
+			"newDesiredReplicas", newDesiredReplicas,
+			"currentEffectiveTime", newestSet.Spec.EffectiveTime,
+			"newEffectiveTime", rd.Spec.EffectiveTime,
+		)
+
 		return ctrl.Result{}, err
 	}
 
@@ -421,9 +440,7 @@ func getSelector(rd *v1alpha1.RunnerDeployment) *metav1.LabelSelector {
 func newRunnerReplicaSet(rd *v1alpha1.RunnerDeployment, commonRunnerLabels []string, scheme *runtime.Scheme) (*v1alpha1.RunnerReplicaSet, error) {
 	newRSTemplate := *rd.Spec.Template.DeepCopy()
 
-	for _, l := range commonRunnerLabels {
-		newRSTemplate.Spec.Labels = append(newRSTemplate.Spec.Labels, l)
-	}
+	newRSTemplate.Spec.Labels = append(newRSTemplate.Spec.Labels, commonRunnerLabels...)
 
 	templateHash := ComputeHash(&newRSTemplate)
 

controllers/runnerreplicaset_controller.go

@@ -32,17 +32,15 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 // RunnerReplicaSetReconciler reconciles a Runner object
 type RunnerReplicaSetReconciler struct {
 	client.Client
-	Log          logr.Logger
-	Recorder     record.EventRecorder
-	Scheme       *runtime.Scheme
-	GitHubClient *github.Client
-	Name         string
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
 }
 
 const (
@@ -205,7 +203,3 @@ func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Named(name).
 		Complete(r)
 }
-
-func registrationOnlyRunnerNameFor(rsName string) string {
-	return rsName + "-registration-only"
-}

controllers/runnerreplicaset_controller_test.go

@@ -52,15 +52,13 @@ func SetupTest(ctx2 context.Context) *corev1.Namespace {
 
 		runnersList = fake.NewRunnersList()
 		server = runnersList.GetServer()
-		ghClient := newGithubClient(server)
 
 		controller := &RunnerReplicaSetReconciler{
-			Client:       mgr.GetClient(),
-			Scheme:       scheme.Scheme,
-			Log:          logf.Log,
-			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient: ghClient,
-			Name:         "runnerreplicaset-" + ns.Name,
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     "runnerreplicaset-" + ns.Name,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")

controllers/runnerset_controller.go

@@ -45,12 +45,13 @@ type RunnerSetReconciler struct {
 	Recorder record.EventRecorder
 	Scheme   *runtime.Scheme
 
-	CommonRunnerLabels     []string
-	GitHubBaseURL          string
-	RunnerImage            string
-	RunnerImagePullSecrets []string
-	DockerImage            string
-	DockerRegistryMirror   string
+	CommonRunnerLabels        []string
+	GitHubClient              *MultiGitHubClient
+	RunnerImage               string
+	RunnerImagePullSecrets    []string
+	DockerImage               string
+	DockerRegistryMirror      string
+	UseRunnerStatusUpdateHook bool
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets,verbs=get;list;watch;create;update;patch;delete
@@ -58,6 +59,7 @@ type RunnerSetReconciler struct {
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update
 
@@ -79,6 +81,8 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	if !runnerSet.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForRunnerSet(runnerSet)
+
 		return ctrl.Result{}, nil
 	}
 
@@ -96,7 +100,7 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
-	desiredStatefulSet, err := r.newStatefulSet(runnerSet)
+	desiredStatefulSet, err := r.newStatefulSet(ctx, runnerSet)
 	if err != nil {
 		r.Recorder.Event(runnerSet, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -129,6 +133,12 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		owners = append(owners, &ss)
 	}
 
+	if res, err := syncVolumes(ctx, r.Client, log, req.Namespace, runnerSet, statefulsets); err != nil {
+		return ctrl.Result{}, err
+	} else if res != nil {
+		return *res, nil
+	}
+
 	res, err := syncRunnerPodsOwners(ctx, r.Client, log, effectiveTime, newDesiredReplicas, func() client.Object { return create.DeepCopy() }, ephemeral, owners)
 	if err != nil || res == nil {
 		return ctrl.Result{}, err
@@ -178,19 +188,50 @@ func getRunnerSetSelector(runnerSet *v1alpha1.RunnerSet) *metav1.LabelSelector {
 var LabelKeyPodMutation = "actions-runner-controller/inject-registration-token"
 var LabelValuePodMutation = "true"
 
-func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
+func (r *RunnerSetReconciler) newStatefulSet(ctx context.Context, runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
 	runnerSetWithOverrides := *runnerSet.Spec.DeepCopy()
 
-	for _, l := range r.CommonRunnerLabels {
-		runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, l)
-	}
+	runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, r.CommonRunnerLabels...)
 
 	template := corev1.Pod{
 		ObjectMeta: runnerSetWithOverrides.StatefulSetSpec.Template.ObjectMeta,
 		Spec:       runnerSetWithOverrides.StatefulSetSpec.Template.Spec,
 	}
 
-	pod, err := newRunnerPod(runnerSet.Name, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL, false)
+	if runnerSet.Spec.RunnerConfig.ContainerMode == "kubernetes" {
+		found := false
+		for i := range template.Spec.Containers {
+			if template.Spec.Containers[i].Name == containerName {
+				found = true
+			}
+		}
+		if !found {
+			template.Spec.Containers = append(template.Spec.Containers, corev1.Container{
+				Name: "runner",
+			})
+		}
+
+		workDir := runnerSet.Spec.RunnerConfig.WorkDir
+		if workDir == "" {
+			workDir = "/runner/_work"
+		}
+		if err := applyWorkVolumeClaimTemplateToPod(&template, runnerSet.Spec.WorkVolumeClaimTemplate, workDir); err != nil {
+			return nil, err
+		}
+
+		template.Spec.ServiceAccountName = runnerSet.Spec.ServiceAccountName
+	}
+
+	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerSet.Name)
+
+	ghc, err := r.GitHubClient.InitForRunnerSet(ctx, runnerSet)
+	if err != nil {
+		return nil, err
+	}
+
+	githubBaseURL := ghc.GithubBaseURL
+
+	pod, err := newRunnerPodWithContainerMode(runnerSet.Spec.RunnerConfig.ContainerMode, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, githubBaseURL, r.UseRunnerStatusUpdateHook)
 	if err != nil {
 		return nil, err
 	}

controllers/schedule_test.go

@@ -605,3 +605,13 @@ func parseAndMatchRecurringPeriod(now time.Time, start, end, frequency, until st
 
 	return MatchSchedule(now, startTime, endTime, RecurrenceRule{Frequency: frequency, UntilTime: untilTime})
 }
+
+func FuzzMatchSchedule(f *testing.F) {
+	start := time.Now()
+	end := time.Now()
+	now := time.Now()
+	f.Fuzz(func(t *testing.T, freq string) {
+		// Verify that it never panics
+		_, _, _ = MatchSchedule(now, start, end, RecurrenceRule{Frequency: freq})
+	})
+}

controllers/sync_volumes.go

@@ -0,0 +1,185 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	labelKeyCleanup               = "pending-cleanup"
+	labelKeyRunnerStatefulSetName = "runner-statefulset-name"
+)
+
+func syncVolumes(ctx context.Context, c client.Client, log logr.Logger, ns string, runnerSet *v1alpha1.RunnerSet, statefulsets []appsv1.StatefulSet) (*ctrl.Result, error) {
+	log = log.WithValues("ns", ns)
+
+	for _, t := range runnerSet.Spec.StatefulSetSpec.VolumeClaimTemplates {
+		for _, sts := range statefulsets {
+			pvcName := fmt.Sprintf("%s-%s-0", t.Name, sts.Name)
+
+			var pvc corev1.PersistentVolumeClaim
+			if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+				if !kerrors.IsNotFound(err) {
+					return nil, err
+				}
+				continue
+			}
+
+			// TODO move this to statefulset reconciler so that we spam this less,
+			// by starting the loop only after the statefulset got deletionTimestamp set.
+			// Perhaps you can just wrap this in a finalizer here.
+			if pvc.Labels[labelKeyRunnerStatefulSetName] == "" {
+				updated := pvc.DeepCopy()
+				updated.Labels[labelKeyRunnerStatefulSetName] = sts.Name
+				if err := c.Update(ctx, updated); err != nil {
+					return nil, err
+				}
+				log.V(1).Info("Added runner-statefulset-name label to PVC", "sts", sts.Name, "pvc", pvcName)
+			}
+		}
+	}
+
+	// PVs are not namespaced hence we don't need client.InNamespace(ns).
+	// If we added that, c.List will silently return zero items.
+	//
+	// This `List` needs to be done in a dedicated reconciler that is registered to the manager via the `For` func.
+	// Otherwise the List func might return outdated contents(I saw status.phase being Bound even after K8s updated it to Released, and it lasted minutes).
+	//
+	// cleanupLabels := map[string]string{
+	// 	labelKeyCleanup: runnerSet.Name,
+	// }
+	// pvList := &corev1.PersistentVolumeList{}
+	// if err := c.List(ctx, pvList, client.MatchingLabels(cleanupLabels)); err != nil {
+	// 	log.Info("retrying pv listing", "ns", ns, "err", err)
+	// 	return nil, err
+	// }
+
+	return nil, nil
+}
+
+func syncPVC(ctx context.Context, c client.Client, log logr.Logger, ns string, pvc *corev1.PersistentVolumeClaim) (*ctrl.Result, error) {
+	stsName := pvc.Labels[labelKeyRunnerStatefulSetName]
+	if stsName == "" {
+		return nil, nil
+	}
+
+	log.V(2).Info("Reconciling runner PVC")
+
+	// TODO: Probably we'd better remove PVCs related to the RunnetSet that is nowhere now?
+	// Otherwise, a bunch of continuously recreated StatefulSet
+	// can leave dangling PVCs forever, which might stress the cluster.
+
+	var sts appsv1.StatefulSet
+	if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: stsName}, &sts); err != nil {
+		if !kerrors.IsNotFound(err) {
+			return nil, err
+		}
+	} else {
+		// We assume that the statefulset is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(1).Info("Retrying sync until statefulset gets removed", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	log = log.WithValues("sts", stsName)
+
+	pvName := pvc.Spec.VolumeName
+
+	if pvName != "" {
+		// If we deleted PVC before unsetting pv.spec.claimRef,
+		// K8s seems to revive the claimRef :thinking:
+		// So we need to mark PV for claimRef unset first, and delete PVC, and finally unset claimRef on PV.
+
+		var pv corev1.PersistentVolume
+		if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvName}, &pv); err != nil {
+			if !kerrors.IsNotFound(err) {
+				return nil, err
+			}
+			return nil, nil
+		}
+
+		pvCopy := pv.DeepCopy()
+		if pvCopy.Labels == nil {
+			pvCopy.Labels = map[string]string{}
+		}
+		pvCopy.Labels[labelKeyCleanup] = stsName
+
+		log.V(2).Info("Scheduling to unset PV's claimRef", "pv", pv.Name)
+
+		// Apparently K8s doesn't reconcile PV immediately after PVC deletion.
+		// So we start a relatively busy loop of PV reconcilation slightly before the PVC deletion,
+		// so that PV can be unbound as soon as possible after the PVC got deleted.
+		if err := c.Update(ctx, pvCopy); err != nil {
+			return nil, err
+		}
+
+		log.Info("Updated PV to unset claimRef")
+
+		// At this point, the PV is still Bound
+
+		log.V(2).Info("Deleting unused PVC")
+
+		if err := c.Delete(ctx, pvc); err != nil {
+			return nil, err
+		}
+
+		log.Info("Deleted unused PVC")
+
+		// At this point, the PV is still "Bound", but we are ready to unset pv.spec.claimRef in pv controller.
+		// Once the pv controller unsets claimRef, the PV becomes "Released", hence available for reuse by another eligible PVC.
+	}
+
+	return nil, nil
+}
+
+func syncPV(ctx context.Context, c client.Client, log logr.Logger, ns string, pv *corev1.PersistentVolume) (*ctrl.Result, error) {
+	if pv.Spec.ClaimRef == nil {
+		return nil, nil
+	}
+
+	log.V(2).Info("Reconciling PV")
+
+	if pv.Labels[labelKeyCleanup] == "" {
+		// We assume that the pvc is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(2).Info("Retrying sync to see if this PV needs to be managed by ARC", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	log.V(2).Info("checking pv phase", "phase", pv.Status.Phase)
+
+	if pv.Status.Phase != corev1.VolumeReleased {
+		// We assume that the pvc is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(1).Info("Retrying sync until pvc gets released", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	// At this point, the PV is still Released
+
+	pvCopy := pv.DeepCopy()
+	delete(pvCopy.Labels, labelKeyCleanup)
+	pvCopy.Spec.ClaimRef = nil
+	log.V(2).Info("Unsetting PV's claimRef", "pv", pv.Name)
+	if err := c.Update(ctx, pvCopy); err != nil {
+		return nil, err
+	}
+
+	log.Info("PV should be Available now")
+
+	// At this point, the PV becomes Available, if it's reclaim policy is "Retain".
+	// I have not yet tested it with "Delete" but perhaps it's deleted automatically after the update?
+	// https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain
+
+	return nil, nil
+}

controllers/testresourcereader.go

@@ -0,0 +1,31 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"reflect"
+
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type testResourceReader struct {
+	objects map[types.NamespacedName]client.Object
+}
+
+func (r *testResourceReader) Get(_ context.Context, nsName types.NamespacedName, obj client.Object) error {
+	ret, ok := r.objects[nsName]
+	if !ok {
+		return &kerrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonNotFound}}
+	}
+	v := reflect.ValueOf(obj)
+	if v.Kind() != reflect.Ptr {
+		return errors.New("obj must be a pointer")
+	}
+
+	v.Elem().Set(reflect.ValueOf(ret).Elem())
+
+	return nil
+}

controllers/testresourcereader_test.go

@@ -0,0 +1,35 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestResourceReader(t *testing.T) {
+	rr := &testResourceReader{
+		objects: map[types.NamespacedName]client.Object{
+			{Namespace: "default", Name: "sec1"}: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "sec1",
+				},
+				Data: map[string][]byte{
+					"foo": []byte("bar"),
+				},
+			},
+		},
+	}
+
+	var sec corev1.Secret
+
+	err := rr.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "sec1"}, &sec)
+	require.NoError(t, err)
+
+	require.Equal(t, []byte("bar"), sec.Data["foo"])
+}

controllers/utils_test.go

@@ -3,6 +3,9 @@ package controllers
 import (
 	"reflect"
 	"testing"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 )
 
 func Test_filterLabels(t *testing.T) {
@@ -32,3 +35,94 @@ func Test_filterLabels(t *testing.T) {
 		})
 	}
 }
+
+func Test_workVolumeClaimTemplateVolumeV1VolumeTransformation(t *testing.T) {
+	storageClassName := "local-storage"
+	workVolumeClaimTemplate := v1alpha1.WorkVolumeClaimTemplate{
+		StorageClassName: storageClassName,
+		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+		Resources:        corev1.ResourceRequirements{},
+	}
+	want := corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+						StorageClassName: &storageClassName,
+						Resources:        corev1.ResourceRequirements{},
+					},
+				},
+			},
+		},
+	}
+
+	got := workVolumeClaimTemplate.V1Volume()
+
+	if got.Name != want.Name {
+		t.Errorf("want name %q, got %q\n", want.Name, got.Name)
+	}
+
+	if got.VolumeSource.Ephemeral == nil {
+		t.Fatal("work volume claim template should transform itself into Ephemeral volume source\n")
+	}
+
+	if got.VolumeSource.Ephemeral.VolumeClaimTemplate == nil {
+		t.Fatal("work volume claim template should have ephemeral volume claim template set\n")
+	}
+
+	gotClassName := *got.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName
+	wantClassName := *want.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName
+	if gotClassName != wantClassName {
+		t.Errorf("expected storage class name %q, got %q\n", wantClassName, gotClassName)
+	}
+
+	gotAccessModes := got.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes
+	wantAccessModes := want.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes
+	if len(gotAccessModes) != len(wantAccessModes) {
+		t.Fatalf("access modes lengths missmatch: got %v, expected %v\n", gotAccessModes, wantAccessModes)
+	}
+
+	diff := make(map[corev1.PersistentVolumeAccessMode]int, len(wantAccessModes))
+	for _, am := range wantAccessModes {
+		diff[am]++
+	}
+
+	for _, am := range gotAccessModes {
+		_, ok := diff[am]
+		if !ok {
+			t.Errorf("got access mode %v that is not in the wanted access modes\n", am)
+		}
+
+		diff[am]--
+		if diff[am] == 0 {
+			delete(diff, am)
+		}
+	}
+
+	if len(diff) != 0 {
+		t.Fatalf("got access modes did not take every access mode into account\nactual: %v expected: %v\n", gotAccessModes, wantAccessModes)
+	}
+}
+
+func Test_workVolumeClaimTemplateV1VolumeMount(t *testing.T) {
+
+	workVolumeClaimTemplate := v1alpha1.WorkVolumeClaimTemplate{
+		StorageClassName: "local-storage",
+		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+		Resources:        corev1.ResourceRequirements{},
+	}
+
+	mountPath := "/test/_work"
+	want := corev1.VolumeMount{
+		MountPath: mountPath,
+		Name:      "work",
+	}
+
+	got := workVolumeClaimTemplate.V1VolumeMount(mountPath)
+
+	if want != got {
+		t.Fatalf("expected volume mount %+v, actual %+v\n", want, got)
+	}
+}

docs/releasenotes/0.24.md

@@ -0,0 +1,54 @@
+# actions-runner-controller v0.24.0
+
+All changes in this release can be found in the milestone https://github.com/actions-runner-controller/actions-runner-controller/milestone/4
+
+This log documents breaking and major enhancements
+
+## Upgrading
+
+In case you're using our Helm chart to deploy ARC, use the chart 0.19.0 or greater. Don't miss upgrading CRDs as usual! Helm doesn't upgrade CRDs.
+
+## BREAKING CHANGE : Support for `--once` is being dropped
+
+> **Warning**: If you're using ARC's official runer image, make sure to update the image tag to `v2.292.0` BEFORE upgrading ARC
+
+In #1385 we changed ARC to NOT automatically set the feature flag `RUNNER_FEATURE_FLAG_EPHEMERAL=true`. If you're using ARC's official runer image, make sure to update the image tag to `v2.292.0` before upgrading ARC, because that's the first runner image release since we changed the default to `--ephemeral`. If you kept using an older runner image after upgrading ARC, you end up using `--once` which is unreliable and had been deprecated since almost a year ago.
+
+>> **Warning**: If you're using a custom runner image, incorporate changes made in #1384 to your runner image dockerfile
+
+If you're building a custom runner image on your own and it still requires the user to specify `RUNNER_FEATURE_FLAG_EPHEMERAL=true` to use `--ephemeral`, check #1384 and update your custom runner image dockerfile accordingly. Otherwise, you may unexpectedly end up with using `--once` after upgrading ARC, because that was the previous default.
+
+Relevant PR(s): #1384, #1385
+
+## FIX : Prevent runner form stucking in Terminating when the container disappeared
+
+We occasionally heard about runnner pods stuck in Terminating after the node and containers running on it disappeared due to, for example, the machine terminated prematurely.
+
+We now set runner pods' restartPolicy to `Never` and remove runner pods stuck in `Waiting` after restarting, so that the pods are more likely to NOT stuck forever.
+
+Relevant PR(s): #1395, #1420
+
+## ENHANCEMENT : Support arbitrarily setting `privileged: true` for runner container
+
+This is a frequently asked feature that alows you to force `privileged: true` in case you don't need docker but still need privileged tasks to be run in a job step.
+
+In combination with a container runtime like `sysbox` this should enable you to run docker builds within the dind sidecar, all without privileges. See [the discussion related to Sysbox](https://github.com/actions-runner-controller/actions-runner-controller/discussions/977) for more information.
+
+Note that we ARC maintainers still have no bandwidth to provide a complete description on how to make ARC work with `sysbox` yet, but almost certainly we'd welcome contributions to the documentation if you managed to make it work.
+
+Relevant PR(s): #1383
+
+## ENHANCEMENT : RunnerSet can now retain PVs accross restarts
+
+This enhancement makes it more practical to use RunnerSet in combination with `volumeClaimTemplates` to make your workflow jobs faster.
+
+Please see our updated ["Custom Volume Mounts" section in the documentation](https://github.com/actions-runner-controller/actions-runner-controller#custom-volume-mounts) for more information. Currently, we cover caching Docker image layers, go mod/build, and PV-backed runner work directory(Although this one is backed by another feature unrelated to this enhancement under the hood).
+
+Relevant PR(s): #1340
+
+## ENHANCEMENT : OpenSSF scorecard adoption
+
+We assessed the project's security by following OpenSSF scorecard checks and adopting OpenSSF best practices.
+It should help you judge the security throughout ARC's development and release processes.
+
+Relevant PR(s): #1461

docs/releasenotes/0.25.md

@@ -0,0 +1,43 @@
+# actions-runner-controller v0.25.0
+
+All planned changes in this release can be found in the milestone https://github.com/actions-runner-controller/actions-runner-controller/milestone/8.
+
+Also see https://github.com/actions-runner-controller/actions-runner-controller/compare/v0.24.1...v0.25.0 for full changelog.
+
+This log documents breaking changes and major enhancements
+
+## Upgrading
+
+In case you're using our Helm chart to deploy ARC, use the chart 0.20.0 or greater. Don't miss upgrading CRDs as usual! Helm doesn't upgrade CRDs.
+
+## BREAKING CHANGE : Support for `--once` has been dropped
+
+In case you're still on ARC v0.23.0 or earlier, please also read [the relevant part of v0.24.0 release note for more information](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/docs/releasenotes/0.24.md#breaking-change--support-for---once-is-being-dropped).
+
+Relevant PR(s): #1580, #1590
+
+## ENHANCEMENT : Support for the new Kubernetes container mode of Actions runner
+
+The GitHub Actions team has recently added `actions/runner` an ability to use [runner container hooks](https://github.com/actions/runner-container-hooks) to run job steps on Kubernetes pods instead of docker containers created by the `docker` command. It allows us to avoid the use of privileged containers while still being able to run container-backed job steps.
+
+To use the new container mode, you set `.spec.template.spec.containerMode` in `RunnerDeployment` to `"kubernetes"`, while defining `.spec.template.spec.workVolumeClaimTemplate`. The volume claim template is used for provisioning and assigning persistent volumes mounted across the runner pod and the job pods for sharing the job workspace.
+
+Before using this feature, we highly recommend you to read [the detailed explanation in the original pull request](https://github.com/actions-runner-controller/actions-runner-controller/pull/1546), and [the new section in ARC's documentation](https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs).
+
+Big kudos to @thboop and the GitHub Actions team for implementing and contributing this feature!
+
+Relevant PR(s): #1546
+
+## FIX : Webhook-based scaling is even more reliable
+
+We fixed a race condition in the webhook-based autoscaler that resulted in not adding a runner when necessary.
+
+The race condition had been happening when it received a webhook event while processing another webhook event and both ended up scaling up the same horizontal runner autoscaler at the same time.
+
+To mitigate that, ARC now uses Kubernetes' Update API instead of Patch to update `HRA.spec.capacityReservations` which is the underlying data structure that makes the webhook-based scaler to add replicas to RunnerDeployment or RunnerSet on demand.
+
+We were also worried about stressing the Kubernetes apiserver when your ARC webhook-based autoscaler received a lot of concurrent webhook events, we also enhanced it to batch the Update API calls for 3 seconds, which basically means it will call the Update API at most once every 3 seconds per webhook-based autoscaler instance.
+
+Lastly, we fixed a bug in the autoscaler that resulted in it to stop adding replicas for newly received webhook events when the desired replicas reached `maxReplicas`.
+
+Relevant PR(s): #1477, #1568

docs/releasenotes/0.26.md

@@ -0,0 +1,99 @@
+# actions-runner-controller v0.26.0
+
+All planned changes in this release can be found in the milestone https://github.com/actions-runner-controller/actions-runner-controller/milestone/9.
+
+Also see https://github.com/actions-runner-controller/actions-runner-controller/compare/v0.24.2...v0.26.0 for full changelog.
+
+This log documents breaking changes and major enhancements
+
+## Upgrading
+
+In case you're using our Helm chart to deploy ARC, use the chart 0.21.0 or greater. Don't miss upgrading CRDs as usual! Helm doesn't upgrade CRDs.
+
+## BREAKING CHANGE : Min GHES version is now 3.6
+
+We've bumped the minimum requirement on GHES to [3.6.0](https://docs.github.com/en/enterprise-server@3.6/admin/release-notes#3.6.0) which has been released in August. The motivator for us was to use the new `visible_to_repository` option added to the list runner groups API for the runner group visibility based autoscaling which is crucial when you have a lot of runner groups that have non-distinct set of labels. If you don't use runner groups at all, ARC may just work, but YMMV.
+
+Relevant PR(s): #158
+
+## ENHANCEMENT : Rootless DinD runners
+
+An awesome GitHub staff added the support for rootless DinD powered runners. Compared to the standard DinD, a rootless DinD gives you an additional layer of security without losing the ability to invoke Docker containers and dokcer builds from within your workflow jobs. [If you aren't using the Kubernetes container mode](https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs), you should be using this new rootless DinD.
+
+Rootless DinD is the recent enhancement to Docker that basically allows you to run the Docker daemon and therefore Docker containers without the reliance on the `root` user. In the context of DinD(Docker-in-Docker) and ARC, this rootless DinD runner still requires a privileged container to function at all. But, the Linux user that runs the Docker daemon and the `actions/runner` agent can now be non-root, which is considered more secure than running DinD within a privileged container, as a random worfklow job is no longer able to run privileged operations.
+
+Before using this feature, we highly recommend you to read [the detailed explanation in the original pull request](https://github.com/actions-runner-controller/actions-runner-controller/pull/1644) and [the new section in ARC's documentation](https://github.com/actions-runner-controller/actions-runner-controller#runner-with-rootless-dind).
+
+Big kudos to @some-natalie for implementing and contributing this feature!
+
+Relevant PR(s): #1644
+
+## ENHANCEMENT : More granular and real-time runner statuses
+
+We added another controller flag and a Helm chart value to enable the new runner status update hook. Once enabled, it exposes more granular runner phases via the runner status.
+
+Previously, every `Runner` resource managed by `RunnerDeployment` was only able to expose these three Phases to e.g. `kubectl get runner` output:
+
+- `Pending`- The runner pod is waiting to be scheduled on any Kubernetes node/
+- `Running`-  The runner pod has been scheduled onto a node and its Linux namespace, containers, and the network has been set up. The primary processes of the containers are running.
+- `Succeeded`- The primary processes of the pod containers have stopped with exit status 0.
+
+As you may have realized, it had been quite useless, as it was a direct copy of the pod phase and tells almost nothing about the runner agent running inside the runner pod and the worfklow job that might be running.
+
+Since #1268 though, it can optionally provide two more phases, and the modified version of the `Running` phase. Once enabled via the controller command-line flag or the Helm chart value, you start to see:
+
+- `Registering`- The runner entrypoint started the runner registration process. Once the registration succeeds, it will update the phase to `Idle`.
+- `Idle`- The runner has been registered to GitHub and it's still waiting for GitHub to assign a workflow job to run.
+- `Running`- GitHub assigned a workflow job and the runner agent started running it.
+
+All the three phases should be more useful than before. For example, `Registering` can tell you that it's (still) unable to register itself against the GitHub Actions service. It it's hanging for minutes at the `Registering` phase, it's very likely you misconfigured your GitHub API credentials or you've somehow broken runner pods so that the runner is unable to register itself. If it's stuck in `Idle` like forever even though you queued some workflow runs and jobs, it's very likely you misconfigured runner labels or the `on` field of your workflow definitions.
+
+Big kudos to @fgalind1 for implementing and contributing this feature!
+
+Relevant PR(s): #1268
+
+## ENHANCEMENT : More Autoscaling-related metrics
+
+We added several more metrics related to the pull-based autoscaling so that you can scrape it via the [Prometheus exposition format](https://github.com/Showmax/prometheus-docs/blob/master/content/docs/instrumenting/exposition_formats.md), track and observe the changes on the graphing, dashboarding and alerting solution of your choice.
+
+For `PercentageRunnersBusy` metric, we added:
+
+- horizontalrunnerautoscaler_replicas_desired
+- horizontalrunnerautoscaler_runners
+- horizontalrunnerautoscaler_runners_registered
+- horizontalrunnerautoscaler_runners_busy
+- horizontalrunnerautoscaler_terminating_busy
+
+For `TotalNumberOfQueuedAndInProgressWorkflowRuns` metric, we added:
+
+- horizontalrunnerautoscaler_necessary_replicas
+- horizontalrunnerautoscaler_workflow_runs_completed
+- horizontalrunnerautoscaler_workflow_runs_in_progress
+- horizontalrunnerautoscaler_workflow_runs_queued
+- horizontalrunnerautoscaler_workflow_runs_unknown
+
+Big kudos to @debugger24 for implementing and contributing this feature!
+
+Relevant PR(s): #1720
+
+## ENHANCEMENT : Improved Multi-tenancy
+
+We had a long-living feature request about reducing the number of ARC instances one needs to maintain to provide self-hosted runners across multiple enterprises and organizations, and here it is. You can now manage as many enterprises and organizations with ARC.
+
+Previously you had to set up and manage an ARC instance per enterprise or in many cases per organization, because ARC was able to handle only one set of GitHub API credentials(PAT or GitHub App). The new multitenancy supports breaks this limitation by introducing the new `githubAPICredentialsFrom` field to the runner spec. You create a Kubernetes secret containing a GitHub API credentials and specify the secret name in `githubAPICredentialsFrom`, so that ARC picks it up and use it at the reconcilation time.
+
+We've written a detailed guide about this feature in the ["Multitenancy" section of the README](https://github.com/actions-runner-controller/actions-runner-controller#multitenancy). Please read it and give it a try!
+
+Lastly, this feature was stabilized by many early testers from the community. Big thanks and kudos to everyone who participated in testing, especially @Jalmeida1994 and @bm1216 for not only finding bugs but also contributing fixes ([#1725](https://github.com/actions-runner-controller/actions-runner-controller/pull/1725) and [#1781](https://github.com/actions-runner-controller/actions-runner-controller/pull/1781)!
+
+Relevant PR(s): #1268
+
+## ENHANCEMENT : Print ARC version number on startup
+
+Our build script now injects the version number of ARC into the executable, and prints it on startup so that you can see from logs that which version of ARC you're currently running. Previously when you are to file a bug report, you had to be extra sure to know which version of ARC you're using and encountering an issue. It's now easier than ever because you can grab the version number show in the logs, without consulting the container image tag of chart's appVersion.
+
+In addition to the logs, ARC is enhanced to send a HTTP `User-Agent` header containing the version number for every GitHub Actions API call ARC makes. You don't usually rely on it but GitHub and GitHub Actions's backend service can rely on it to collect the metrics about which versions of ARC folks are using.
+
+Big kudos to @ViktorLindgren95 for implementing and contributing this feature!
+
+Relevant PR(s): #1659

github/fake/runners.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v47/github"
 	"github.com/gorilla/mux"
 )
 

github/github.go

@@ -10,11 +10,12 @@ import (
 	"sync"
 	"time"
 
+	"github.com/actions-runner-controller/actions-runner-controller/build"
 	"github.com/actions-runner-controller/actions-runner-controller/github/metrics"
 	"github.com/actions-runner-controller/actions-runner-controller/logging"
-	"github.com/bradleyfalzon/ghinstallation"
+	"github.com/bradleyfalzon/ghinstallation/v2"
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v47/github"
 	"github.com/gregjones/httpcache"
 	"golang.org/x/oauth2"
 )
@@ -42,6 +43,7 @@ type Client struct {
 	mu        sync.Mutex
 	// GithubBaseURL to Github without API suffix.
 	GithubBaseURL string
+	IsEnterprise  bool
 }
 
 type BasicAuthTransport struct {
@@ -94,8 +96,10 @@ func (c *Config) NewClient() (*Client, error) {
 
 	var client *github.Client
 	var githubBaseURL string
+	var isEnterprise bool
 	if len(c.EnterpriseURL) > 0 {
 		var err error
+		isEnterprise = true
 		client, err = github.NewEnterpriseClient(c.EnterpriseURL, c.EnterpriseURL, httpClient)
 		if err != nil {
 			return nil, fmt.Errorf("enterprise client creation failed: %v", err)
@@ -134,14 +138,13 @@ func (c *Config) NewClient() (*Client, error) {
 			}
 		}
 	}
-
-	client.UserAgent = "actions-runner-controller"
-
+	client.UserAgent = "actions-runner-controller/" + build.Version
 	return &Client{
 		Client:        client,
 		regTokens:     map[string]*github.RegistrationToken{},
 		mu:            sync.Mutex{},
 		GithubBaseURL: githubBaseURL,
+		IsEnterprise:  isEnterprise,
 	}, nil
 }
 
@@ -243,37 +246,27 @@ func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string)
 	return runners, nil
 }
 
-// ListOrganizationRunnerGroups returns all the runner groups defined in the organization and
-// inherited to the organization from an enterprise.
-func (c *Client) ListOrganizationRunnerGroups(ctx context.Context, org string) ([]*github.RunnerGroup, error) {
-	var runnerGroups []*github.RunnerGroup
-
-	opts := github.ListOptions{PerPage: 100}
-	for {
-		list, res, err := c.Client.Actions.ListOrganizationRunnerGroups(ctx, org, &opts)
-		if err != nil {
-			return runnerGroups, fmt.Errorf("failed to list organization runner groups: %w", err)
-		}
-
-		runnerGroups = append(runnerGroups, list.RunnerGroups...)
-		if res.NextPage == 0 {
-			break
-		}
-		opts.Page = res.NextPage
-	}
-
-	return runnerGroups, nil
-}
-
 // ListOrganizationRunnerGroupsForRepository returns all the runner groups defined in the organization and
 // inherited to the organization from an enterprise.
 // We can remove this when google/go-github library is updated to support this.
 func (c *Client) ListOrganizationRunnerGroupsForRepository(ctx context.Context, org, repo string) ([]*github.RunnerGroup, error) {
 	var runnerGroups []*github.RunnerGroup
 
-	opts := github.ListOptions{PerPage: 100}
+	var opts github.ListOrgRunnerGroupOptions
+
+	opts.PerPage = 100
+
+	repoName := repo
+	parts := strings.Split(repo, "/")
+	if len(parts) == 2 {
+		repoName = parts[1]
+	}
+	// This must be the repo name without the owner part, so in case the repo is "myorg/myrepo" the repo name
+	// passed to visible_to_repository must be "myrepo".
+	opts.VisibleToRepository = repoName
+
 	for {
-		list, res, err := c.listOrganizationRunnerGroupsVisibleToRepo(ctx, org, repo, &opts)
+		list, res, err := c.Actions.ListOrganizationRunnerGroups(ctx, org, &opts)
 		if err != nil {
 			return runnerGroups, fmt.Errorf("failed to list organization runner groups: %w", err)
 		}
@@ -309,42 +302,6 @@ func (c *Client) ListRunnerGroupRepositoryAccesses(ctx context.Context, org stri
 	return repos, nil
 }
 
-// listOrganizationRunnerGroupsVisibleToRepo lists all self-hosted runner groups configured in an organization which can be used by the repository.
-//
-// GitHub API docs: https://docs.github.com/en/rest/reference/actions#list-self-hosted-runner-groups-for-an-organization
-func (c *Client) listOrganizationRunnerGroupsVisibleToRepo(ctx context.Context, org, repo string, opts *github.ListOptions) (*github.RunnerGroups, *github.Response, error) {
-	repoName := repo
-	parts := strings.Split(repo, "/")
-	if len(parts) == 2 {
-		repoName = parts[1]
-	}
-
-	u := fmt.Sprintf("orgs/%v/actions/runner-groups?visible_to_repository=%v", org, repoName)
-
-	if opts != nil {
-		if opts.PerPage > 0 {
-			u = fmt.Sprintf("%v&per_page=%v", u, opts.PerPage)
-		}
-
-		if opts.Page > 0 {
-			u = fmt.Sprintf("%v&page=%v", u, opts.Page)
-		}
-	}
-
-	req, err := c.Client.NewRequest("GET", u, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	groups := &github.RunnerGroups{}
-	resp, err := c.Client.Do(ctx, req, &groups)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return groups, resp, nil
-}
-
 // cleanup removes expired registration tokens.
 func (c *Client) cleanup() {
 	c.mu.Lock()
@@ -462,7 +419,6 @@ func splitOwnerAndRepo(repo string) (string, string, error) {
 	}
 	return chunk[0], chunk[1], nil
 }
-
 func getEnterpriseApiUrl(baseURL string) (string, error) {
 	baseEndpoint, err := url.Parse(baseURL)
 	if err != nil {