chore: bump chart to app 0.24.1 (#1531 )

e2e: Enable buildx in more images (#1530 )
chore: update bug_report.yml (#1529 )
2025-12-10 03:13:15 +00:00 · 2022-06-15 11:34:55 +09:00 · 2022-06-14 09:29:30 +01:00 · 2022-06-14 09:29:06 +01:00 · 2022-06-14 09:42:31 +09:00 · 2022-06-14 09:10:13 +09:00
60 changed files with 2259 additions and 541 deletions
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -17,6 +17,12 @@ body:
    label: Helm Chart Version
    description: Run `helm list` and see what's shown under CHART VERSION. Any release tags prefixed with `actions-runner-controller-` are for chart releases
    placeholder: ex. 0.11.0
+- type: input
+  id: cert-manager-version
+  attributes:
+    label: CertManager Version
+    description: Run `kubectl get po -o yaml $CERT_MANAGER_POD` and see the image tag, or run `helm list` and see what's shown under APP VERSION for your cert-manager Helm release.
+    placeholder: ex. 1.8
 - type: dropdown
  id: deployment-method
  attributes:
@@ -29,6 +35,17 @@ body:
      - Other
  validations:
    required: true
+- type: textarea
+  id: cert-manager
+  attributes:
+    label: cert-manager installation
+    description: Confirm that you've installed cert-manager correctly by answering a few questions
+    placeholder: |
+      - Did you follow https://github.com/actions-runner-controller/actions-runner-controller#installation? If not, describe the installation process so that we can reproduce your environment.
+      - Are you sure you've installed cert-manager from an official source?
+      (Note that we won't provide user support for cert-manager itself. Make sure cert-manager is fully working before testing ARC or reporting a bug
+  validations:
+    required: true
 - type: checkboxes
  id: checks
  attributes:
@@ -41,7 +58,7 @@ body:
      required: true
    - label: My actions-runner-controller version (v0.x.y) does support the feature
      required: true
-    - label: I've already upgraded ARC to the latest and it didn't fix the issue
+    - label: I've already upgraded ARC (including the CRDs, see charts/actions-runner-controller/docs/UPGRADING.md for details) to the latest and it didn't fix the issue
      required: true
 - type: textarea
  id: resource-definitions
@@ -113,7 +130,7 @@ body:
  id: controller-logs
  attributes:
    label: Controller Logs
-    description: "Include logs from `actions-runner-controller`'s controller-manager pod"
+    description: "NEVER EVER OMIT THIS! Include logs from `actions-runner-controller`'s controller-manager pod"
    render: shell
    placeholder: |
      To grab controller logs:
--- a/.github/actions/setup-docker-environment/action.yaml
+++ b/.github/actions/setup-docker-environment/action.yaml
@@ -29,23 +29,21 @@ runs:
      shell: bash

    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2

    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
      with:
        version: latest

    - name: Login to DockerHub
-      if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
      with:
        username: ${{ inputs.username }}
        password: ${{ inputs.password }}

    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v1
-      if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+      uses: docker/login-action@v2
      with:
        registry: ghcr.io
        username: ${{ inputs.ghcr_username }}
--- a/.github/lock.yml
+++ b/.github/lock.yml
@@ -1,25 +0,0 @@
-# Configuration for Lock Threads
-# Repo: https://github.com/dessant/lock-threads-app
-# App: https://github.com/apps/lock
-
-# Number of days of inactivity before a closed issue or pull request is locked
-daysUntilLock: 7
-
-# Skip issues and pull requests created before a given timestamp. Timestamp must
-# follow ISO 8601 (`YYYY-MM-DD`). Set to `false` to disable
-skipCreatedBefore: false
-
-# Issues and pull requests with these labels will be ignored. Set to `[]` to disable
-exemptLabels: []
-
-# Label to add before locking, such as `outdated`. Set to `false` to disable
-lockLabel: false
-
-# Comment to post before locking. Set to `false` to disable
-lockComment: >
-  This thread has been automatically locked since there has not been
-  any recent activity after it was closed. Please open a new issue for
-  related bugs.
-
-# Assign `resolved` as the reason for locking. Set to `false` to disable
-setLockReason: true
--- a/.github/renovate.json5
+++ b/.github/renovate.json5
@@ -13,7 +13,7 @@
    {
      // use https://github.com/actions/runner/releases
      "fileMatch": [
-        ".github/workflows/runners.yml"
+        ".github/workflows/runners.yaml"
      ],
      "matchStrings": ["RUNNER_VERSION: +(?<currentValue>.*?)\\n"],
      "depNameTemplate": "actions/runner",
@@ -30,8 +30,8 @@
    },
    {
      "fileMatch": [
-        "runner/Dockerfile",
-        "runner/Dockerfile.dindrunner"
+        "runner/actions-runner.dockerfile",
+        "runner/actions-runner-dind.dockerfile"
      ],
      "matchStrings": ["RUNNER_VERSION=+(?<currentValue>.*?)\\n"],
      "depNameTemplate": "actions/runner",
--- a/.github/workflows/publish-arc.yaml
+++ b/.github/workflows/publish-arc.yaml
@@ -1,26 +1,23 @@
-name: Publish Controller Image
+name: Publish ARC

 on:
  release:
-    types: [published]
+    types:
+      - published

 jobs:
-  build:
-    runs-on: ubuntu-latest
+  release-controller:
    name: Release
+    runs-on: ubuntu-latest
    env:
      DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
    steps:
-      - name: Set outputs
-        id: vars
-        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
-
      - name: Checkout
        uses: actions/checkout@v3

      - uses: actions/setup-go@v3
        with:
-          go-version: '1.17.7'
+          go-version: '1.18.2'

      - name: Install tools
        run: |
@@ -39,25 +36,20 @@ jobs:
      - name: Upload artifacts
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: make github-release
+        run: |
+          make github-release

-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
+      - name: Setup Docker Environment
+        id: vars
+        uses: ./.github/actions/setup-docker-environment
        with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
+          username: ${{ env.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+          ghcr_username: ${{ github.actor }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and Push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
        with:
          file: Dockerfile
          platforms: linux/amd64,linux/arm64
@@ -66,4 +58,6 @@ jobs:
            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest
            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}
            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

--- a/.github/workflows/publish-canary.yaml
+++ b/.github/workflows/publish-canary.yaml
@@ -0,0 +1,58 @@
+name: Publish Canary Image
+
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - '**.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/workflows/validate-chart.yaml'
+      - '.github/workflows/publish-chart.yaml'
+      - '.github/workflows/publish-arc.yaml'
+      - '.github/workflows/runners.yaml'
+      - '.github/workflows/validate-entrypoint.yaml'
+      - '.github/renovate.*'
+      - 'runner/**'
+      - '.gitignore'
+      - 'PROJECT'
+      - 'LICENSE'
+      - 'Makefile'
+
+# https://docs.github.com/en/rest/overview/permissions-required-for-github-apps
+permissions:
+  contents: read
+  packages: write  
+
+jobs:
+  canary-build:
+    name: Build and Publish Canary Image  
+    runs-on: ubuntu-latest
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Docker Environment
+        id: vars
+        uses: ./.github/actions/setup-docker-environment
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+          ghcr_username: ${{ github.actor }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Considered unstable builds
+      # See Issue #285, PR #286, and PR #323 for more information
+      - name: Build and Push
+        uses: docker/build-push-action@v3
+        with:
+          file: Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary
+            ghcr.io/actions-runner-controller/actions-runner-controller:canary
+          cache-from: type=gha,scope=arc-canary
+          cache-to: type=gha,mode=max,scope=arc-canary
--- a/.github/workflows/on-push-master-publish-chart.yml
+++ b/.github/workflows/on-push-master-publish-chart.yml
@@ -1,4 +1,4 @@
-name: Publish helm chart
+name: Publish Helm Chart

 on:
  push:
@@ -6,7 +6,7 @@ on:
      - master
    paths:
      - 'charts/**'
-      - '.github/workflows/on-push-master-publish-chart.yml'
+      - '.github/workflows/publish-chart.yaml'
      - '!charts/actions-runner-controller/docs/**'
      - '!**.md'
  workflow_dispatch:
@@ -15,10 +15,13 @@ env:
  KUBE_SCORE_VERSION: 1.10.0
  HELM_VERSION: v3.8.0

+permissions:
+  contents: read
+
 jobs:
  lint-chart:
-    runs-on: ubuntu-latest
    name: Lint Chart
+    runs-on: ubuntu-latest
    outputs:
      publish-chart: ${{ steps.publish-chart-step.outputs.publish }}
    steps:
@@ -49,9 +52,9 @@ jobs:
              --enable-optional-test container-security-context-readonlyrootfilesystem

      # python is a requirement for the chart-testing action below (supports yamllint among other tests)
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
        with:
-          python-version: 3.7
+          python-version: '3.7'

      - name: Set up chart-testing
        uses: helm/chart-testing-action@v2.2.1
@@ -65,22 +68,23 @@ jobs:
          fi

      - name: Run chart-testing (lint)
-        run: ct lint --config charts/.ci/ct-config.yaml
+        run: |
+          ct lint --config charts/.ci/ct-config.yaml

      - name: Create kind cluster
-        uses: helm/kind-action@v1.2.0
        if: steps.list-changed.outputs.changed == 'true'
+        uses: helm/kind-action@v1.2.0

      # We need cert-manager already installed in the cluster because we assume the CRDs exist
      - name: Install cert-manager
+        if: steps.list-changed.outputs.changed == 'true'      
        run: |
          helm repo add jetstack https://charts.jetstack.io --force-update
          helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
-        if: steps.list-changed.outputs.changed == 'true'

      - name: Run chart-testing (install)
-        run: ct install --config charts/.ci/ct-config.yaml
        if: steps.list-changed.outputs.changed == 'true'
+        run: ct install --config charts/.ci/ct-config.yaml

      # WARNING: This relies on the latest release being inat the top of the JSON from GitHub and a clean chart.yaml
      - name: Check if Chart Publish is Needed
@@ -99,8 +103,11 @@ jobs:
  publish-chart:
    if: needs.lint-chart.outputs.publish-chart == 'true'
    needs: lint-chart
-    runs-on: ubuntu-latest
    name: Publish Chart
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # for helm/chart-releaser-action to push chart release and create a release
+    

    steps:
      - name: Checkout
--- a/.github/workflows/run-codeql.yaml
+++ b/.github/workflows/run-codeql.yaml
@@ -0,0 +1,32 @@
+name: Run CodeQL
+
+on:
+  push:
+    branches: 
+      - master
+  pull_request:
+    branches:
+      - master
+  schedule:
+    - cron: '30 1 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: go
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
--- a/.github/workflows/run-stale.yaml
+++ b/.github/workflows/run-stale.yaml
@@ -1,12 +1,18 @@
-name: 'Close stale issues and PRs'
+name: Run Stale Bot
 on:
  schedule:
-    # 01:30 every day
    - cron: '30 1 * * *'

+permissions:
+  contents: read
+
 jobs:
  stale:
+    name: Run Stale
    runs-on: ubuntu-latest
+    permissions:
+      issues: write         # for actions/stale to close stale issues
+      pull-requests: write  # for actions/stale to close stale PRs
    steps:
      - uses: actions/stale@v5
        with:
--- a/.github/workflows/runners.yaml
+++ b/.github/workflows/runners.yaml
@@ -2,7 +2,7 @@ name: Runners

 on:
  pull_request:
-    types: 
+    types:
      - opened
      - synchronize
      - reopened
@@ -12,21 +12,21 @@ on:
    paths:
      - 'runner/**'
      - '!runner/Makefile'
-      - .github/workflows/runners.yml
+      - '.github/workflows/runners.yaml'
      - '!**.md'

 env:
-  RUNNER_VERSION: 2.290.1
+  RUNNER_VERSION: 2.293.0
  DOCKER_VERSION: 20.10.12
-  DOCKERHUB_USERNAME: summerwind
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}

 jobs:
-  build:
+  build-runners:
+    name: Build ${{ matrix.name }}-${{ matrix.os-name }}-${{ matrix.os-version }}
    runs-on: ubuntu-latest
    permissions:
      packages: write
      contents: read
-    name: Build ${{ matrix.name }}-${{ matrix.os-name }}-${{ matrix.os-version }}
    strategy:
      fail-fast: false
      matrix:
@@ -34,11 +34,9 @@ jobs:
          - name: actions-runner
            os-name: ubuntu
            os-version: 20.04
-            dockerfile: Dockerfile
          - name: actions-runner-dind
            os-name: ubuntu
            os-version: 20.04
-            dockerfile: Dockerfile.dindrunner

    steps:
      - name: Checkout
@@ -47,17 +45,17 @@ jobs:
      - name: Setup Docker Environment
        id: vars
        uses: ./.github/actions/setup-docker-environment
-        with: 
+        with:
          username: ${{ env.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
          ghcr_username: ${{ github.actor }}
          ghcr_password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and Push Versioned Tags
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
        with:
          context: ./runner
-          file: ./runner/${{ matrix.dockerfile }}
+          file: ./runner/${{ matrix.name }}.dockerfile
          platforms: linux/amd64,linux/arm64
          push: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
          build-args: |
@@ -70,5 +68,5 @@ jobs:
            ghcr.io/${{ github.repository }}/${{ matrix.name }}:latest
            ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
            ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=build-${{ matrix.name }}
+          cache-to: type=gha,mode=max,scope=build-${{ matrix.name }}
--- a/.github/workflows/validate-arc.yaml
+++ b/.github/workflows/validate-arc.yaml
@@ -1,45 +1,59 @@
-name: CI
+name: Validate ARC

 on:
  pull_request:
    branches:
      - master
    paths-ignore:
-      - .github/workflows/runners.yml
-      - .github/workflows/on-push-lint-charts.yml
-      - .github/workflows/on-push-master-publish-chart.yml
-      - .github/workflows/release.yml
-      - .github/workflows/test-entrypoint.yml
-      - .github/workflows/wip.yml
-      - 'runner/**'
      - '**.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/workflows/publish-canary.yaml'
+      - '.github/workflows/validate-chart.yaml'
+      - '.github/workflows/publish-chart.yaml'
+      - '.github/workflows/runners.yaml'
+      - '.github/workflows/publish-arc.yaml'
+      - '.github/workflows/validate-entrypoint.yaml'
+      - '.github/renovate.*'
+      - 'runner/**'
      - '.gitignore'
+      - 'PROJECT'
+      - 'LICENSE'
+      - 'Makefile'
+
+permissions:
+  contents: read

 jobs:
-  test:
+  test-controller:
+    name: Test ARC
    runs-on: ubuntu-latest
-    name: Test
    steps:
    - name: Checkout
      uses: actions/checkout@v3
-    - uses: actions/setup-go@v3
+
+    - name: Set-up Go
+      uses: actions/setup-go@v3
      with:
-        go-version: '1.17.7'
+        go-version: '1.18.2'
        check-latest: false
-    - run: go version
+    
    - uses: actions/cache@v3
      with:
        path: ~/go/pkg/mod
        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
        restore-keys: |
          ${{ runner.os }}-go-
+
    - name: Install kubebuilder
      run: |
        curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_linux_amd64.tar.gz
        tar zxvf kubebuilder_2.3.2_linux_amd64.tar.gz
        sudo mv kubebuilder_2.3.2_linux_amd64 /usr/local/kubebuilder
+
    - name: Run tests
-      run: make test
+      run: |
+        make test
+
    - name: Verify manifests are up-to-date
      run: |
        make manifests
--- a/.github/workflows/on-push-lint-charts.yml
+++ b/.github/workflows/on-push-lint-charts.yml
@@ -1,10 +1,10 @@
-name: Lint and Test Charts
+name: Validate Helm Chart

 on:
  push:
    paths:
      - 'charts/**'
-      - '.github/workflows/on-push-lint-charts.yml'
+      - '.github/workflows/validate-chart.yaml'
      - '!charts/actions-runner-controller/docs/**'
      - '!**.md'
  workflow_dispatch:
@@ -12,10 +12,13 @@ env:
  KUBE_SCORE_VERSION: 1.10.0
  HELM_VERSION: v3.8.0

+permissions:
+  contents: read
+
 jobs:
-  lint-test:
-    runs-on: ubuntu-latest
+  validate-chart:
    name: Lint Chart
+    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -44,9 +47,9 @@ jobs:
              --enable-optional-test container-security-context-readonlyrootfilesystem

      # python is a requirement for the chart-testing action below (supports yamllint among other tests)
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
        with:
-          python-version: 3.7
+          python-version: '3.7'

      - name: Set up chart-testing
        uses: helm/chart-testing-action@v2.2.1
@@ -60,7 +63,8 @@ jobs:
          fi

      - name: Run chart-testing (lint)
-        run: ct lint --config charts/.ci/ct-config.yaml
+        run: |
+          ct lint --config charts/.ci/ct-config.yaml

      - name: Create kind cluster
        uses: helm/kind-action@v1.2.0
@@ -68,10 +72,11 @@ jobs:

      # We need cert-manager already installed in the cluster because we assume the CRDs exist
      - name: Install cert-manager
+        if: steps.list-changed.outputs.changed == 'true'
        run: |
          helm repo add jetstack https://charts.jetstack.io --force-update
          helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
-        if: steps.list-changed.outputs.changed == 'true'

      - name: Run chart-testing (install)
-        run: ct install --config charts/.ci/ct-config.yaml
+        run: |
+          ct install --config charts/.ci/ct-config.yaml
--- a/.github/workflows/validate-runners.yaml
+++ b/.github/workflows/validate-runners.yaml
@@ -1,4 +1,4 @@
-name: Unit tests for entrypoint
+name: Validate Runners

 on:
  pull_request:
@@ -9,13 +9,17 @@ on:
      - 'test/entrypoint/**'
      - '!**.md'

+permissions:
+  contents: read
+
 jobs:
-  test:
-    runs-on: ubuntu-latest
+  test-runner-entrypoint:
    name: Test entrypoint
+    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v3
-    - name: Run unit tests for entrypoint.sh
+
+    - name: Run tests
      run: |
        make acceptance/runner/entrypoint
--- a/.github/workflows/wip.yml
+++ b/.github/workflows/wip.yml
@@ -1,51 +0,0 @@
-name: Publish Canary Image
-
-on:
-  push:
-    branches:
-      - master
-    paths-ignore:
-      - .github/workflows/runners.yml
-      - .github/workflows/on-push-lint-charts.yml
-      - .github/workflows/on-push-master-publish-chart.yml
-      - .github/workflows/release.yml
-      - .github/workflows/test-entrypoint.yml
-      - "runner/**"
-      - "**.md"
-      - ".gitignore"
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    name: Build and Publish Canary Image
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
-      # Considered unstable builds
-      # See Issue #285, PR #286, and PR #323 for more information
-      - name: Build and Push
-        uses: docker/build-push-action@v2
-        with:
-          file: Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary
--- a/2
+++ b/2
@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.17 as builder
+FROM --platform=$BUILDPLATFORM golang:1.18.2 as builder

 WORKDIR /workspace

--- a/9
+++ b/9
@@ -5,7 +5,7 @@ else
 endif
 DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
 VERSION ?= latest
-RUNNER_VERSION ?= 2.290.1
+RUNNER_VERSION ?= 2.293.0
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}
@@ -15,7 +15,6 @@ TEST_ORG_REPO ?=
 TEST_EPHEMERAL ?= false
 SYNC_PERIOD ?= 1m
 USE_RUNNERSET ?=
-RUNNER_FEATURE_FLAG_EPHEMERAL ?=
 KUBECONTEXT ?= kind-acceptance
 CLUSTER ?= acceptance
 CERT_MANAGER_VERSION ?= v1.1.1
@@ -57,6 +56,7 @@ GO_TEST_ARGS ?= -short
 # Run tests
 test: generate fmt vet manifests
 	go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
+	go test -fuzz=Fuzz -fuzztime=10s -run=Fuzz* ./controllers

 test-with-deps: kube-apiserver etcd kubectl
 	# See https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#pkg-constants
@@ -188,7 +188,6 @@ acceptance/deploy:
 	TEST_ORG=${TEST_ORG} TEST_ORG_REPO=${TEST_ORG_REPO} SYNC_PERIOD=${SYNC_PERIOD} \
 	USE_RUNNERSET=${USE_RUNNERSET} \
 	TEST_EPHEMERAL=${TEST_EPHEMERAL} \
-	RUNNER_FEATURE_FLAG_EPHEMERAL=${RUNNER_FEATURE_FLAG_EPHEMERAL} \
 	acceptance/deploy.sh

 acceptance/tests:
@@ -223,7 +222,7 @@ ifeq (, $(wildcard $(GOBIN)/controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 ;\
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 endif
@@ -243,7 +242,7 @@ ifeq (, $(wildcard $(GOBIN)/yq))
 	YQ_TMP_DIR=$$(mktemp -d) ;\
 	cd $$YQ_TMP_DIR ;\
 	go mod init tmp ;\
-	go get github.com/mikefarah/yq/v3@3.4.0 ;\
+	go install github.com/mikefarah/yq/v3@3.4.0 ;\
 	rm -rf $$YQ_TMP_DIR ;\
 	}
 endif
--- a/README.md
+++ b/README.md
@@ -1,11 +1,13 @@
 # actions-runner-controller (ARC)

+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6061/badge)](https://bestpractices.coreinfrastructure.org/projects/6061)
 [![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners)

 This controller operates self-hosted runners for GitHub Actions on your Kubernetes cluster.

 ToC:

+- [People](#people)
 - [Status](#status)
 - [About](#about)
 - [Installation](#installation)
@@ -13,14 +15,14 @@ ToC:
 - [Setting Up Authentication with GitHub API](#setting-up-authentication-with-github-api)
  - [Deploying Using GitHub App Authentication](#deploying-using-github-app-authentication)
  - [Deploying Using PAT Authentication](#deploying-using-pat-authentication)
- [Deploying Multiple Controllers](#deploying-multiple-controllers)  
+- [Deploying Multiple Controllers](#deploying-multiple-controllers)
 - [Usage](#usage)
  - [Repository Runners](#repository-runners)
  - [Organization Runners](#organization-runners)
  - [Enterprise Runners](#enterprise-runners)
  - [RunnerDeployments](#runnerdeployments)
  - [RunnerSets](#runnersets)
-  - [Persistent Runners](#persistent-runners)  
+  - [Persistent Runners](#persistent-runners)
  - [Autoscaling](#autoscaling)
    - [Anti-Flapping Configuration](#anti-flapping-configuration)
    - [Pull Driven Scaling](#pull-driven-scaling)
@@ -40,6 +42,20 @@ ToC:
 - [Contributing](#contributing)


+## People
+
+`actions-runner-controller` is an open-source project currently developed and maintained in collaboration with maintainers @mumoshu and @toast-gear, various [contributors](https://github.com/actions-runner-controller/actions-runner-controller/graphs/contributors), and the [awesome community](https://github.com/actions-runner-controller/actions-runner-controller/discussions), mostly in their spare time.
+
+If you think the project is awesome and it's becoming a basis for your important business, consider [sponsoring us](https://github.com/sponsors/actions-runner-controller)!
+
+In case you are already the employer of one of contributors, sponsoring via GitHub Sponsors might not be an option. Just support them in other means!
+
+We don't currently have [any sponsors dedicated to this project yet](https://github.com/sponsors/actions-runner-controller).
+
+However, [HelloFresh](https://www.hellofreshgroup.com/en/) has recently started sponsoring @mumoshu for this project along with his other works. A part of their sponsorship will enable @mumoshu to add an E2E test to keep ARC even more reliable on AWS. Thank you for your sponsorship!
+
+[<img src="https://user-images.githubusercontent.com/22009/170898715-07f02941-35ec-418b-8cd4-251b422fa9ac.png" width="219" height="71" />](https://careers.hellofresh.com/)
+
 ## Status

 Even though actions-runner-controller is used in production environments, it is still in its early stage of development, hence versioned 0.x.
@@ -59,7 +75,7 @@ By default, actions-runner-controller uses [cert-manager](https://cert-manager.i

 - [Installing cert-manager on Kubernetes](https://cert-manager.io/docs/installation/kubernetes/)

-Subsequent to this, install the custom resource definitions and actions-runner-controller with `kubectl` or `helm`. This will create an actions-runner-system namespace in your Kubernetes and deploy the required resources.
+After installing cert-manager, install the custom resource definitions and actions-runner-controller with `kubectl` or `helm`. This will create an actions-runner-system namespace in your Kubernetes and deploy the required resources.

 **Kubectl Deployment:**

@@ -207,7 +223,7 @@ Log-in to a GitHub account that has `admin` privileges for the repository, and [

 _Note: When you deploy enterprise runners they will get access to organizations, however, access to the repositories themselves is **NOT** allowed by default. Each GitHub organization must allow enterprise runner groups to be used in repositories as an initial one-time configuration step, this only needs to be done once after which it is permanent for that runner group._

-_Note: GitHub does not document exactly what permissions you get with each PAT scope beyond a vague description. The best documentation they provide on the topic can be found [here](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps) if you wish to review. The docs target OAuth apps and so are incomplete and may not be 100% accurate._ 
+_Note: GitHub does not document exactly what permissions you get with each PAT scope beyond a vague description. The best documentation they provide on the topic can be found [here](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps) if you wish to review. The docs target OAuth apps and so are incomplete and may not be 100% accurate._

 ---

@@ -429,7 +445,7 @@ spec:
      securityContext:
        # All level/role/type/user values will vary based on your SELinux policies.
        # See https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html/container_security_guide/docker_selinux_security_policy for information about SELinux with containers
-        seLinuxOptions: 
+        seLinuxOptions:
          level: "s0"
          role: "system_r"
          type: "super_t"
@@ -444,6 +460,17 @@ spec:
          requests:
            cpu: "2.0"
            memory: "4Gi"
+        # This is an advanced configuration. Don't touch it unless you know what you're doing.
+        securityContext:
+          # Usually, the runner container's privileged field is derived from dockerdWithinRunnerContainer.
+          # But in the case where you need to run privileged job steps even if you don't use docker/don't need dockerd within the runner container,
+          # just specified `privileged: true` like this.
+          # See https://github.com/actions-runner-controller/actions-runner-controller/issues/1282
+          # Do note that specifying `privileged: false` while using dind is very likely to fail, even if you use some vm-based container runtimes
+          # like firecracker and kata. Basically they run containers within dedicated micro vms and so
+          # it's more like you can use `privileged: true` safer with those runtimes.
+          #
+          # privileged: true
      - name: docker
        resources:
          limits:
@@ -461,7 +488,6 @@ Under the hood, `RunnerSet` relies on Kubernetes's `StatefulSet` and Mutating We
 **Limitations**

 * For autoscaling the `RunnerSet` kind only supports pull driven scaling or the `workflow_job` event for webhook driven scaling.
-* Whilst `RunnerSets` support all runner modes as well as autoscaling, currently PVs are **NOT** automatically cleaned up as they are still bound to their respective PVCs when a runner is deleted by the controller. This has **major** implications when using `RunnerSets` in the standard runner mode, `ephemeral: true`, see [persistent runners](#persistent-runners) for more details. As a result of this, using the default ephemeral configuration or implementing autoscaling for your `RunnerSets`, you will get a build-up of PVCs and PVs without some sort of custom solution for cleaning up.

 ### Persistent Runners

@@ -489,7 +515,7 @@ A `RunnerDeployment` or `RunnerSet` can scale the number of runners between `min

 #### Anti-Flapping Configuration

-For both pull driven or webhook driven scaling an anti-flapping implementation is included, by default a runner won't be scaled down within 10 minutes of it having been scaled up. 
+For both pull driven or webhook driven scaling an anti-flapping implementation is included, by default a runner won't be scaled down within 10 minutes of it having been scaled up.

 This anti-flap configuration also has the final say on if a runner can be scaled down or not regardless of the chosen scaling method.

@@ -536,7 +562,7 @@ spec:

 > To configure webhook driven scaling see the [Webhook Driven Scaling](#webhook-driven-scaling) section

-The pull based metrics are configured in the `metrics` attribute of a HRA (see snippet below). The period between polls is defined by the controller's `--sync-period` flag. If this flag isn't provided then the controller defaults to a sync period of `1m`, this can be configured in seconds or minutes. 
+The pull based metrics are configured in the `metrics` attribute of a HRA (see snippet below). The period between polls is defined by the controller's `--sync-period` flag. If this flag isn't provided then the controller defaults to a sync period of `1m`, this can be configured in seconds or minutes.

 Be aware that the shorter the sync period the quicker you will consume your rate limit budget, depending on your environment this may or may not be a risk. Consider monitoring ARCs rate limit budget when configuring this feature to find the optimal performance sync period.

@@ -554,7 +580,7 @@ spec:
  minReplicas: 1
  maxReplicas: 5
  # Your chosen scaling metrics here
-  metrics: [] 
+  metrics: []
 ```

 **Metric Options:**
@@ -706,24 +732,118 @@ _[see the values documentation for all configuration options](https://github.com
 ```console
 $ helm upgrade --install --namespace actions-runner-system --create-namespace \
             --wait actions-runner-controller actions-runner-controller/actions-runner-controller \
-             --set "githubWebhookServer.enabled=true,githubWebhookServer.ports[0].nodePort=33080"
+             --set "githubWebhookServer.enabled=true,service.type=NodePort,githubWebhookServer.ports[0].nodePort=33080"
 ```

-The above command will result in exposing the node port 33080 for Webhook events. Usually, you need to create an
-external load balancer targeted to the node port, and register the hostname or the IP address of the external load balancer
-to the GitHub Webhook.
+The above command will result in exposing the node port 33080 for Webhook events.
+Usually, you need to create an external load balancer targeted to the node port,
+and register the hostname or the IP address of the external load balancer to the GitHub Webhook.

-Once you were able to confirm that the Webhook server is ready and running from GitHub - this is usually verified by the
-GitHub sending PING events to the Webhook server - create or update your `HorizontalRunnerAutoscaler` resources
-by learning the following configuration examples.
+**With a custom Kubernetes ingress controller:**
+
+> **CAUTION:** The Kubernetes ingress controllers described below is just a suggestion from the community and
+> the ARC team will not provide any user support for ingress controllers as it's not a part of this project.
+>
+> The following guide on creating an ingress has been contributed by the awesome ARC community and is provided here as-is.
+> You may, however, still be able to ask for help on the community on GitHub Discussions if you have any problems.
+
+Kubernetes provides `Ingress` resources to let you configure your ingress controller to expose a Kubernetes service.
+If you plan to expose ARC via Ingress, you might not be required to make it a `NodePort` service
+(although nothing would prevent an ingress controller to expose NodePort services too):
+
+```console
+$ helm upgrade --install --namespace actions-runner-system --create-namespace \
+             --wait actions-runner-controller actions-runner-controller/actions-runner-controller \
+             --set "githubWebhookServer.enabled=true"
+```
+
+The command above will create a new deployment and a service for receiving Github Webhooks on the `actions-runner-system` namespace.
+
+Now we need to expose this service so that GitHub can send these webhooks over the network with TSL protection.
+
+You can do it in any way you prefer, here we'll suggest doing it with a k8s Ingress.
+For the sake of this example we'll expose this service on the following URL:
+
+- https://your.domain.com/actions-runner-controller-github-webhook-server
+
+Where `your.domain.com` should be replaced by your own domain.
+
+> Note: This step assumes you already have a configured `cert-manager` and domain name for your cluster.
+
+Let's start by creating an Ingress file called `arc-webhook-server.yaml` with the following contents:
+
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: actions-runner-controller-github-webhook-server
+  namespace: actions-runner-system
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
+spec:
+  tls:
+  - hosts:
+    - your.domain.com
+    secretName: your-tls-secret-name
+  rules:
+    - http:
+        paths:
+          - path: /actions-runner-controller-github-webhook-server
+            pathType: Prefix
+            backend:
+              service:
+                name: actions-runner-controller-github-webhook-server
+                port:
+                  number: 80
+```
+
+Make sure to set the `spec.tls.secretName` to the name of your TLS secret and
+`spec.tls.hosts[0]` to your own domain.
+
+Then create this resource on your cluster with the following command:
+
+```bash
+kubectl apply -n actions-runner-system -f arc-webhook-server.yaml
+```
+
+**Configuring GitHub for sending webhooks for our newly created webhook server:**
+
+After this step your webhook server should be ready to start receiving webhooks from GitHub.
+
+To configure GitHub to start sending you webhooks, go to the settings page of your repository
+or organization then click on `Webhooks`, then on `Add webhook`.
+
+There set the "Payload URL" field with the webhook URL you just created,
+if you followed the example ingress above the URL would be something like this:
+
+- https://your.domain.com/actions-runner-controller-github-webhook-server
+
+> Remember to replace `your.domain.com` with your own domain.
+
+Then click on "let me select individual events" and choose `Workflow Jobs`.
+
+You may also want to choose the following event(s) if you use it as a scale trigger in your HRA spec:
+
+- Check runs
+- Pushes
+- Pull Requests
+
+Later you can remove any of these you are not using to reduce the amount of data sent to your server.
+
+Then click on `Add Webhook`.
+
+GitHub will then send a `ping` event to your webhook server to check if it is working, if it is you'll see a green V mark
+alongside your webhook on the Settings -> Webhooks page.
+
+Once you were able to confirm that the Webhook server is ready and running from GitHub create or update your
+`HorizontalRunnerAutoscaler` resources by learning the following configuration examples.

 - [Example 1: Scale on each `workflow_job` event](#example-1-scale-on-each-workflow_job-event)
 - [Example 2: Scale up on each `check_run` event](#example-2-scale-up-on-each-check_run-event)
 - [Example 3: Scale on each `pull_request` event against a given set of branches](#example-3-scale-on-each-pull_request-event-against-a-given-set-of-branches)
 - [Example 4: Scale on each `push` event](#example-4-scale-on-each-push-event)

-**Note:** All these examples should have **minReplicas** & **maxReplicas** as mandatory parameters even for webhook driven scaling. 
-
 ##### Example 1: Scale on each `workflow_job` event

 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
@@ -735,16 +855,23 @@ The most flexible webhook GitHub offers is the `workflow_job` webhook, it includ
 This webhook should cover most people's needs, please experiment with this webhook first before considering the others.

 ```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
-   name: example-runners
+  name: example-runners
 spec:
  template:
    spec:
      repository: example/myrepo
 ---
+apiVersion: actions.summerwind.dev/v1alpha1
 kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runners
 spec:
+  scaleDownDelaySecondsAfterScaleOut: 300
+  minReplicas: 1
+  maxReplicas: 10
  scaleTargetRef:
    name: example-runners
    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
@@ -778,6 +905,8 @@ spec:
 ---
 kind: HorizontalRunnerAutoscaler
 spec:
+  minReplicas: 1
+  maxReplicas: 10
  scaleTargetRef:
    name: example-runners
    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
@@ -804,6 +933,8 @@ spec:
 ---
 kind: HorizontalRunnerAutoscaler
 spec:
+  minReplicas: 1
+  maxReplicas: 10
  scaleTargetRef:
    name: example-runners
    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
@@ -834,6 +965,8 @@ spec:
 ---
 kind: HorizontalRunnerAutoscaler
 spec:
+  minReplicas: 1
+  maxReplicas: 10
  scaleTargetRef:
    name: example-runners
    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
@@ -862,6 +995,8 @@ spec:
 ---
 kind: HorizontalRunnerAutoscaler
 spec:
+  minReplicas: 1
+  maxReplicas: 10
  scaleTargetRef:
    name: example-runners
    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
@@ -1079,7 +1214,7 @@ spec:
      # Valid only when dockerdWithinRunnerContainer=false
      dockerEnv:
        - name: HTTP_PROXY
-          value: http://example.com      
+          value: http://example.com
      # Docker sidecar container image tweaks examples below, only applicable if dockerdWithinRunnerContainer = false
      dockerdContainerResources:
        limits:
@@ -1138,13 +1273,26 @@ spec:
      # This must match the name of a RuntimeClass resource available on the cluster.
      # More info: https://kubernetes.io/docs/concepts/containers/runtime-class
      runtimeClassName: "runc"
+      # This is an advanced configuration. Don't touch it unless you know what you're doing.
+      containers:
+      - name: runner
+        # Usually, the runner container's privileged field is derived from dockerdWithinRunnerContainer.
+        # But in the case where you need to run privileged job steps even if you don't use docker/don't need dockerd within the runner container,
+        # just specified `privileged: true` like this.
+        # See https://github.com/actions-runner-controller/actions-runner-controller/issues/1282
+        # Do note that specifying `privileged: false` while using dind is very likely to fail, even if you use some vm-based container runtimes
+        # like firecracker and kata. Basically they run containers within dedicated micro vms and so
+        # it's more like you can use `privileged: true` safer with those runtimes.
+        #
+        # privileged: true
 ```

 ### Custom Volume mounts
 You can configure your own custom volume mounts. For example to have the work/docker data in memory or on NVME SSD, for
 i/o intensive builds. Other custom volume mounts should be possible as well, see [kubernetes documentation](https://kubernetes.io/docs/concepts/storage/volumes/)

-**RAM Disk Runner**<br />
+#### RAM Disk
+
 Example how to place the runner work dir, docker sidecar and /tmp within the runner onto a ramdisk.
 ```yaml
 kind: RunnerDeployment
@@ -1170,7 +1318,8 @@ spec:
      emphemeral: true # recommended to not leak data between builds.
 ```

-**NVME SSD Runner**<br />
+#### NVME SSD
+
 In this example we provide NVME backed storage for the workdir, docker sidecar and /tmp within the runner.
 Here we use a working example on GKE, which will provide the NVME disk at /mnt/disks/ssd0.  We will be placing the respective volumes in subdirs here and in order to be able to run multiple runners we will use the pod name as a prefix for subdirectories. Also the disk will fill up over time and disk space will not be freed until the node is removed.

@@ -1218,6 +1367,125 @@ spec:
    emphemeral: true # VERY important. otherwise data inside the workdir and /tmp is not cleared between builds
 ```

+#### Docker image layers caching
+
+> **Note**: Ensure that the volume mount is added to the container that is running the Docker daemon.
+
+`docker` stores pulled and built image layers in the [daemon's (note not client)](https://docs.docker.com/get-started/overview/#docker-architecture) [local storage area](https://docs.docker.com/storage/storagedriver/#sharing-promotes-smaller-images) which is usually at `/var/lib/docker`.
+
+By leveraging RunnerSet's dynamic PV provisioning feature and your CSI driver, you can let ARC maintain a pool of PVs that are
+reused across runner pods to retain `/var/lib/docker`.
+
+_Be sure to add the volume mount to the container that is supposed to run the docker daemon._
+
+By default, ARC creates a sidecar container named `docker` within the runner pod for running the docker daemon. In that case,
+it's where you need the volume mount so that the manifest looks like:
+
+```yaml
+kind: RunnerSet
+metadata:
+  name: example
+spec:
+  template:
+    spec:
+      containers:
+      - name: docker
+        volumeMounts:
+        - name: var-lib-docker
+          mountPath: /var/lib/docker
+  volumeClaimtemplates:
+  - metadata:
+      name: var-lib-docker
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: var-lib-docker
+```
+
+With `dockerdWithinRunnerContainer: true`, you need to add the volume mount to the `runner` container.
+
+#### Go module and build caching
+
+`Go` is known to cache builds under `$HOME/.cache/go-build` and downloaded modules under `$HOME/pkg/mod`.
+The module cache dir can be customized by setting `GOMOD_CACHE` so by setting it to somewhere under `$HOME/.cache`,
+we can have a single PV to host both build and module cache, which might improve Go module downloading and building time.
+
+```yaml
+kind: RunnerSet
+metadata:
+  name: example
+spec:
+  template:
+    spec:
+      containers:
+      - name: runner
+        env:
+        - name: GOMODCACHE
+          value: "/home/runner/.cache/go-mod"
+        volumeMounts:
+        - name: cache
+          mountPath: "/home/runner/.cache"
+  volumeClaimTemplates:
+  - metadata:
+      name: cache
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: cache
+```
+
+#### PV-backed runner work directory
+
+ARC works by automatically creating runner pods for running [`actions/runner`](https://github.com/actions/runner) and [running `config.sh`](https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners#adding-a-self-hosted-runner-to-a-repository) which you had to ran manually without ARC.
+
+`config.sh` is the script provided by `actions/runner` to pre-configure the runner process before being started. One of the options provided by `config.sh` is `--work`,
+which specifies the working directory where the runner runs your workflow jobs in.
+
+The volume and the partition that hosts the work directory should have several or dozens of GBs free space that might be used by your workflow jobs.
+
+By default, ARC uses `/runner/_work` as work directory, which is powered by Kubernetes's `emptyDir`. [`emptyDir` is usually backed by a directory created within a host's volume](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir), somewhere under `/var/lib/kuberntes/pods`. Therefore
+your host's volume that is backing `/var/lib/kubernetes/pods` must have enough free space to serve all the concurrent runner pods that might be deployed onto your host at the same time.
+
+So, in case you see a job failure seemingly due to "disk full", it's very likely you need to reconfigure your host to have more free space.
+
+In case you can't rely on host's volume, consider using `RunnerSet` and backing the work directory with a ephemeral PV.
+
+Kubernetes 1.23 or greater provides the support for [generic ephemeral volumes](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes), which is designed to support this exact use-case. It's defined in the Pod spec API so it isn't currently available for `RunnerDeployment`. `RunnerSet` is based on Kubernetes' `StatefulSet` which mostly embeds the Pod spec under `spec.template.spec`, so there you go.
+
+```yaml
+kind: RunnerSet
+metadata:
+  name: example
+spec:
+  template:
+    spec:
+      containers:
+      - name: runner
+        volumeMounts:
+        - mountPath: /runner/_work
+          name: work
+      - name: docker
+        volumeMounts:
+        - mountPath: /runner/_work
+          name: work
+      volumes:
+      - name: work
+        ephemeral:
+          volumeClaimTemplate:
+            spec:
+              accessModes: [ "ReadWriteOnce" ]
+              storageClassName: "runner-work-dir"
+              resources:
+                requests:
+                  storage: 10Gi
+```
+
 ### Runner Labels

 To run a workflow job on a self-hosted runner, you can use the following syntax in your workflow:
@@ -1311,12 +1579,11 @@ spec:
        # Disables automatic runner updates
        - name: DISABLE_RUNNER_UPDATE
          value: "true"
-        # Configure runner with --ephemeral instead of --once flag
+        # Configure runner with legacy --once instead of --ephemeral flag
        # WARNING | THIS ENV VAR IS DEPRECATED AND WILL BE REMOVED
-        # IN A FUTURE VERSION OF ARC. IN 0.22.0 ARC SETS --ephemeral VIA 
-        # THE CONTROLLER SETTING THIS ENV VAR ON POD CREATION.
-        # THIS ENV VAR WILL BE REMOVED, SEE ISSUE #1196 FOR DETAILS
-        - name: RUNNER_FEATURE_FLAG_EPHEMERAL
+        # THIS ENV VAR WILL BE REMOVED SOON.
+        # SEE ISSUE #1196 FOR DETAILS
+        - name: RUNNER_FEATURE_FLAG_ONCE
          value: "true"
 ```

--- a/SECURITY.md
+++ b/SECURITY.md
@@ -0,0 +1,22 @@
+# Security Policy
+
+##  Sponsoring the project
+
+This project is maintained by a small team of two and therefore lacks the resource to provide security fixes in a timely manner.
+
+If you have important business(es) that relies on this project, please consider sponsoring the project so that the maintainer(s) can commit to providing such service.
+
+Please refer to https://github.com/sponsors/actions-runner-controller for available tiers.
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.23.0  | :white_check_mark: |
+| < 0.23.0| :x:                |
+
+## Reporting a Vulnerability
+
+To report a security issue, please email ykuoka+arcsecurity(at)gmail.com with a description of the issue, the steps you took to create the issue, affected versions, and, if known, mitigations for the issue.
+
+A maintainer will try to respond within 5 working days. If the issue is confirmed as a vulnerability, a Security Advisory will be opened. This project tries to follow a 90 day disclosure timeline.
--- a/TROUBLESHOOTING.md
+++ b/TROUBLESHOOTING.md
@@ -1,10 +1,28 @@
 # Troubleshooting

-* [Invalid header field value](#invalid-header-field-value)
-* [Runner coming up before network available](#runner-coming-up-before-network-available)
-* [Deployment fails on GKE due to webhooks](#deployment-fails-on-gke-due-to-webhooks)
+* [Tools](#tools)
+* [Installation](#installation)
+  * [Invalid header field value](#invalid-header-field-value)
+  * [Deployment fails on GKE due to webhooks](#deployment-fails-on-gke-due-to-webhooks)
+* [Operations](#operations)
+  * [Stuck runner kind or backing pod](#stuck-runner-kind-or-backing-pod)
+  * [Delay in jobs being allocated to runners](#delay-in-jobs-being-allocated-to-runners)
+  * [Runner coming up before network available](#runner-coming-up-before-network-available)
+  * [Outgoing network action hangs indefinitely](#outgoing-network-action-hangs-indefinitely)
+  * [Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns](#unable-to-scale-to-zero-with-totalnumberofqueuedandinprogressworkflowruns)

-## Invalid header field value
+## Tools
+
+A list of tools which are helpful for troubleshooting
+
+* https://github.com/rewanthtammana/kubectl-fields Kubernetes resources hierarchy parsing tool
+* https://github.com/stern/stern Multi pod and container log tailing for Kubernetes
+
+## Installation
+
+Troubeshooting runbooks that relate to ARC installation problems
+
+### Invalid header field value

 **Problem**

@@ -23,7 +41,103 @@ Your base64'ed PAT token has a new line at the end, it needs to be created witho
 * `echo -n $TOKEN | base64`
 * Create the secret as described in the docs using the shell and documented flags

-## Runner coming up before network available
+
+### Deployment fails on GKE due to webhooks
+
+**Problem**
+
+Due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster:
+
+```
+Internal error occurred: failed calling webhook "mutate.runner.actions.summerwind.dev": 
+Post https://webhook-service.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runner?timeout=10s: 
+context deadline exceeded
+```
+
+**Solution**<br />
+
+To fix this, you may either:
+
+1. Configure the webhook to use another port, such as 443 or 10250, [each of
+   which allow traffic by default](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules).
+
+   ```sh
+   # With helm, you'd set `webhookPort` to the port number of your choice
+   # See https://github.com/actions-runner-controller/actions-runner-controller/pull/1410/files for more information
+   helm upgrade --install --namespace actions-runner-system --create-namespace \
+                --wait actions-runner-controller actions-runner-controller/actions-runner-controller \
+                --set webhookPort=10250
+   ```
+
+2. Set up a firewall rule to allow the master node to connect to the default
+   webhook port. The exact way to do this may vary, but the following script
+   should point you in the right direction:
+
+   ```sh
+   # 1) Retrieve the network tag automatically given to the worker nodes
+   # NOTE: this only works if you have only one cluster in your GCP project. You will have to manually inspect the result of this command to find the tag for the cluster you want to target
+   WORKER_NODES_TAG=$(gcloud compute instances list --format='text(tags.items[0])' --filter='metadata.kubelet-config:*' | grep tags | awk '{print $2}' | sort | uniq)
+
+   # 2) Take note of the VPC network in which you deployed your cluster
+   # NOTE this only works if you have only one network in which you deploy your clusters
+   NETWORK=$(gcloud compute instances list --format='text(networkInterfaces[0].network)' --filter='metadata.kubelet-config:*' | grep networks | awk -F'/' '{print $NF}' | sort | uniq)
+
+   # 3) Get the master source ip block
+   SOURCE=$(gcloud container clusters describe <cluster-name> --region <region> | grep masterIpv4CidrBlock| cut -d ':' -f 2 | tr -d ' ')
+
+   gcloud compute firewall-rules create k8s-cert-manager --source-ranges $SOURCE --target-tags $WORKER_NODES_TAG  --allow TCP:9443 --network $NETWORK
+   ```
+
+## Operations
+
+Troubeshooting runbooks that relate to ARC operational problems
+
+### Stuck runner kind or backing pod
+
+**Problem**
+
+Sometimes either the runner kind (`kubectl get runners`) or it's underlying pod can get stuck in a terminating state for various reasons. You can get the kind unstuck by removing its finaliser using something like this:
+
+**Solution**
+
+Remove the finaliser from the relevent runner kind or pod
+
+```
+# Get all kind runners and remove the finalizer
+$ kubectl get runners --no-headers | awk {'print $1'} | xargs kubectl patch runner --type merge -p '{"metadata":{"finalizers":null}}'
+
+# Get all pods that are stuck terminating and remove the finalizer
+$ kubectl -n get pods | grep Terminating | awk {'print $1'} | xargs kubectl patch pod -p '{"metadata":{"finalizers":null}}'
+```
+
+_Note the code assumes you have already selected the namespace your runners are in and that they 
+are in a namespace not shared with anything else_
+
+### Delay in jobs being allocated to runners
+
+**Problem**
+
+ARC isn't involved in jobs actually getting allocated to a runner. ARC is responsible for orchestrating runners and the runner lifecycle. Why some people see large delays in job allocation is not clear however it has been https://github.com/actions-runner-controller/actions-runner-controller/issues/1387#issuecomment-1122593984 that this is caused from the self-update process somehow.
+
+**Solution**
+
+Disable the self-update process in your runner manifests
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeployment-with-sleep
+spec:
+  template:
+    spec:
+      ...
+      env:
+        - name: DISABLE_RUNNER_UPDATE
+          value: "true"
+```
+
+### Runner coming up before network available

 **Problem**

@@ -61,40 +175,48 @@ metadata:
 spec:
  template:
    spec:
+      ...
      env:
-        # This runner's entrypoint script will have a 5 seconds delay 
-        # as a first action within the entrypoint script
        - name: STARTUP_DELAY_IN_SECONDS
          value: "5"
 ```

-## Deployment fails on GKE due to webhooks
+## Outgoing network action hangs indefinitely

 **Problem**

-Due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster:
+Some random outgoing network actions hangs indefinitely. This could be because your cluster does not give Docker the standard MTU of 1500, you can check this out by running `ip link` in a pod that encounters the problem and reading the outgoing interface's MTU value. If it is smaller than 1500, then try the following.

-```
-Internal error occurred: failed calling webhook "mutate.runner.actions.summerwind.dev": 
-Post https://webhook-service.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runner?timeout=10s: 
-context deadline exceeded
+**Solution**
+
+Add a `dockerMTU` key in your runner's spec with the value you read on the outgoing interface. For instance:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: github-runner
+  namespace: github-system
+spec:
+  replicas: 6
+  template:
+    spec:
+      dockerMTU: 1400
+      repository: $username/$repo
+      env: []
 ```

-**Solution**<br />
+There may be more places you need to tweak for MTU.
+Please consult issues like #651 for more information.

-To fix this, you need to set up a firewall rule to allow the master node to connect to the webhook port.
-The exact way to do this may wary, but the following script should point you in the right direction:
+## Unable to scale to zero with TotalNumberOfQueuedAndInProgressWorkflowRuns

-```
-# 1) Retrieve the network tag automatically given to the worker nodes
-# NOTE: this only works if you have only one cluster in your GCP project. You will have to manually inspect the result of this command to find the tag for the cluster you want to target
-WORKER_NODES_TAG=$(gcloud compute instances list --format='text(tags.items[0])' --filter='metadata.kubelet-config:*' | grep tags | awk '{print $2}' | sort | uniq)
+**Problem**

-# 2) Take note of the VPC network in which you deployed your cluster
-# NOTE this only works if you have only one network in which you deploy your clusters
-NETWORK=$(gcloud compute instances list --format='text(networkInterfaces[0].network)' --filter='metadata.kubelet-config:*' | grep networks | awk -F'/' '{print $NF}' | sort | uniq)
+HRA doesn't scale the RunnerDeployment to zero, even though you did configure HRA correctly, to have a pull-based scaling metric `TotalNumberOfQueuedAndInProgressWorkflowRuns`, and set `minReplicas: 0`.

-# 3) Get the master source ip block
-SOURCE=$(gcloud container clusters describe <cluster-name> --region <region> | grep masterIpv4CidrBlock| cut -d ':' -f 2 | tr -d ' ')
-gcloud compute firewall-rules create k8s-cert-manager --source-ranges $SOURCE --target-tags $WORKER_NODES_TAG  --allow TCP:9443 --network $NETWORK
-```
+**Solution**
+
+You very likely have some dangling workflow jobs stuck in `queued` or `in_progress` as seen in [#1057](https://github.com/actions-runner-controller/actions-runner-controller/issues/1057#issuecomment-1133439061).
+
+Manually call [the "list workflow runs" API](https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository), and [remove the dangling workflow job(s)](https://docs.github.com/en/rest/actions/workflow-runs#delete-a-workflow-run).
--- a/acceptance/deploy.sh
+++ b/acceptance/deploy.sh
@@ -99,7 +99,7 @@ if [ -n "${TEST_ORG}" ]; then

  if [ -n "${TEST_ORG_GROUP}" ]; then
    if [ "${USE_RUNNERSET}" != "false" ]; then
-      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orgroupg-runnerset envsubst | kubectl apply -f -
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerset envsubst | kubectl apply -f -
    else
      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl apply -f -
    fi
--- a/acceptance/testdata/runnerdeploy.envsubst.yaml
+++ b/acceptance/testdata/runnerdeploy.envsubst.yaml
@@ -19,11 +19,6 @@ spec:

      ephemeral: ${TEST_EPHEMERAL}

-      # Whether to pass --ephemeral (true) or --once (false, deprecated)
-      env:
-      - name: RUNNER_FEATURE_FLAG_EPHEMERAL
-        value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
-
      #
      # dockerd within runner container
      #
@@ -57,7 +52,8 @@ spec:
  scaleTargetRef:
    name: ${NAME}
  scaleUpTriggers:
-  - githubEvent: {}
+  - githubEvent:
+      workflowJob: {}
    amount: 1
    duration: "10m"
  minReplicas: ${RUNNER_MIN_REPLICAS}
--- a/acceptance/testdata/runnerset.envsubst.yaml
+++ b/acceptance/testdata/runnerset.envsubst.yaml
@@ -1,3 +1,59 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-runner-work-dir
+  labels:
+    content: ${NAME}-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}
+# In kind environments, the provider writes:
+#   /var/lib/docker/volumes/KIND_NODE_CONTAINER_VOL_ID/_data/local-path-provisioner/PV_NAME
+# It can be hundreds of gigabytes depending on what you cache in the test workflow. Beware to not encounter `no space left on device` errors!
+# If you did encounter no space errorrs try:
+#   docker system prune
+#   docker buildx prune #=> frees up /var/lib/docker/volumes/buildx_buildkit_container-builder0_state
+#   sudo rm -rf /var/lib/docker/volumes/KIND_NODE_CONTAINER_VOL_ID/_data/local-path-provisioner #=> frees up local-path-provisioner's data
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-var-lib-docker
+  labels:
+    content: ${NAME}-var-lib-docker
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-cache
+  labels:
+    content: ${NAME}-cache
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-runner-tool-cache
+  labels:
+    content: ${NAME}-runner-tool-cache
+provisioner: rancher.io/local-path
+reclaimPolicy: Retain
+volumeBindingMode: WaitForFirstConsumer
+---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerSet
 metadata:
@@ -62,8 +118,122 @@ spec:
        env:
        - name: RUNNER_FEATURE_FLAG_EPHEMERAL
          value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
-      #- name: docker
-      #  #image: mumoshu/actions-runner-dind:dev
+        - name: GOMODCACHE
+          value: "/home/runner/.cache/go-mod"
+        # PV-backed runner work dir
+        volumeMounts:
+        - name: work
+          mountPath: /runner/_work
+        # Cache docker image layers, in case dockerdWithinRunnerContainer=true
+        - name: var-lib-docker
+          mountPath: /var/lib/docker
+        # Cache go modules and builds
+        # - name: gocache
+        #   # Run `goenv | grep GOCACHE` to verify the path is correct for your env
+        #   mountPath: /home/runner/.cache/go-build
+        # - name: gomodcache
+        #   # Run `goenv | grep GOMODCACHE` to verify the path is correct for your env
+        #   # mountPath: /home/runner/go/pkg/mod
+        - name: cache
+          # go: could not create module cache: stat /home/runner/.cache/go-mod: permission denied
+          mountPath: "/home/runner/.cache"
+        - name: runner-tool-cache
+          # This corresponds to our runner image's default setting of RUNNER_TOOL_CACHE=/opt/hostedtoolcache.
+          #
+          # In case you customize the envvar in both runner and docker containers of the runner pod spec,
+          # You'd need to change this mountPath accordingly.
+          #
+          # The tool cache directory is defined in actions/toolkit's tool-cache module:
+          # https://github.com/actions/toolkit/blob/2f164000dcd42fb08287824a3bc3030dbed33687/packages/tool-cache/src/tool-cache.ts#L621-L638
+          #
+          # Many setup-* actions like setup-go utilizes the tool-cache module to download and cache installed binaries:
+          # https://github.com/actions/setup-go/blob/56a61c9834b4a4950dbbf4740af0b8a98c73b768/src/installer.ts#L144
+          mountPath: "/opt/hostedtoolcache"
+      # Valid only when dockerdWithinRunnerContainer=false
+      - name: docker
+        # PV-backed runner work dir
+        volumeMounts:
+        - name: work
+          mountPath: /runner/_work
+        # Cache docker image layers, in case dockerdWithinRunnerContainer=false
+        - name: var-lib-docker
+          mountPath: /var/lib/docker
+        # image: mumoshu/actions-runner-dind:dev
+
+        # For buildx cache
+        - name: cache
+          mountPath: "/home/runner/.cache"
+      volumes:
+      - name: work
+        ephemeral:
+          volumeClaimTemplate:
+            spec:
+              accessModes:
+              - ReadWriteOnce
+              storageClassName: "${NAME}-runner-work-dir"
+              resources:
+                requests:
+                  storage: 10Gi
+  volumeClaimTemplates:
+  - metadata:
+      name: vol1
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}
+      ## Dunno which provider supports auto-provisioning with selector.
+      ## At least the rancher local path provider stopped with:
+      ##  waiting for a volume to be created, either by external provisioner "rancher.io/local-path" or manually created by system administrator
+      # selector:
+      #    matchLabels:
+      #      runnerset-volume-id: ${NAME}-vol1
+  - metadata:
+      name: vol2
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}
+      # selector:
+      #    matchLabels:
+      #      runnerset-volume-id: ${NAME}-vol2
+  - metadata:
+      name: var-lib-docker
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}-var-lib-docker
+  - metadata:
+      name: cache
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}-cache
+  - metadata:
+      name: runner-tool-cache
+      # It turns out labels doesn't distinguish PVs across PVCs and the
+      # end result is PVs are reused by wrong PVCs.
+      # The correct way seems to be to differentiate storage class per pvc template.
+      # labels:
+      #   id: runner-tool-cache
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Mi
+      storageClassName: ${NAME}-runner-tool-cache
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: HorizontalRunnerAutoscaler
@@ -74,7 +244,8 @@ spec:
    kind: RunnerSet
    name: ${NAME}
  scaleUpTriggers:
-  - githubEvent: {}
+  - githubEvent:
+      workflowJob: {}
    amount: 1
    duration: "10m"
  minReplicas: ${RUNNER_MIN_REPLICAS}
--- a/charts/actions-runner-controller/Chart.yaml
+++ b/charts/actions-runner-controller/Chart.yaml
@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.18.0
+version: 0.19.1

 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.23.0
+appVersion: 0.24.1

 home: https://github.com/actions-runner-controller/actions-runner-controller

--- a/charts/actions-runner-controller/README.md
+++ b/charts/actions-runner-controller/README.md
@@ -12,6 +12,7 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 |----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------|
 | `labels`                                                 | Set labels to apply to all resources in the chart                                                                          |                                                                      |
 | `replicaCount`                                           | Set the number of controller pods                                                                                          | 1                                                                    |
+| `webhookPort`                                            | Set the containerPort for the webhook Pod                                                                                  | 9443                                                                 |
 | `syncPeriod`                                             | Set the period in which the controler reconciles the desired runners count                                                 | 10m                                                                  |
 | `enableLeaderElection`                                   | Enable election configuration                                                                                              | true                                                                 |
 | `leaderElectionId`                                       | Set the election ID for the controller group                                                                               |                                                                      |
--- a/charts/actions-runner-controller/templates/deployment.yaml
+++ b/charts/actions-runner-controller/templates/deployment.yaml
@@ -44,6 +44,7 @@ spec:
        {{- if .Values.leaderElectionId }}
        - "--leader-election-id={{ .Values.leaderElectionId }}"
        {{- end }}
+        - "--port={{ .Values.webhookPort }}"
        - "--sync-period={{ .Values.syncPeriod }}"
        - "--default-scale-down-delay={{ .Values.defaultScaleDownDelay }}"
        - "--docker-image={{ .Values.image.dindSidecarRepositoryAndTag }}"
@@ -125,7 +126,7 @@ spec:
        name: manager
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        ports:
-        - containerPort: 9443
+        - containerPort: {{ .Values.webhookPort }}
          name: webhook-server
          protocol: TCP
        {{- if not .Values.metrics.proxy.enabled }}
--- a/charts/actions-runner-controller/templates/githubwebhook.ingress.yaml
+++ b/charts/actions-runner-controller/templates/githubwebhook.ingress.yaml
@@ -11,6 +11,7 @@ apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
  name: {{ $fullName }}
+  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "actions-runner-controller.labels" . | nindent 4 }}
  {{- with .Values.githubWebhookServer.ingress.annotations }}
@@ -36,6 +37,9 @@ spec:
    - host: {{ .host | quote }}
      http:
        paths:
+          {{- if .extraPaths }}
+          {{- toYaml .extraPaths | nindent 10 }}
+          {{- end }}
          {{- range .paths }}
          - path: {{ .path }}
            {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
--- a/charts/actions-runner-controller/templates/manager_role.yaml
+++ b/charts/actions-runner-controller/templates/manager_role.yaml
@@ -195,6 +195,28 @@ rules:
  verbs:
  - create
  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
  - coordination.k8s.io
  resources:
--- a/charts/actions-runner-controller/templates/webhook_service.yaml
+++ b/charts/actions-runner-controller/templates/webhook_service.yaml
@@ -13,7 +13,7 @@ spec:
  type: {{ .Values.service.type }}
  ports:
    - port: 443
-      targetPort: 9443
+      targetPort: {{ .Values.webhookPort }}
      protocol: TCP
      name: https
  selector:
--- a/charts/actions-runner-controller/values.yaml
+++ b/charts/actions-runner-controller/values.yaml
@@ -6,6 +6,7 @@ labels: {}

 replicaCount: 1

+webhookPort: 9443
 syncPeriod: 1m
 defaultScaleDownDelay: 10m

@@ -108,7 +109,7 @@ metrics:
    enabled: true
    image:
      repository: quay.io/brancz/kube-rbac-proxy
-      tag: v0.11.0
+      tag: v0.12.0

 resources:
  {}
@@ -223,6 +224,20 @@ githubWebhookServer:
        paths: []
        #  - path: /*
        #    pathType: ImplementationSpecific
+        # Extra paths that are not automatically connected to the server. This is useful when working with annotation based services.
+        extraPaths: []
+        # - path: /*
+        #   backend:
+        #     serviceName: ssl-redirect
+        #     servicePort: use-annotation
+        ## for Kubernetes >=1.19 (when "networking.k8s.io/v1" is used)
+        # - path: /*
+        #   pathType: Prefix
+        #   backend:
+        #     service:
+        #       name: ssl-redirect
+        #       port:
+        #         name: use-annotation
    tls: []
    #  - secretName: chart-example-tls
    #    hosts:
--- a/config/rbac/role.yaml
+++ b/config/rbac/role.yaml
@@ -202,6 +202,29 @@ rules:
  verbs:
  - create
  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
  - ""
  resources:
--- a/controllers/constants.go
+++ b/controllers/constants.go
@@ -47,8 +47,6 @@ const (
 	// A pod that is timed out can be terminated if needed.
 	registrationTimeout = 10 * time.Minute

-	defaultRegistrationCheckInterval = time.Minute
-
 	// DefaultRunnerPodRecreationDelayAfterWebhookScale is the delay until syncing the runners with the desired replicas
 	// after a webhook-based scale up.
 	// This is used to prevent ARC from recreating completed runner pods that are deleted soon without being used at all.
--- a/controllers/horizontal_runner_autoscaler_webhook_on_push.go
+++ b/controllers/horizontal_runner_autoscaler_webhook_on_push.go
@@ -15,10 +15,6 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event

 		push := g.Push

-		if push == nil {
-			return false
-		}
-
-		return true
+		return push != nil
 	}
 }
--- a/controllers/new_runner_pod_test.go
+++ b/controllers/new_runner_pod_test.go
@@ -7,12 +7,43 @@ import (
 	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 )

+func newWorkGenericEphemeralVolume(t *testing.T, storageReq string) corev1.Volume {
+	GBs, err := resource.ParseQuantity(storageReq)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	return corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes: []corev1.PersistentVolumeAccessMode{
+							corev1.ReadWriteOnce,
+						},
+						StorageClassName: strPtr("runner-work-dir"),
+						Resources: corev1.ResourceRequirements{
+							Requests: corev1.ResourceList{
+								corev1.ResourceStorage: GBs,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func TestNewRunnerPod(t *testing.T) {
+	workGenericEphemeralVolume := newWorkGenericEphemeralVolume(t, "10Gi")
+
 	type testcase struct {
 		description string

@@ -106,10 +137,6 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "DOCKER_CERT_PATH",
 							Value: "/certs/client",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -159,7 +186,7 @@ func TestNewRunnerPod(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}

@@ -228,10 +255,6 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -245,7 +268,7 @@ func TestNewRunnerPod(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}

@@ -310,10 +333,6 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -327,7 +346,7 @@ func TestNewRunnerPod(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}

@@ -400,8 +419,87 @@ func TestNewRunnerPod(t *testing.T) {
 				DockerEnabled: boolPtr(false),
 			},
 			want: newTestPod(dockerDisabled, func(p *corev1.Pod) {
-				// TODO
-				// p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+				p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (with explicit volumeMount)",
+			template: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "runner",
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "work",
+									MountPath: "/runner/_work",
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						workGenericEphemeralVolume,
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					workGenericEphemeralVolume,
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+				}
+				p.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+					{
+						Name:      "work",
+						MountPath: "/runner/_work",
+					},
+					{
+						Name:      "runner",
+						MountPath: "/runner",
+					},
+					{
+						Name:      "certs-client",
+						MountPath: "/certs/client",
+						ReadOnly:  true,
+					},
+				}
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (without explicit volumeMount)",
+			template: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Volumes: []corev1.Volume{
+						workGenericEphemeralVolume,
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					workGenericEphemeralVolume,
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+				}
 			}),
 		},
 	}
@@ -417,14 +515,20 @@ func TestNewRunnerPod(t *testing.T) {
 	for i := range testcases {
 		tc := testcases[i]
 		t.Run(tc.description, func(t *testing.T) {
-			got, err := newRunnerPod("runner", tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, false)
+			got, err := newRunnerPod("runner", tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
 			require.NoError(t, err)
 			require.Equal(t, tc.want, got)
 		})
 	}
 }

+func strPtr(s string) *string {
+	return &s
+}
+
 func TestNewRunnerPodFromRunnerController(t *testing.T) {
+	workGenericEphemeralVolume := newWorkGenericEphemeralVolume(t, "10Gi")
+
 	type testcase struct {
 		description string

@@ -532,10 +636,6 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "DOCKER_CERT_PATH",
 							Value: "/certs/client",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -593,7 +693,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}

@@ -669,10 +769,6 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -694,7 +790,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}

@@ -770,10 +866,6 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
-						{
-							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
-							Value: "true",
-						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -795,7 +887,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			RestartPolicy: corev1.RestartPolicyNever,
 		},
 	}

@@ -904,7 +996,97 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			},

 			want: newTestPod(dockerDisabled, func(p *corev1.Pod) {
-				// p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+				p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (with explicit volumeMount)",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerPodSpec: arcv1alpha1.RunnerPodSpec{
+						Containers: []corev1.Container{
+							{
+								Name: "runner",
+								VolumeMounts: []corev1.VolumeMount{
+									{
+										Name:      "work",
+										MountPath: "/runner/_work",
+									},
+								},
+							},
+						},
+						Volumes: []corev1.Volume{
+							workGenericEphemeralVolume,
+						},
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					workGenericEphemeralVolume,
+				}
+				p.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+					{
+						Name:      "work",
+						MountPath: "/runner/_work",
+					},
+					{
+						Name:      "runner",
+						MountPath: "/runner",
+					},
+					{
+						Name:      "certs-client",
+						MountPath: "/certs/client",
+						ReadOnly:  true,
+					},
+				}
+			}),
+		},
+		{
+			description: "Mount generic ephemeral volume onto work (without explicit volumeMount)",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerPodSpec: arcv1alpha1.RunnerPodSpec{
+						Volumes: []corev1.Volume{
+							workGenericEphemeralVolume,
+						},
+					},
+				},
+			},
+			want: newTestPod(base, func(p *corev1.Pod) {
+				p.Spec.Volumes = []corev1.Volume{
+					{
+						Name: "runner",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					{
+						Name: "certs-client",
+						VolumeSource: corev1.VolumeSource{
+							EmptyDir: &corev1.EmptyDirVolumeSource{},
+						},
+					},
+					workGenericEphemeralVolume,
+				}
 			}),
 		},
 	}
--- a/controllers/persistent_volume_claim_controller.go
+++ b/controllers/persistent_volume_claim_controller.go
@@ -0,0 +1,74 @@
+/*
+Copyright 2022 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// RunnerPersistentVolumeClaimReconciler reconciles a PersistentVolume object
+type RunnerPersistentVolumeClaimReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
+}
+
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerPersistentVolumeClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("pvc", req.NamespacedName)
+
+	var pvc corev1.PersistentVolumeClaim
+	if err := r.Get(ctx, req.NamespacedName, &pvc); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	res, err := syncPVC(ctx, r.Client, log, req.Namespace, &pvc)
+
+	if res == nil {
+		res = &ctrl.Result{}
+	}
+
+	return *res, err
+}
+
+func (r *RunnerPersistentVolumeClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerpersistentvolumeclaim-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.PersistentVolumeClaim{}).
+		Named(name).
+		Complete(r)
+}
--- a/controllers/persistent_volume_controller.go
+++ b/controllers/persistent_volume_controller.go
@@ -0,0 +1,72 @@
+/*
+Copyright 2022 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// RunnerPersistentVolumeReconciler reconciles a PersistentVolume object
+type RunnerPersistentVolumeReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
+}
+
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerPersistentVolumeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("pv", req.NamespacedName)
+
+	var pv corev1.PersistentVolume
+	if err := r.Get(ctx, req.NamespacedName, &pv); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	res, err := syncPV(ctx, r.Client, log, req.Namespace, &pv)
+	if res == nil {
+		res = &ctrl.Result{}
+	}
+
+	return *res, err
+}
+
+func (r *RunnerPersistentVolumeReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerpersistentvolume-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.PersistentVolume{}).
+		Named(name).
+		Complete(r)
+}
--- a/controllers/pod_runner_token_injector.go
+++ b/controllers/pod_runner_token_injector.go
@@ -78,9 +78,7 @@ func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Reque

 	updated.Annotations[AnnotationKeyTokenExpirationDate] = ts

-	if pod.Spec.RestartPolicy != corev1.RestartPolicyOnFailure {
-		updated.Spec.RestartPolicy = corev1.RestartPolicyOnFailure
-	}
+	forceRunnerPodRestartPolicyNever(updated)

 	buf, err := json.Marshal(updated)
 	if err != nil {
--- a/controllers/runner_controller.go
+++ b/controllers/runner_controller.go
@@ -47,15 +47,11 @@ const (

 	retryDelayOnGitHubAPIRateLimitError = 30 * time.Second

-	// This is an annotation internal to actions-runner-controller and can change in backward-incompatible ways
-	annotationKeyRegistrationOnly = "actions-runner-controller/registration-only"
-
-	EnvVarOrg                        = "RUNNER_ORG"
-	EnvVarRepo                       = "RUNNER_REPO"
-	EnvVarEnterprise                 = "RUNNER_ENTERPRISE"
-	EnvVarEphemeral                  = "RUNNER_EPHEMERAL"
-	EnvVarRunnerFeatureFlagEphemeral = "RUNNER_FEATURE_FLAG_EPHEMERAL"
-	EnvVarTrue                       = "true"
+	EnvVarOrg        = "RUNNER_ORG"
+	EnvVarRepo       = "RUNNER_REPO"
+	EnvVarEnterprise = "RUNNER_ENTERPRISE"
+	EnvVarEphemeral  = "RUNNER_EPHEMERAL"
+	EnvVarTrue       = "true"
 )

 // RunnerReconciler reconciles a Runner object
@@ -207,6 +203,24 @@ func runnerPodOrContainerIsStopped(pod *corev1.Pod) bool {
 	return stopped
 }

+func ephemeralRunnerContainerStatus(pod *corev1.Pod) *corev1.ContainerStatus {
+	if getRunnerEnv(pod, "RUNNER_EPHEMERAL") != "true" {
+		return nil
+	}
+
+	for _, status := range pod.Status.ContainerStatuses {
+		if status.Name != containerName {
+			continue
+		}
+
+		status := status
+
+		return &status
+	}
+
+	return nil
+}
+
 func (r *RunnerReconciler) processRunnerDeletion(runner v1alpha1.Runner, ctx context.Context, log logr.Logger, pod *corev1.Pod) (reconcile.Result, error) {
 	finalizers, removed := removeFinalizer(runner.ObjectMeta.Finalizers, finalizerName)

@@ -349,31 +363,56 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {

 	if len(runner.Spec.Containers) == 0 {
 		template.Spec.Containers = append(template.Spec.Containers, corev1.Container{
-			Name:            "runner",
-			ImagePullPolicy: runner.Spec.ImagePullPolicy,
-			EnvFrom:         runner.Spec.EnvFrom,
-			Env:             runner.Spec.Env,
-			Resources:       runner.Spec.Resources,
+			Name: "runner",
 		})

 		if (runner.Spec.DockerEnabled == nil || *runner.Spec.DockerEnabled) && (runner.Spec.DockerdWithinRunnerContainer == nil || !*runner.Spec.DockerdWithinRunnerContainer) {
 			template.Spec.Containers = append(template.Spec.Containers, corev1.Container{
-				Name:         "docker",
-				VolumeMounts: runner.Spec.DockerVolumeMounts,
-				Resources:    runner.Spec.DockerdContainerResources,
-				Env:          runner.Spec.DockerEnv,
+				Name: "docker",
 			})
 		}
 	} else {
 		template.Spec.Containers = runner.Spec.Containers
 	}

+	for i, c := range template.Spec.Containers {
+		switch c.Name {
+		case "runner":
+			if c.ImagePullPolicy == "" {
+				template.Spec.Containers[i].ImagePullPolicy = runner.Spec.ImagePullPolicy
+			}
+			if len(c.EnvFrom) == 0 {
+				template.Spec.Containers[i].EnvFrom = runner.Spec.EnvFrom
+			}
+			if len(c.Env) == 0 {
+				template.Spec.Containers[i].Env = runner.Spec.Env
+			}
+			if len(c.Resources.Requests) == 0 {
+				template.Spec.Containers[i].Resources.Requests = runner.Spec.Resources.Requests
+			}
+			if len(c.Resources.Limits) == 0 {
+				template.Spec.Containers[i].Resources.Limits = runner.Spec.Resources.Limits
+			}
+		case "docker":
+			if len(c.VolumeMounts) == 0 {
+				template.Spec.Containers[i].VolumeMounts = runner.Spec.DockerVolumeMounts
+			}
+			if len(c.Resources.Limits) == 0 {
+				template.Spec.Containers[i].Resources.Limits = runner.Spec.DockerdContainerResources.Limits
+			}
+			if len(c.Resources.Requests) == 0 {
+				template.Spec.Containers[i].Resources.Requests = runner.Spec.DockerdContainerResources.Requests
+			}
+			if len(c.Env) == 0 {
+				template.Spec.Containers[i].Env = runner.Spec.DockerEnv
+			}
+		}
+	}
+
 	template.Spec.SecurityContext = runner.Spec.SecurityContext
 	template.Spec.EnableServiceLinks = runner.Spec.EnableServiceLinks

-	registrationOnly := metav1.HasAnnotation(runner.ObjectMeta, annotationKeyRegistrationOnly)
-
-	pod, err := newRunnerPod(runner.Name, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL, registrationOnly)
+	pod, err := newRunnerPod(runner.Name, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL)
 	if err != nil {
 		return pod, err
 	}
@@ -487,7 +526,7 @@ func mutatePod(pod *corev1.Pod, token string) *corev1.Pod {
 	return updated
 }

-func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, registrationOnly bool) (corev1.Pod, error) {
+func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
 	var (
 		privileged                bool = true
 		dockerdInRunner           bool = runnerSpec.DockerdWithinRunnerContainer != nil && *runnerSpec.DockerdWithinRunnerContainer
@@ -559,14 +598,6 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 		},
 	}

-	if registrationOnly {
-		env = append(env, corev1.EnvVar{
-			Name:  "RUNNER_REGISTRATION_ONLY",
-			Value: "true",
-		},
-		)
-	}
-
 	var seLinuxOptions *corev1.SELinuxOptions
 	if template.Spec.SecurityContext != nil {
 		seLinuxOptions = template.Spec.SecurityContext.SELinuxOptions
@@ -624,14 +655,15 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 	if runnerContainer.SecurityContext == nil {
 		runnerContainer.SecurityContext = &corev1.SecurityContext{}
 	}
-	// Runner need to run privileged if it contains DinD
-	runnerContainer.SecurityContext.Privileged = &dockerdInRunnerPrivileged
+
+	if runnerContainer.SecurityContext.Privileged == nil {
+		// Runner need to run privileged if it contains DinD
+		runnerContainer.SecurityContext.Privileged = &dockerdInRunnerPrivileged
+	}

 	pod := template.DeepCopy()

-	if pod.Spec.RestartPolicy == "" {
-		pod.Spec.RestartPolicy = "OnFailure"
-	}
+	forceRunnerPodRestartPolicyNever(pod)

 	if mtu := runnerSpec.DockerMTU; mtu != nil && dockerdInRunner {
 		runnerContainer.Env = append(runnerContainer.Env, []corev1.EnvVar{
@@ -709,13 +741,18 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 			)
 		}

-		pod.Spec.Volumes = append(pod.Spec.Volumes,
-			corev1.Volume{
-				Name: "work",
-				VolumeSource: corev1.VolumeSource{
-					EmptyDir: &corev1.EmptyDirVolumeSource{},
+		if ok, _ := workVolumePresent(pod.Spec.Volumes); !ok {
+			pod.Spec.Volumes = append(pod.Spec.Volumes,
+				corev1.Volume{
+					Name: "work",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
 				},
-			},
+			)
+		}
+
+		pod.Spec.Volumes = append(pod.Spec.Volumes,
 			corev1.Volume{
 				Name: "certs-client",
 				VolumeSource: corev1.VolumeSource{
@@ -724,11 +761,16 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 			},
 		)

+		if ok, _ := workVolumeMountPresent(runnerContainer.VolumeMounts); !ok {
+			runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts,
+				corev1.VolumeMount{
+					Name:      "work",
+					MountPath: workDir,
+				},
+			)
+		}
+
 		runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts,
-			corev1.VolumeMount{
-				Name:      "work",
-				MountPath: workDir,
-			},
 			corev1.VolumeMount{
 				Name:      "certs-client",
 				MountPath: "/certs/client",
@@ -830,12 +872,6 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 		}
 	}

-	// TODO Remove this once we remove RUNNER_FEATURE_FLAG_EPHEMERAL from runner's entrypoint.sh
-	// and make --ephemeral the default option.
-	if getRunnerEnv(pod, EnvVarRunnerFeatureFlagEphemeral) == "" {
-		setRunnerEnv(pod, EnvVarRunnerFeatureFlagEphemeral, EnvVarTrue)
-	}
-
 	return *pod, nil
 }

--- a/controllers/runner_graceful_stop.go
+++ b/controllers/runner_graceful_stop.go
@@ -113,10 +113,28 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 		// Happens e.g. when dind is in runner and run completes
 		log.Info("Runner pod has been stopped with a successful status.")
 	} else if pod != nil && pod.Annotations[AnnotationKeyRunnerCompletionWaitStartTimestamp] != "" {
-		log.Info("Runner pod is annotated to wait for completion")
+		ct := ephemeralRunnerContainerStatus(pod)
+		if ct == nil {
+			log.Info("Runner pod is annotated to wait for completion, and the runner container is not ephemeral")

-		return &ctrl.Result{RequeueAfter: retryDelay}, nil
-	} else if ok, err := unregisterRunner(ctx, ghClient, enterprise, organization, repository, runner, *runnerID); err != nil {
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		lts := ct.LastTerminationState.Terminated
+		if lts == nil {
+			log.Info("Runner pod is annotated to wait for completion, and the runner container is not restarting")
+
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		// Prevent runner pod from stucking in Terminating.
+		// See https://github.com/actions-runner-controller/actions-runner-controller/issues/1369
+		log.Info("Deleting runner pod anyway because it has stopped prematurely. This may leave a dangling runner resource in GitHub Actions",
+			"lastState.exitCode", lts.ExitCode,
+			"lastState.message", lts.Message,
+			"pod.phase", pod.Status.Phase,
+		)
+	} else if ok, err := unregisterRunner(ctx, ghClient, enterprise, organization, repository, *runnerID); err != nil {
 		if errors.Is(err, &gogithub.RateLimitError{}) {
 			// We log the underlying error when we failed calling GitHub API to list or unregisters,
 			// or the runner is still busy.
@@ -175,7 +193,7 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 			// So we can just wait for the completion without actively retrying unregistration.
 			ephemeral := getRunnerEnv(pod, EnvVarEphemeral)
 			if ephemeral == "true" {
-				pod, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerCompletionWaitStartTimestamp, time.Now().Format(time.RFC3339))
+				_, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerCompletionWaitStartTimestamp, time.Now().Format(time.RFC3339))
 				if err != nil {
 					return &ctrl.Result{}, err
 				}
@@ -352,7 +370,7 @@ func setRunnerEnv(pod *corev1.Pod, key, value string) {
 // There isn't a single right grace period that works for everyone.
 // The longer the grace period is, the earlier a cluster resource shortage can occur due to throttoled runner pod deletions,
 // while the shorter the grace period is, the more likely you may encounter the race issue.
-func unregisterRunner(ctx context.Context, client *github.Client, enterprise, org, repo, name string, id int64) (bool, error) {
+func unregisterRunner(ctx context.Context, client *github.Client, enterprise, org, repo string, id int64) (bool, error) {
 	// For the record, historically ARC did not try to call RemoveRunner on a busy runner, but it's no longer true.
 	// The reason ARC did so was to let a runner running a job to not stop prematurely.
 	//
--- a/controllers/runner_pod.go
+++ b/controllers/runner_pod.go
@@ -0,0 +1,22 @@
+package controllers
+
+import corev1 "k8s.io/api/core/v1"
+
+// Force the runner pod managed by either RunnerDeployment and RunnerSet to have restartPolicy=Never.
+// See https://github.com/actions-runner-controller/actions-runner-controller/issues/1369 for more context.
+//
+// This is to prevent runner pods from stucking in Terminating when a K8s node disappeared along with the runnr pod and the runner container within it.
+//
+// Previously, we used restartPolicy of OnFailure, it turned wrong later, and therefore we now set Never.
+//
+// When the restartPolicy is OnFailure and the node disappeared, runner pods on the node seem to stuck in state.terminated==nil, state.waiting!=nil, and state.lastTerminationState!=nil,
+// and will ever become Running.
+// It's probably due to that the node onto which the pods have been scheduled will ever come back, hence the container restart attempt swill ever succeed,
+// the pods stuck waiting for successful restarts forever.
+//
+// By forcing runner pods to never restart, we hope there will be no chances of pods being stuck waiting.
+func forceRunnerPodRestartPolicyNever(pod *corev1.Pod) {
+	if pod.Spec.RestartPolicy != corev1.RestartPolicyNever {
+		pod.Spec.RestartPolicy = corev1.RestartPolicyNever
+	}
+}
--- a/controllers/runnerdeployment_controller.go
+++ b/controllers/runnerdeployment_controller.go
@@ -421,9 +421,7 @@ func getSelector(rd *v1alpha1.RunnerDeployment) *metav1.LabelSelector {
 func newRunnerReplicaSet(rd *v1alpha1.RunnerDeployment, commonRunnerLabels []string, scheme *runtime.Scheme) (*v1alpha1.RunnerReplicaSet, error) {
 	newRSTemplate := *rd.Spec.Template.DeepCopy()

-	for _, l := range commonRunnerLabels {
-		newRSTemplate.Spec.Labels = append(newRSTemplate.Spec.Labels, l)
-	}
+	newRSTemplate.Spec.Labels = append(newRSTemplate.Spec.Labels, commonRunnerLabels...)

 	templateHash := ComputeHash(&newRSTemplate)

--- a/controllers/runnerreplicaset_controller.go
+++ b/controllers/runnerreplicaset_controller.go
@@ -205,7 +205,3 @@ func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Named(name).
 		Complete(r)
 }
-
-func registrationOnlyRunnerNameFor(rsName string) string {
-	return rsName + "-registration-only"
-}
--- a/controllers/runnerset_controller.go
+++ b/controllers/runnerset_controller.go
@@ -58,6 +58,7 @@ type RunnerSetReconciler struct {
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update

@@ -129,6 +130,12 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		owners = append(owners, &ss)
 	}

+	if res, err := syncVolumes(ctx, r.Client, log, req.Namespace, runnerSet, statefulsets); err != nil {
+		return ctrl.Result{}, err
+	} else if res != nil {
+		return *res, nil
+	}
+
 	res, err := syncRunnerPodsOwners(ctx, r.Client, log, effectiveTime, newDesiredReplicas, func() client.Object { return create.DeepCopy() }, ephemeral, owners)
 	if err != nil || res == nil {
 		return ctrl.Result{}, err
@@ -181,16 +188,14 @@ var LabelValuePodMutation = "true"
 func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
 	runnerSetWithOverrides := *runnerSet.Spec.DeepCopy()

-	for _, l := range r.CommonRunnerLabels {
-		runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, l)
-	}
+	runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, r.CommonRunnerLabels...)

 	template := corev1.Pod{
 		ObjectMeta: runnerSetWithOverrides.StatefulSetSpec.Template.ObjectMeta,
 		Spec:       runnerSetWithOverrides.StatefulSetSpec.Template.Spec,
 	}

-	pod, err := newRunnerPod(runnerSet.Name, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL, false)
+	pod, err := newRunnerPod(runnerSet.Name, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL)
 	if err != nil {
 		return nil, err
 	}
--- a/controllers/schedule_test.go
+++ b/controllers/schedule_test.go
@@ -605,3 +605,13 @@ func parseAndMatchRecurringPeriod(now time.Time, start, end, frequency, until st

 	return MatchSchedule(now, startTime, endTime, RecurrenceRule{Frequency: frequency, UntilTime: untilTime})
 }
+
+func FuzzMatchSchedule(f *testing.F) {
+	start := time.Now()
+	end := time.Now()
+	now := time.Now()
+	f.Fuzz(func(t *testing.T, freq string) {
+		// Verify that it never panics
+		_, _, _ = MatchSchedule(now, start, end, RecurrenceRule{Frequency: freq})
+	})
+}
--- a/controllers/sync_volumes.go
+++ b/controllers/sync_volumes.go
@@ -0,0 +1,181 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	labelKeyCleanup               = "pending-cleanup"
+	labelKeyRunnerStatefulSetName = "runner-statefulset-name"
+)
+
+func syncVolumes(ctx context.Context, c client.Client, log logr.Logger, ns string, runnerSet *v1alpha1.RunnerSet, statefulsets []appsv1.StatefulSet) (*ctrl.Result, error) {
+	log = log.WithValues("ns", ns)
+
+	for _, t := range runnerSet.Spec.StatefulSetSpec.VolumeClaimTemplates {
+		for _, sts := range statefulsets {
+			pvcName := fmt.Sprintf("%s-%s-0", t.Name, sts.Name)
+
+			var pvc corev1.PersistentVolumeClaim
+			if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+				if !kerrors.IsNotFound(err) {
+					return nil, err
+				}
+				continue
+			}
+
+			// TODO move this to statefulset reconciler so that we spam this less,
+			// by starting the loop only after the statefulset got deletionTimestamp set.
+			// Perhaps you can just wrap this in a finalizer here.
+			if pvc.Labels[labelKeyRunnerStatefulSetName] == "" {
+				updated := pvc.DeepCopy()
+				updated.Labels[labelKeyRunnerStatefulSetName] = sts.Name
+				if err := c.Update(ctx, updated); err != nil {
+					return nil, err
+				}
+				log.V(1).Info("Added runner-statefulset-name label to PVC", "sts", sts.Name, "pvc", pvcName)
+			}
+		}
+	}
+
+	// PVs are not namespaced hence we don't need client.InNamespace(ns).
+	// If we added that, c.List will silently return zero items.
+	//
+	// This `List` needs to be done in a dedicated reconciler that is registered to the manager via the `For` func.
+	// Otherwise the List func might return outdated contents(I saw status.phase being Bound even after K8s updated it to Released, and it lasted minutes).
+	//
+	// cleanupLabels := map[string]string{
+	// 	labelKeyCleanup: runnerSet.Name,
+	// }
+	// pvList := &corev1.PersistentVolumeList{}
+	// if err := c.List(ctx, pvList, client.MatchingLabels(cleanupLabels)); err != nil {
+	// 	log.Info("retrying pv listing", "ns", ns, "err", err)
+	// 	return nil, err
+	// }
+
+	return nil, nil
+}
+
+func syncPVC(ctx context.Context, c client.Client, log logr.Logger, ns string, pvc *corev1.PersistentVolumeClaim) (*ctrl.Result, error) {
+	stsName := pvc.Labels[labelKeyRunnerStatefulSetName]
+	if stsName == "" {
+		return nil, nil
+	}
+
+	log.V(2).Info("Reconciling runner PVC")
+
+	var sts appsv1.StatefulSet
+	if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: stsName}, &sts); err != nil {
+		if !kerrors.IsNotFound(err) {
+			return nil, err
+		}
+	} else {
+		// We assume that the statefulset is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(1).Info("Retrying sync until statefulset gets removed", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	log = log.WithValues("sts", stsName)
+
+	pvName := pvc.Spec.VolumeName
+
+	if pvName != "" {
+		// If we deleted PVC before unsetting pv.spec.claimRef,
+		// K8s seems to revive the claimRef :thinking:
+		// So we need to mark PV for claimRef unset first, and delete PVC, and finally unset claimRef on PV.
+
+		var pv corev1.PersistentVolume
+		if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvName}, &pv); err != nil {
+			if !kerrors.IsNotFound(err) {
+				return nil, err
+			}
+			return nil, nil
+		}
+
+		pvCopy := pv.DeepCopy()
+		if pvCopy.Labels == nil {
+			pvCopy.Labels = map[string]string{}
+		}
+		pvCopy.Labels[labelKeyCleanup] = stsName
+
+		log.V(2).Info("Scheduling to unset PV's claimRef", "pv", pv.Name)
+
+		// Apparently K8s doesn't reconcile PV immediately after PVC deletion.
+		// So we start a relatively busy loop of PV reconcilation slightly before the PVC deletion,
+		// so that PV can be unbound as soon as possible after the PVC got deleted.
+		if err := c.Update(ctx, pvCopy); err != nil {
+			return nil, err
+		}
+
+		log.Info("Updated PV to unset claimRef")
+
+		// At this point, the PV is still Bound
+
+		log.V(2).Info("Deleting unused PVC")
+
+		if err := c.Delete(ctx, pvc); err != nil {
+			return nil, err
+		}
+
+		log.Info("Deleted unused PVC")
+
+		// At this point, the PV is still "Bound", but we are ready to unset pv.spec.claimRef in pv controller.
+		// Once the pv controller unsets claimRef, the PV becomes "Released", hence available for reuse by another eligible PVC.
+	}
+
+	return nil, nil
+}
+
+func syncPV(ctx context.Context, c client.Client, log logr.Logger, ns string, pv *corev1.PersistentVolume) (*ctrl.Result, error) {
+	if pv.Spec.ClaimRef == nil {
+		return nil, nil
+	}
+
+	log.V(2).Info("Reconciling PV")
+
+	if pv.Labels[labelKeyCleanup] == "" {
+		// We assume that the pvc is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(2).Info("Retrying sync to see if this PV needs to be managed by ARC", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	log.V(2).Info("checking pv phase", "phase", pv.Status.Phase)
+
+	if pv.Status.Phase != corev1.VolumeReleased {
+		// We assume that the pvc is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(1).Info("Retrying sync until pvc gets released", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	// At this point, the PV is still Released
+
+	pvCopy := pv.DeepCopy()
+	delete(pvCopy.Labels, labelKeyCleanup)
+	pvCopy.Spec.ClaimRef = nil
+	log.V(2).Info("Unsetting PV's claimRef", "pv", pv.Name)
+	if err := c.Update(ctx, pvCopy); err != nil {
+		return nil, err
+	}
+
+	log.Info("PV should be Available now")
+
+	// At this point, the PV becomes Available, if it's reclaim policy is "Retain".
+	// I have not yet tested it with "Delete" but perhaps it's deleted automatically after the update?
+	// https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain
+
+	return nil, nil
+}
--- a/docs/releasenotes/0.24.md
+++ b/docs/releasenotes/0.24.md
@@ -0,0 +1,54 @@
+# actions-runner-controller v0.24.0
+
+All changes in this release can be found in the milestone https://github.com/actions-runner-controller/actions-runner-controller/milestone/4
+
+This log documents breaking and major enhancements
+
+## Upgrading
+
+In case you're using our Helm chart to deploy ARC, use the chart 0.19.0 or greater. Don't miss upgrading CRDs as usual! Helm doesn't upgrade CRDs.
+
+## BREAKING CHANGE : Support for `--once` is being dropped
+
+> **Warning**: If you're using ARC's official runer image, make sure to update the image tag to `v2.292.0` BEFORE upgrading ARC
+
+In #1385 we changed ARC to NOT automatically set the feature flag `RUNNER_FEATURE_FLAG_EPHEMERAL=true`. If you're using ARC's official runer image, make sure to update the image tag to `v2.292.0` before upgrading ARC, because that's the first runner image release since we changed the default to `--ephemeral`. If you kept using an older runner image after upgrading ARC, you end up using `--once` which is unreliable and had been deprecated since almost a year ago.
+
+>> **Warning**: If you're using a custom runner image, incorporate changes made in #1384 to your runner image dockerfile
+
+If you're building a custom runner image on your own and it still requires the user to specify `RUNNER_FEATURE_FLAG_EPHEMERAL=true` to use `--ephemeral`, check #1384 and update your custom runner image dockerfile accordingly. Otherwise, you may unexpectedly end up with using `--once` after upgrading ARC, because that was the previous default.
+
+Relevant PR(s): #1384, #1385
+
+## FIX : Prevent runner form stucking in Terminating when the container disappeared
+
+We occasionally heard about runnner pods stuck in Terminating after the node and containers running on it disappeared due to, for example, the machine terminated prematurely.
+
+We now set runner pods' restartPolicy to `Never` and remove runner pods stuck in `Waiting` after restarting, so that the pods are more likely to NOT stuck forever.
+
+Relevant PR(s): #1395, #1420
+
+## ENHANCEMENT : Support arbitrarily setting `privileged: true` for runner container
+
+This is a frequently asked feature that alows you to force `privileged: true` in case you don't need docker but still need privileged tasks to be run in a job step.
+
+In combination with a container runtime like `sysbox` this should enable you to run docker builds within the dind sidecar, all without privileges. See [the discussion related to Sysbox](https://github.com/actions-runner-controller/actions-runner-controller/discussions/977) for more information.
+
+Note that we ARC maintainers still have no bandwidth to provide a complete description on how to make ARC work with `sysbox` yet, but almost certainly we'd welcome contributions to the documentation if you managed to make it work.
+
+Relevant PR(s): #1383
+
+## ENHANCEMENT : RunnerSet can now retain PVs accross restarts
+
+This enhancement makes it more practical to use RunnerSet in combination with `volumeClaimTemplates` to make your workflow jobs faster.
+
+Please see our updated ["Custom Volume Mounts" section in the documentation](https://github.com/actions-runner-controller/actions-runner-controller#custom-volume-mounts) for more information. Currently, we cover caching Docker image layers, go mod/build, and PV-backed runner work directory(Although this one is backed by another feature unrelated to this enhancement under the hood).
+
+Relevant PR(s): #1340
+
+## ENHANCEMENT : OpenSSF scorecard adoption
+
+We assessed the project's security by following OpenSSF scorecard checks and adopting OpenSSF best practices.
+It should help you judge the security throughout ARC's development and release processes.
+
+Relevant PR(s): #1461
--- a/github/github.go
+++ b/github/github.go
@@ -12,7 +12,7 @@ import (

 	"github.com/actions-runner-controller/actions-runner-controller/github/metrics"
 	"github.com/actions-runner-controller/actions-runner-controller/logging"
-	"github.com/bradleyfalzon/ghinstallation"
+	"github.com/bradleyfalzon/ghinstallation/v2"
 	"github.com/go-logr/logr"
 	"github.com/google/go-github/v39/github"
 	"github.com/gregjones/httpcache"
--- a/go.mod
+++ b/go.mod
@@ -1,23 +1,23 @@
 module github.com/actions-runner-controller/actions-runner-controller

-go 1.17
+go 1.18

 require (
-	github.com/bradleyfalzon/ghinstallation v1.1.1
+	github.com/bradleyfalzon/ghinstallation/v2 v2.0.4
 	github.com/davecgh/go-spew v1.1.1
-	github.com/go-logr/logr v1.2.0
+	github.com/go-logr/logr v1.2.3
 	github.com/google/go-cmp v0.5.8
 	github.com/google/go-github/v39 v39.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.17.0
-	github.com/prometheus/client_golang v1.12.1
-	github.com/stretchr/testify v1.7.0
+	github.com/onsi/gomega v1.19.0
+	github.com/prometheus/client_golang v1.12.2
+	github.com/stretchr/testify v1.7.1
 	github.com/teambition/rrule-go v1.8.0
 	go.uber.org/zap v1.21.0
-	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a
+	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
 	gomodules.xyz/jsonpatch/v2 v2.2.0
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
@@ -30,14 +30,14 @@ require (
 	cloud.google.com/go v0.81.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-github/v29 v29.0.2 // indirect
+	github.com/google/go-github/v41 v41.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
@@ -57,7 +57,7 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
--- a/go.sum
+++ b/go.sum
@@ -78,12 +78,11 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bradleyfalzon/ghinstallation v1.1.1 h1:pmBXkxgM1WeF8QYvDLT5kuQiHMcmf+X015GI0KM/E3I=
-github.com/bradleyfalzon/ghinstallation v1.1.1/go.mod h1:vyCmHTciHx/uuyN82Zc3rXN3X2KTK8nUTCrTMwAhcug=
+github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 h1:tXKVfhE7FcSkhkv0UwkLvPDeZ4kz6OXd0PKPlFqf81M=
+github.com/bradleyfalzon/ghinstallation/v2 v2.0.4/go.mod h1:B40qPqJxWE0jDZgOR1JmaMy+4AY1eBP+IByOvqyAKp0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -112,7 +111,6 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
@@ -153,8 +151,9 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk=
 github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -171,6 +170,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
+github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -222,15 +223,12 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
-github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
 github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ=
 github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg=
+github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -369,14 +367,14 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
-github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
+github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -396,8 +394,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
+github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -458,11 +456,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/teambition/rrule-go v1.7.2 h1:goEajFWYydfCgavn2m/3w5U+1b3PGqPUHx/fFSVfTy0=
-github.com/teambition/rrule-go v1.7.2/go.mod h1:mBJ1Ht5uboJ6jexKdNUJg2NcwP8uUMNvStWXlJD3MvU=
 github.com/teambition/rrule-go v1.8.0 h1:a/IX5s56hGkFF+nRlJUooZU/45OTeeldBGL29nDKIHw=
 github.com/teambition/rrule-go v1.8.0/go.mod h1:Ieq5AbrKGciP1V//Wq8ktsTXwSwJHDD5mD/wLBGl3p4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -506,17 +503,14 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
-go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
 go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -608,13 +602,13 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -629,8 +623,8 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a h1:qfl7ob3DIEs3Ml9oLuPwY2N04gymzAW04WsUQHIClgM=
-golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
+golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -707,7 +701,6 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -793,7 +786,6 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
 gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
@@ -948,31 +940,16 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg=
-k8s.io/api v0.23.4 h1:85gnfXQOWbJa1SiWGpE9EEtHs0UVvDyIsSMpEtl2D4E=
-k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI=
 k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
 k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
-k8s.io/apiextensions-apiserver v0.23.0 h1:uii8BYmHYiT2ZTAJxmvc3X8UhNYMxl2A0z0Xq3Pm+WY=
-k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4=
 k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI=
 k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ=
-k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc=
-k8s.io/apimachinery v0.23.4 h1:fhnuMd/xUL3Cjfl64j5ULKZ1/J9n8NuQEgNL+WXWfdM=
-k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
 k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0=
 k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
-k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4=
 k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw=
-k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
-k8s.io/client-go v0.23.4 h1:YVWvPeerA2gpUudLelvsolzH7c2sFoXXR5wM/sWqNFU=
-k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0=
 k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8=
 k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
-k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
 k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
-k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8=
-k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=
 k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE=
 k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
@@ -983,23 +960,17 @@ k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
-sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
-sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
 sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA=
 sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
--- a/hack/signrel/README.md
+++ b/hack/signrel/README.md
@@ -0,0 +1,67 @@
+# signrel
+
+`signrel` is the utility command for downloading `actions-runner-controller` release assets, sigining those, and uploading the signature files.
+
+## Verifying Release Assets
+
+For users, browse https://keys.openpgp.org/search?q=D8078411E3D8400B574EDB0441B69B728F095A87 and download the public key, or refer to [the instruction](https://keys.openpgp.org/about/usage#gnupg-retrieve) to import the key onto your machine.
+
+Next, you'll want to verify the signature of the download asset somehow.
+
+With `gpg`, you would usually do that by downloading both the asset and the signature files from our specific release page, and run `gpg --verify` like:
+
+```console
+# Download the asset
+curl -LO https://github.com/actions-runner-controller/actions-runner-controller/releases/download/v0.23.0/actions-runner-controller.yaml
+
+# Download the signature file
+curl -LO https://github.com/actions-runner-controller/actions-runner-controller/releases/download/v0.23.0/actions-runner-controller.yaml.asc
+
+# Verify
+gpg --verify actions-runner-controller.yaml{.asc,}
+```
+
+On succesful verification, the gpg command would output:
+
+```
+gpg: Signature made Tue 10 May 2022 04:15:32 AM UTC
+gpg:                using RSA key D8078411E3D8400B574EDB0441B69B728F095A87
+gpg: checking the trustdb
+gpg: marginals needed: 3  completes needed: 1  trust model: pgp
+gpg: depth: 0  valid:   1  signed:   0  trust: 0-, 0q, 0n, 0m, 0f, 1u
+gpg: next trustdb check due at 2024-05-09
+gpg: Good signature from "Yusuke Kuoka <ykuoka@gmail.com>" [ultimate]
+```
+
+## Signing Release Assets
+
+Assuming you are a maintainer of the project who has admin permission, run the command like the below to sign assets and upload the signature files:
+
+```console
+$ cd hack/signrel
+
+$ for v in v0.23.0 actions-runner-controller-0.18.0 v0.22.3 v0.22.2 actions-runner-controller-0.17.2; do TAG=$v go run . sign; done
+
+Downloading actions-runner-controller.yaml to downloads/v0.23.0/actions-runner-controller.yaml
+Uploading downloads/v0.23.0/actions-runner-controller.yaml.asc
+downloads/v0.23.0/actions-runner-controller.yaml.asc has been already uploaded
+Downloading actions-runner-controller-0.18.0.tgz to downloads/actions-runner-controller-0.18.0/actions-runner-controller-0.18.0.tgz
+Uploading downloads/actions-runner-controller-0.18.0/actions-runner-controller-0.18.0.tgz.asc
+Upload completed: *snip*
+Downloading actions-runner-controller.yaml to downloads/v0.22.3/actions-runner-controller.yaml
+Uploading downloads/v0.22.3/actions-runner-controller.yaml.asc
+Upload completed: *snip*
+Downloading actions-runner-controller.yaml to downloads/v0.22.2/actions-runner-controller.yaml
+Uploading downloads/v0.22.2/actions-runner-controller.yaml.asc
+Upload completed: *snip*
+Downloading actions-runner-controller-0.17.2.tgz to downloads/actions-runner-controller-0.17.2/actions-runner-controller-0.17.2.tgz
+Uploading downloads/actions-runner-controller-0.17.2/actions-runner-controller-0.17.2.tgz.asc
+Upload completed: *snip*
+actions-runner-controller-0.17.2.tgz.asc"}
+```
+
+To retrieve all the available release tags, run:
+
+```
+$ go run . tags | jq -r .[].tag_name
+```
--- a/hack/signrel/go.mod
+++ b/hack/signrel/go.mod
@@ -0,0 +1,3 @@
+module github.com/actions-runner-controller/actions-runner-controller/hack/sigrel
+
+go 1.17
--- a/hack/signrel/main.go
+++ b/hack/signrel/main.go
@@ -0,0 +1,325 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	a := &githubReleaseAsset{}
+
+	if len(os.Args) != 2 {
+		fmt.Fprintf(os.Stderr, "Invalid command: %v\n", os.Args)
+		fmt.Fprintf(os.Stderr, "USAGE: signrel [list-tags|sign-assets]\n")
+		os.Exit(2)
+	}
+
+	switch cmd := os.Args[1]; cmd {
+	case "tags":
+		listTags(a)
+	case "sign":
+		tag := os.Getenv("TAG")
+		sign(a, tag)
+	default:
+		fmt.Fprintf(os.Stderr, "Unknown command %s\n", cmd)
+		os.Exit(2)
+	}
+}
+
+func listTags(a *githubReleaseAsset) {
+	_, err := a.getRecentReleases(owner, repo)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func sign(a *githubReleaseAsset, tag string) {
+	if err := a.Download(tag, "downloads"); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+type Release struct {
+	ID int64 `json:"id"`
+}
+
+type Asset struct {
+	Name string `json:"name"`
+	ID   int64  `json:"id"`
+	URL  string `json:"url"`
+}
+
+type AssetsResponse struct {
+	Assets []Asset
+}
+
+type githubReleaseAsset struct {
+}
+
+const (
+	owner = "actions-runner-controller"
+	repo  = "actions-runner-controller"
+)
+
+// GetFile downloads the give URL into the given path. The URL must
+// reference a single file. If possible, the Getter should check if
+// the remote end contains the same file and no-op this operation.
+func (a *githubReleaseAsset) Download(tag string, dstDir string) error {
+	release, err := a.getReleaseByTag(owner, repo, tag)
+	if err != nil {
+		return err
+	}
+
+	assets, err := a.getAssetsByReleaseID(owner, repo, release.ID)
+	if err != nil {
+		return err
+	}
+
+	d := filepath.Join(dstDir, tag)
+	if err := os.MkdirAll(d, 0755); err != nil {
+		return err
+	}
+
+	for _, asset := range assets {
+		if strings.HasSuffix(asset.Name, ".asc") {
+			continue
+		}
+
+		p := filepath.Join(d, asset.Name)
+		fmt.Fprintf(os.Stderr, "Downloading %s to %s\n", asset.Name, p)
+		if err := a.getFile(p, owner, repo, asset.ID); err != nil {
+			return err
+		}
+
+		if info, _ := os.Stat(p + ".asc"); info == nil {
+			_, err := a.sign(p)
+			if err != nil {
+				return err
+			}
+		}
+
+		sig := p + ".asc"
+
+		fmt.Fprintf(os.Stdout, "Uploading %s\n", sig)
+
+		if err := a.upload(sig, release.ID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (a *githubReleaseAsset) getRecentReleases(owner, repo string) (*Release, error) {
+	reqURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", owner, repo)
+
+	req, err := http.NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if gt := os.Getenv("GITHUB_TOKEN"); gt != "" {
+		req.Header = make(http.Header)
+		req.Header.Add("authorization", "token "+gt)
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET %s: %s", reqURL, res.Status)
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Fprintf(os.Stdout, "%s\n", string(body))
+
+	return nil, nil
+}
+
+func (a *githubReleaseAsset) getReleaseByTag(owner, repo, tag string) (*Release, error) {
+	var release Release
+
+	reqURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", owner, repo, tag)
+
+	req, err := http.NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if gt := os.Getenv("GITHUB_TOKEN"); gt != "" {
+		req.Header = make(http.Header)
+		req.Header.Add("authorization", "token "+gt)
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET %s: %s", reqURL, res.Status)
+	}
+
+	d := json.NewDecoder(res.Body)
+
+	if err := d.Decode(&release); err != nil {
+		return nil, err
+	}
+
+	return &release, nil
+}
+
+func (a *githubReleaseAsset) getAssetsByReleaseID(owner, repo string, releaseID int64) ([]Asset, error) {
+	var assets []Asset
+
+	reqURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/%d/assets", owner, repo, releaseID)
+
+	req, err := http.NewRequest("GET", reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if gt := os.Getenv("GITHUB_TOKEN"); gt != "" {
+		req.Header = make(http.Header)
+		req.Header.Add("authorization", "token "+gt)
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET %s: %s", reqURL, res.Status)
+	}
+
+	d := json.NewDecoder(res.Body)
+
+	if err := d.Decode(&assets); err != nil {
+		return nil, err
+	}
+
+	return assets, nil
+}
+
+func (a *githubReleaseAsset) getFile(dst string, owner, repo string, assetID int64) error {
+	// Create all the parent directories if needed
+	dir := filepath.Dir(dst)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("mkdir %s: %w", dir, err)
+	}
+
+	req, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/assets/%d", owner, repo, assetID), nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.Header = make(http.Header)
+	if gt := os.Getenv("GITHUB_TOKEN"); gt != "" {
+		req.Header.Add("authorization", "token "+gt)
+	}
+	req.Header.Add("accept", "application/octet-stream")
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, os.FileMode(0666))
+	if err != nil {
+		return fmt.Errorf("open file %s: %w", dst, err)
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, res.Body); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *githubReleaseAsset) sign(path string) (string, error) {
+	pass := os.Getenv("SIGNREL_PASSWORD")
+	cmd := exec.Command("gpg", "--armor", "--detach-sign", "--pinentry-mode", "loopback", "--passphrase", pass, path)
+	cap, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "gpg: %s", string(cap))
+		return "", err
+	}
+	return path + ".asc", nil
+}
+
+func (a *githubReleaseAsset) upload(sig string, releaseID int64) error {
+	assetName := filepath.Base(sig)
+	url := fmt.Sprintf("https://uploads.github.com/repos/%s/%s/releases/%d/assets?name=%s", owner, repo, releaseID, assetName)
+	f, err := os.Open(sig)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	size, err := f.Seek(0, 2)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Seek(0, 0)
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("POST", url, f)
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.Header = make(http.Header)
+	if gt := os.Getenv("GITHUB_TOKEN"); gt != "" {
+		req.Header.Add("authorization", "token "+gt)
+	}
+	req.Header.Add("content-type", "application/octet-stream")
+
+	req.ContentLength = size
+	req.Header.Add("accept", "application/vnd.github.v3+json")
+
+	// blob, _ := httputil.DumpRequestOut(req, true)
+	// fmt.Printf("%s\n", blob)
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+
+	if res.StatusCode == 422 {
+		fmt.Fprintf(os.Stdout, "%s has been already uploaded\n", sig)
+		return nil
+	}
+
+	if res.StatusCode >= 300 {
+		return fmt.Errorf("unexpected http status %d: %s", res.StatusCode, body)
+	}
+
+	fmt.Fprintf(os.Stdout, "Upload completed: %s\n", body)
+
+	return err
+}
--- a/main.go
+++ b/main.go
@@ -71,6 +71,7 @@ func main() {
 		metricsAddr          string
 		enableLeaderElection bool
 		leaderElectionId     string
+		port                 int
 		syncPeriod           time.Duration

 		gitHubAPICacheDuration time.Duration
@@ -98,8 +99,8 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&leaderElectionId, "leader-election-id", "actions-runner-controller", "Controller id for leader election.")
-	flag.StringVar(&runnerImage, "runner-image", defaultRunnerImage, "The image name of self-hosted runner container.")
-	flag.StringVar(&dockerImage, "docker-image", defaultDockerImage, "The image name of docker sidecar container.")
+	flag.StringVar(&runnerImage, "runner-image", defaultRunnerImage, "The image name of self-hosted runner container to use by default if one isn't defined in yaml.")
+	flag.StringVar(&dockerImage, "docker-image", defaultDockerImage, "The image name of docker sidecar container to use by default if one isn't defined in yaml.")
 	flag.Var(&runnerImagePullSecrets, "runner-image-pull-secret", "The default image-pull secret name for self-hosted runner container.")
 	flag.StringVar(&dockerRegistryMirror, "docker-registry-mirror", "", "The default Docker Registry Mirror used by runners.")
 	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
@@ -113,6 +114,7 @@ func main() {
 	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
 	flag.DurationVar(&gitHubAPICacheDuration, "github-api-cache-duration", 0, "DEPRECATED: The duration until the GitHub API cache expires. Setting this to e.g. 10m results in the controller tries its best not to make the same API call within 10m to reduce the chance of being rate-limited. Defaults to mostly the same value as sync-period. If you're tweaking this in order to make autoscaling more responsive, you'll probably want to tweak sync-period, too")
 	flag.DurationVar(&defaultScaleDownDelay, "default-scale-down-delay", controllers.DefaultScaleDownDelay, "The approximate delay for a scale down followed by a scale up, used to prevent flapping (down->up->down->... loop)")
+	flag.IntVar(&port, "port", 9443, "The port to which the admission webhook endpoint should bind")
 	flag.DurationVar(&syncPeriod, "sync-period", 1*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled.")
 	flag.Var(&commonRunnerLabels, "common-runner-labels", "Runner labels in the K1=V1,K2=V2,... format that are inherited all the runners created by the controller. See https://github.com/actions-runner-controller/actions-runner-controller/issues/321 for more information")
 	flag.StringVar(&namespace, "watch-namespace", "", "The namespace to watch for custom resources. Set to empty for letting it watch for all namespaces.")
@@ -136,7 +138,7 @@ func main() {
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
 		LeaderElectionID:   leaderElectionId,
-		Port:               9443,
+		Port:               port,
 		SyncPeriod:         &syncPeriod,
 		Namespace:          namespace,
 	})
@@ -216,9 +218,11 @@ func main() {
 		"github-api-cache-duration", gitHubAPICacheDuration,
 		"default-scale-down-delay", defaultScaleDownDelay,
 		"sync-period", syncPeriod,
-		"runner-image", runnerImage,
-		"docker-image", dockerImage,
+		"default-runner-image", runnerImage,
+		"default-docker-image", dockerImage,
 		"common-runnner-labels", commonRunnerLabels,
+		"leader-election-enabled", enableLeaderElection,
+		"leader-election-id", leaderElectionId,
 		"watch-namespace", namespace,
 	)

@@ -238,6 +242,18 @@ func main() {
 		GitHubClient: ghClient,
 	}

+	runnerPersistentVolumeReconciler := &controllers.RunnerPersistentVolumeReconciler{
+		Client: mgr.GetClient(),
+		Log:    log.WithName("runnerpersistentvolume"),
+		Scheme: mgr.GetScheme(),
+	}
+
+	runnerPersistentVolumeClaimReconciler := &controllers.RunnerPersistentVolumeClaimReconciler{
+		Client: mgr.GetClient(),
+		Log:    log.WithName("runnerpersistentvolumeclaim"),
+		Scheme: mgr.GetScheme(),
+	}
+
 	if err = runnerPodReconciler.SetupWithManager(mgr); err != nil {
 		log.Error(err, "unable to create controller", "controller", "RunnerPod")
 		os.Exit(1)
@@ -248,6 +264,16 @@ func main() {
 		os.Exit(1)
 	}

+	if err = runnerPersistentVolumeReconciler.SetupWithManager(mgr); err != nil {
+		log.Error(err, "unable to create controller", "controller", "RunnerPersistentVolume")
+		os.Exit(1)
+	}
+
+	if err = runnerPersistentVolumeClaimReconciler.SetupWithManager(mgr); err != nil {
+		log.Error(err, "unable to create controller", "controller", "RunnerPersistentVolumeClaim")
+		os.Exit(1)
+	}
+
 	if err = (&actionsv1alpha1.Runner{}).SetupWebhookWithManager(mgr); err != nil {
 		log.Error(err, "unable to create webhook", "webhook", "Runner")
 		os.Exit(1)
--- a/runner/.dockerignore
+++ b/runner/.dockerignore
@@ -0,0 +1,2 @@
+*.dockerfile
+Makefile
--- a/runner/Makefile
+++ b/runner/Makefile
@@ -4,7 +4,7 @@ DIND_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind
 TAG ?= latest
 TARGETPLATFORM ?= $(shell arch)

-RUNNER_VERSION ?= 2.290.1
+RUNNER_VERSION ?= 2.293.0
 DOCKER_VERSION ?= 20.10.12

 # default list of platforms for which multiarch image is built
@@ -25,8 +25,18 @@ else
 endif

 docker-build-ubuntu:
-	docker build --build-arg TARGETPLATFORM=${TARGETPLATFORM} --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} .
-	docker build --build-arg TARGETPLATFORM=${TARGETPLATFORM} --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${DIND_RUNNER_NAME}:${TAG} -f Dockerfile.dindrunner .
+	docker build \
+	  --build-arg TARGETPLATFORM=${TARGETPLATFORM} \
+	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+	  -f actions-runner.dockerfile \
+	  -t ${NAME}:${TAG} .
+	docker build \
+	  --build-arg TARGETPLATFORM=${TARGETPLATFORM} \
+	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+	  -f actions-runner-dind.dockerfile \
+	  -t ${DIND_RUNNER_NAME}:${TAG} .

 docker-push-ubuntu:
 	docker push ${NAME}:${TAG}
@@ -36,17 +46,17 @@ docker-buildx-ubuntu:
 	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
    export DOCKER_BUILDKIT=1
 	@if ! docker buildx ls | grep -q container-builder; then\
-		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	  docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
 	fi
 	docker buildx build --platform ${PLATFORMS} \
-		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
-		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
-		-t "${NAME}:${TAG}" \
-		-f Dockerfile \
-		. ${PUSH_ARG}
+	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+	  -f actions-runner.dockerfile \
+	  -t "${NAME}:${TAG}" \
+	  . ${PUSH_ARG}
 	docker buildx build --platform ${PLATFORMS} \
-		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
-		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
-		-t "${DIND_RUNNER_NAME}:${TAG}" \
-		-f Dockerfile.dindrunner \
-		. ${PUSH_ARG}
+	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+	  -f actions-runner-dind.dockerfile \
+	  -t "${DIND_RUNNER_NAME}:${TAG}" \
+	  . ${PUSH_ARG}
--- a/runner/actions-runner-dind.dockerfile
+++ b/runner/actions-runner-dind.dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:20.04

 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.290.1
+ARG RUNNER_VERSION=2.293.0
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
--- a/runner/actions-runner.dockerfile
+++ b/runner/actions-runner.dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:20.04

 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.290.1
+ARG RUNNER_VERSION=2.293.0
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
--- a/runner/entrypoint.sh
+++ b/runner/entrypoint.sh
@@ -76,7 +76,7 @@ cd ${RUNNER_HOME}
 # past that point, it's all relative pathes from /runner

 config_args=()
-if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" == "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
+if [ "${RUNNER_FEATURE_FLAG_ONCE:-}" != "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
  config_args+=(--ephemeral)
  log.debug 'Passing --ephemeral to config.sh to enable the ephemeral runner.'
 fi
@@ -134,17 +134,17 @@ cat .runner
 #     -H "Authorization: bearer ${GITHUB_TOKEN}"
 #     https://api.github.com/repos/USER/REPO/actions/runners/171

-if [ -z "${UNITTEST:-}" ]; then
+# Hack due to the DinD volumes
+if [ -z "${UNITTEST:-}" ] && [ -e ./externalstmp ]; then
  mkdir -p ./externals
-  # Hack due to the DinD volumes
  mv ./externalstmp/* ./externals/
 fi

 args=()
-if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" != "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
+if [ "${RUNNER_FEATURE_FLAG_ONCE:-}" == "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
  args+=(--once)
  log.warning 'Passing --once is deprecated and will be removed as an option' \
-    'from the image and actions-runner-controller at the release of 0.24.0.' \
+    'from the image and actions-runner-controller at the release of 0.25.0.' \
    'Upgrade to GHES => 3.3 to continue using actions-runner-controller. If' \
    'you are using github.com ignore this warning.'
 fi
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -3,6 +3,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strconv"
 	"time"
@@ -36,24 +37,26 @@ var (
 			EnableBuildX: true,
 		},
 		{
-			Dockerfile: "../../runner/Dockerfile",
+			Dockerfile: "../../runner/actions-runner.dockerfile",
 			Args: []testing.BuildArg{
 				{
 					Name:  "RUNNER_VERSION",
-					Value: "2.289.2",
+					Value: "2.291.1",
 				},
 			},
-			Image: runnerImage,
+			Image:        runnerImage,
+			EnableBuildX: true,
 		},
 		{
-			Dockerfile: "../../runner/Dockerfile.dindrunner",
+			Dockerfile: "../../runner/actions-runner-dind.dockerfile",
 			Args: []testing.BuildArg{
 				{
 					Name:  "RUNNER_VERSION",
-					Value: "2.289.2",
+					Value: "2.291.1",
 				},
 			},
-			Image: runnerDindImage,
+			Image:        runnerDindImage,
+			EnableBuildX: true,
 		},
 	}

@@ -137,6 +140,10 @@ func TestE2E(t *testing.T) {
 	t.Run("Verify workflow run result", func(t *testing.T) {
 		env.verifyActionsWorkflowRun(t)
 	})
+
+	if os.Getenv("ARC_E2E_NO_CLEANUP") != "" {
+		t.FailNow()
+	}
 }

 func TestE2ERunnerDeploy(t *testing.T) {
@@ -179,7 +186,9 @@ func TestE2ERunnerDeploy(t *testing.T) {
 		env.verifyActionsWorkflowRun(t)
 	})

-	t.FailNow()
+	if os.Getenv("ARC_E2E_NO_CLEANUP") != "" {
+		t.FailNow()
+	}
 }

 type env struct {
@@ -198,7 +207,6 @@ type env struct {
 	githubTokenWebhook                                       string
 	testEnterprise                                           string
 	testEphemeral                                            string
-	featureFlagEphemeral                                     *bool
 	scaleDownDelaySecondsAfterScaleOut                       int64
 	minReplicas                                              int64
 	dockerdWithinRunnerContainer                             bool
@@ -232,11 +240,7 @@ func initTestEnv(t *testing.T) *env {
 	e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO", "")
 	e.testEnterprise = testing.Getenv(t, "TEST_ENTERPRISE", "")
 	e.testEphemeral = testing.Getenv(t, "TEST_EPHEMERAL", "")
-	e.testJobs = createTestJobs(id, testResultCMNamePrefix, 20)
-
-	if ephemeral, err := strconv.ParseBool(testing.Getenv(t, "TEST_FEATURE_FLAG_EPHEMERAL", "")); err == nil {
-		e.featureFlagEphemeral = &ephemeral
-	}
+	e.testJobs = createTestJobs(id, testResultCMNamePrefix, 6)

 	e.scaleDownDelaySecondsAfterScaleOut, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT", "10"), 10, 32)
 	e.minReplicas, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_MIN_REPLICAS", "1"), 10, 32)
@@ -308,10 +312,6 @@ func (e *env) installActionsRunnerController(t *testing.T) {
 		fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
 	}

-	if e.featureFlagEphemeral != nil {
-		varEnv = append(varEnv, fmt.Sprintf("RUNNER_FEATURE_FLAG_EPHEMERAL=%v", *e.featureFlagEphemeral))
-	}
-
 	if e.useApp {
 		varEnv = append(varEnv,
 			"ACCEPTANCE_TEST_SECRET_TYPE=app",
@@ -407,6 +407,62 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 				{
 					Uses: testing.ActionsCheckoutV2,
 				},
+				{
+					// This might be the easiest way to handle permissions without use of securityContext
+					// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
+					Run: "sudo chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
+				},
+				{
+					// This might be the easiest way to handle permissions without use of securityContext
+					// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
+					Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
+				},
+				{
+					Uses: "actions/setup-go@v3",
+					With: &testing.With{
+						GoVersion: "1.18.2",
+					},
+				},
+				{
+					Run: "go version",
+				},
+				{
+					Run: "go build .",
+				},
+				{
+					// https://github.com/docker/buildx/issues/413#issuecomment-710660155
+					// To prevent setup-buildx-action from failing with:
+					//   error: could not create a builder instance with TLS data loaded from environment. Please use `docker context create <context-name>` to create a context for current environment and then create a builder instance with `docker buildx create <context-name>`
+					Run: "docker context create mycontext",
+				},
+				{
+					Run: "docker context use mycontext",
+				},
+				{
+					Name: "Set up Docker Buildx",
+					Uses: "docker/setup-buildx-action@v1",
+					With: &testing.With{
+						BuildkitdFlags: "--debug",
+						Endpoint:       "mycontext",
+						// As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build`
+						// so we need to use `docker buildx build` in the next step
+						Install: false,
+					},
+				},
+				{
+					Run: "docker buildx build --platform=linux/amd64 " +
+						"--cache-from=type=local,src=/home/runner/.cache/buildx " +
+						"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max " +
+						".",
+				},
+				{
+					// https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
+					// See https://github.com/moby/buildkit/issues/1896 for why this is needed
+					Run: "rm -rf /home/runner/.cache/buildx && mv /home/runner/.cache/buildx-new /home/runner/.cache/buildx",
+				},
+				{
+					Run: "ls -lah /home/runner/.cache/*",
+				},
 				{
 					Uses: "azure/setup-kubectl@v1",
 					With: &testing.With{
--- a/test/entrypoint/should_work_use_once_switch/test.sh
+++ b/test/entrypoint/should_work_use_once_switch/test.sh
@@ -1,90 +0,0 @@
-#!/usr/bin/env bash
-
-# UNITTEST: should work legacy once switch set
-# Will simulate a scenario where RUNNER_FEATURE_FLAG_EPHEMERAL=false. expects:
-# - the configuration step to be run exactly once
-# - the entrypoint script to exit with no error
-# - the run.sh script to run with the --once flag
-
-source ../assets/logging.sh
-
-entrypoint_log() {
-  while read I; do
-    printf "\tentrypoint.sh: $I\n"
-  done
-}
-
-log "Setting up test area"
-export RUNNER_HOME=testarea
-mkdir -p ${RUNNER_HOME}
-
-log "Setting up the test"
-export UNITTEST=true
-export RUNNER_NAME="example_runner_name"
-export RUNNER_REPO="myorg/myrepo"
-export RUNNER_TOKEN="xxxxxxxxxxxxx"
-export RUNNER_FEATURE_FLAG_EPHEMERAL="false"
-export RUNNER_EPHEMERAL="true"
-
-# run.sh and config.sh get used by the runner's real entrypoint.sh and are part of actions/runner.
-# We change symlink dummy versions so the entrypoint.sh can run allowing us to test the real entrypoint.sh
-log "Symlink dummy config.sh and run.sh"
-ln -s ../../assets/config.sh ${RUNNER_HOME}/config.sh
-ln -s ../../assets/run.sh ${RUNNER_HOME}/run.sh
-
-cleanup() {
-  rm -rf ${RUNNER_HOME}
-  unset UNITTEST
-  unset RUNNERHOME
-  unset RUNNER_NAME
-  unset RUNNER_REPO
-  unset RUNNER_TOKEN
-  unset RUNNER_EPHEMERAL
-  unset RUNNER_FEATURE_FLAG_EPHEMERAL
-}
-
-# Always run cleanup when test ends regardless of how it ends
-trap cleanup SIGINT SIGTERM SIGQUIT EXIT
-
-log "Running the entrypoint"
-log ""
-
-# run.sh and config.sh get used by the runner's real entrypoint.sh and are part of actions/runner.
-# We change symlink dummy versions so the entrypoint.sh can run allowing us to test the real entrypoint.sh
-../../../runner/entrypoint.sh 2> >(entrypoint_log)
-
-if [ "$?" != "0" ]; then
-  error "==========================================="
-  error "FAIL | Entrypoint script did not exit successfully"
-  exit 1
-fi
-
-log "Testing if we went through the configuration step only once"
-count=`cat ${RUNNER_HOME}/counter || echo "not_found"`
-if [ ${count} != "1" ]; then
-  error "==============================================="
-  error "FAIL | The configuration step was not run exactly once"
-  exit 1
-fi
-
-success "PASS | The configuration ran ${count} time(s)"
-
-log "Testing if the configuration included the --once flag"
-if ! grep -q -- '--once' ${RUNNER_HOME}/runner_args; then
-  error "==============================================="
-  error "FAIL | The configuration did not include the --once flag, config printed below:"
-  exit 1
-fi
-
-success "PASS | The --once argument was passed in"
-
-log "Testing if run.sh ran"
-if [ ! -f "${RUNNER_HOME}/run_sh_ran" ]; then
-  error "=============================="
-  error "FAIL | The runner service has not run"
-  exit 1
-fi
-success "PASS | run.sh ran"
-success ""
-success "==========================="
-success "Test completed successfully"
--- a/testing/workflow.go
+++ b/testing/workflow.go
@@ -42,5 +42,13 @@ type Step struct {
 }

 type With struct {
-	Version string `json:"version,omitempty"`
+	Version   string `json:"version,omitempty"`
+	GoVersion string `json:"go-version,omitempty"`
+
+	// https://github.com/docker/setup-buildx-action#inputs
+	BuildkitdFlags string `json:"buildkitd-flags,omitempty"`
+	Install        bool   `json:"install,omitempty"`
+	// This can be either the address or the context name
+	// https://github.com/docker/buildx/blob/master/docs/reference/buildx_create.md#description
+	Endpoint string `json:"endpoint,omitempty"`
 }
Author	SHA1	Message	Date
Callum Tait	7fd5e24961	chore: bump chart to app 0.24.1 (#1531 )	2022-06-15 11:34:55 +09:00
Yusuke Kuoka	9974b1a2b7	e2e: Enable buildx in more images (#1530 )	2022-06-14 09:29:30 +01:00
Yusuke Kuoka	bd91b73fd9	chore: update bug_report.yml (#1529 )	2022-06-14 09:29:06 +01:00
Callum Tait	a7ae910ee4	docs: add CRD disclaimer to bug report (#1516 )	2022-06-14 09:42:31 +09:00
Callum Tait	2733c36d0e	ci: publish controller canary to github packages (#1524 ) * ci: publish controller canary to github packages * ci: include image name	2022-06-14 09:10:13 +09:00
Yusuke Kuoka	0ef9a22cd4	Fix confusing PV controller log (#1526 ) Ref #1511	2022-06-14 08:35:04 +09:00
Renovate Bot	933b0c7888	chore(deps): update dependency actions/runner to v2.293.0	2022-06-13 17:09:29 +00:00
renovate[bot]	1b7ec33135	chore(deps): update actions/setup-python action to v4 (#1514 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-06-13 14:07:52 +01:00
Callum Tait	a62882d243	ci: fix permisions (#1512 ) * ci: fix permisions * chore: change to trigger build * ci: add write permission to packages * ci: remove conditionals for docker logins * Update controllers/utils_test.go Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-06-09 10:25:56 +09:00
Callum Tait	0cd13fe51d	ci: align pipeline files and setups (#1484 ) * ci: align pipeline files and setups * ci: more changes * ci: various changes * ci: fix setup-helm action ref * ci: better pipeline name * ci: more format aligning * ci: more format aligning * ci: better job name * ci: supports multiple languages * ci: better pipeline and job names * ci: do a verb-noun thing for consistency * ci: use 'arc' when talking holistically * ci: add caching scope * ci: put canary in a scope * ci: fix syntax error * ci: better pipeline and job names * ci: better job name Co-authored-by: toast-gear <toast-gear@users.noreply.github.com>	2022-06-08 10:04:14 +09:00
Vinícius Garcia	01c8dc237e	Fix example manifests for webhooks-based scaling (#1354 ) * Fix example manifests for webhook based scaling I tried running these on my k8s cluster and I got some easy to fix errors, so I am committing them here. * Fix example manifests for webhook autoscaling with workflow_jobs * Fix the explation on how to setup webhooks on your cluster * Replace unclear comment with actual code examples There was a comment instructing users to add minReplicas and maxReplicas to all the HRA yamls, so I just removed it and added these attributes to the yamls themselves for clarity. * Make clear that using the ingress example is just a suggestion * Apply some text improvements suggested by @mumoshu * Update examples so the webhook server is exposed on a NodePort Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com> * Remove an unnecessary field from one the examples Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com> * Apply suggestion from @mumoshu Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com> * Remove namespace fields from webhook autoscaler examples This change was suggested by @mumoshu * Apply final suggestion from @mumoshu Co-authored-by: Callum Tait <15716903+toast-gear@users.noreply.github.com> Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-06-07 08:33:09 +09:00
Yusuke Kuoka	7c4db63718	chart: Bump appVersion to 0.24.0 (#1505 )	2022-06-03 22:01:35 +09:00
Yusuke Kuoka	3d88b9630a	doc: Add "people" section (#1498 ) Ref #1497	2022-05-31 09:29:15 +09:00
Yusuke Kuoka	1152e6b31d	Add release note for v0.24.0 (#1493 )	2022-05-30 09:10:36 +09:00
renovate[bot]	ac27df8301	chore(deps): update dependency actions/runner to v2.292.0 (#1475 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-27 09:49:46 +09:00
Yusuke Kuoka	9dd26168d6	Fix confusing logs from pv and pvc controllers (#1487 ) Ref https://github.com/actions-runner-controller/actions-runner-controller/issues/1321#issuecomment-1137431212	2022-05-26 18:21:23 +09:00
Yusuke Kuoka	18bfb28c0b	e2e: ARC_E2E_NO_CLEANUP to prevent cleanup (#1470 ) A small improvement to our E2E test suite which allows you to set `ARC_E2E_NO_CLEANUP=whatever` to let it prevent the kind cluster cleanup on successful test run, so that you can rerun it without waiting for the new kind cluster to come up.	2022-05-26 10:59:50 +09:00
renovate[bot]	84210e900b	chore(deps): update actions/setup-python digest to fff15a2 (#1458 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-25 12:12:22 +09:00
Yusuke Kuoka	ef3313d147	doc: Use RunnerSet to retain various cache by leveraging PV (#1464 ) * doc: Use RunnerSet to retain various cache In relation to #1286 and as a follow-up for #1340 * docs: clarify client vs daemon * docs: better wording * Separate RunnerSet examples for docker iimage layer caching * Revert changes on testdata as it is going to be added via #1471 instead * Update README.md Co-authored-by: Callum Tait <15716903+toast-gear@users.noreply.github.com> * fixup! Update README.md * Remove the outdated RunnerSet limitation Co-authored-by: Callum Tait <15716903+toast-gear@users.noreply.github.com>	2022-05-25 11:09:36 +09:00
Yusuke Kuoka	c7eea169ad	test: Add test for runner with generic ephemeral volume as "work" (#1472 ) This adds the test to verify the runner pod generation logic for the case that you use a generic ephemeral volume as "work". It is almost an adaptation of the test cases writetn for RunnerSet in #1471, to RunnerDeployment and Runner.	2022-05-25 10:37:23 +09:00
Yusuke Kuoka	63be0223ad	fix: Avoid duplicate volume and mount name error for generic ephemeral volume as "work" (#1471 ) * fix: Avoid duplicate volume and mount name error for generic ephemeral volume as "work" While manually testing configurations being documented in #1464, I discovered that the use of dynamic ephemeral volume for "work" directory was not working correctly due to the valiadation error. This fixes the runner pod generation logic to not add the default volume and volume mount for "work" dir, so that the error disappears. Ref #1464 * e2e: Ensure work generic ephemeral volume to work as expected	2022-05-22 10:25:50 +09:00
Yusuke Kuoka	5bbea772f7	doc: enhance troubleshooting guide with the scale-to-zero issue (#1469 ) Ref https://github.com/actions-runner-controller/actions-runner-controller/issues/1057#issuecomment-1133439061	2022-05-21 19:00:40 +01:00
Callum Tait	2aa3f1e142	chore: remove stale app config (#1465 )	2022-05-21 14:19:41 +09:00
Yusuke Kuoka	3e988afc09	test: add fuzzing to the test suite (#1463 ) As a part of #1298, we add fuzzing based on Go test's fuzzing support to the test suite	2022-05-19 13:34:23 +01:00
Yusuke Kuoka	84210f3d2b	Bump Go to 1.18.2 (#1462 ) As a part of #1298, I'm going to use Go fuzzing which is availabls since Go 1.18. Co-authored-by: Callum Tait <15716903+toast-gear@users.noreply.github.com>	2022-05-19 10:33:31 +01:00
Yusuke Kuoka	536692181b	docs: Add CII Best Practices badge to README (#1461 ) Ref https://github.com/actions-runner-controller/actions-runner-controller/issues/1298	2022-05-19 10:16:39 +01:00
renovate[bot]	23403172cb	chore(deps): update dependency golang to v1.18.2 (#1229 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-19 17:36:31 +09:00
renovate[bot]	8a8ec43364	chore(deps): update github/codeql-action action to v2.1.11 (#1455 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-18 09:02:26 +09:00
Felipe Galindo Sanchez	78c01fd31d	cleanup some unused code and minor refactors (#1274 ) Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-05-16 18:38:32 +09:00
Bernardo Meurer	bf45aa9f6b	refactor(runner/entrypoint): don't mv externalstmp if it's not there (#1315 )	2022-05-16 18:37:37 +09:00
Callum Tait	b5aa1750bb	ci: match renovate with new dockerfile names (#1453 )	2022-05-16 18:15:44 +09:00
Richard Fussenegger	cdc9d20e7a	Renamed Runner Dockerfiles (#1248 ) Renamed the runner dockerfiles so that we have proper syntax highlighting for them, as well as a consistent way to map from the image name to the dockerfile. Added a `.dockerignore` file to avoid uploading things to the daemon that we never use.	2022-05-16 11:41:28 +09:00
Hyeonmin Park	8035d6d9f8	chart: Add extraPaths to Ingress of GitHub Webhook Server (#1129 ) * chart: Add extraPaths to Ingress of GitHub Webhook Server * Update charts/actions-runner-controller/templates/githubwebhook.ingress.yaml Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com> * Prefix the toYaml expression to remove the extra newline before extra paths Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-05-16 11:34:56 +09:00
Callum Tait	65f7ee92a6	refactor: remove registration runner dead code (#1260 ) We had some dead code left over from the removal of registration runners. Registration runners were removed in #859 #1207 Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-05-16 11:23:39 +09:00
Matéo Mévollon	fca8a538db	docs: document the Docker MTU problem in troubleshooting guide (#1257 ) * docs: document the Docker MTU problem * Update TROUBLESHOOTING.md Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-05-16 11:13:05 +09:00
Nicholas Farley	95ddc77245	Allow customizing the controller webhook port (#1410 ) Closes #1314 Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-05-16 10:33:13 +09:00
Yusuke Kuoka	b5194fd75a	Enhance RunnerSet to optionally retain PVs accross restarts (#1340 ) * Enhance RunnerSet to optionally retain PVs accross restarts This is our initial attempt to bring back the ability to retain PVs across runner pod restarts when using RunnerSet. The implementation is composed of two new controllers, `runnerpersistentvolumeclaim-controller` and `runnerpersistentvolume-controller`. It all starts from our existing `runnerset-controller`. The controller now tries to mark any PVCs created by StatefulSets created for the RunnerSet. Once the controller terminated statefulsets, their corresponding PVCs are clean up by `runnerpersistentvolumeclaim-controller`, then PVs are unbound from their corresponding PVCs by `runnerpersistentvolume-controller` so that they can be reused by future PVCs createf for future StatefulSets that shares the same same StorageClass. Ref #1286 * Update E2E test suite to cover runner, docker, and go caching with RunnerSet + PVs Ref #1286	2022-05-16 09:26:48 +09:00
renovate[bot]	adf69bbea0	fix(deps): update module github.com/prometheus/client_golang to v1.12.2 (#1448 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-16 09:19:55 +09:00
renovate[bot]	b43ef70ac6	fix(deps): update module github.com/bradleyfalzon/ghinstallation/v2 to v2.0.4 (#1452 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-16 08:59:53 +09:00
Yusuke Kuoka	f1caebbaf0	Update codeql.yml (#1451 ) Give up pinning deps with commit IDs because PRs were unreviewable due to missing changelog and it sends PRs for every commit to the master/main branch of the deps, which is undesired. We only need updates for tagged releases!	2022-05-16 08:59:29 +09:00
renovate[bot]	ede28f5046	chore(deps): update helm values quay.io/brancz/kube-rbac-proxy to v0.12.0 (#1323 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-16 08:50:12 +09:00
shettarvinay	f08ab1490d	Update Dockerfile, github/github.go, go.mod and go.sum for fixing CVE-2020-2616 and CVE-2022-24921 (#1230 ) Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>	2022-05-16 08:45:44 +09:00
renovate[bot]	772ca57056	fix(deps): update module github.com/stretchr/testify to v1.7.1 (#1228 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-16 08:43:41 +09:00
renovate[bot]	51b13e3bab	fix(deps): update module github.com/onsi/gomega to v1.19.0 (#1069 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-16 08:41:59 +09:00
Michael Kuhnt	81017b130f	fix(chart): add missing namespace to webhook.ingress (#1417 ) The ingress needs to be deployed in the very same namespace as the service it is forwarding to.	2022-05-16 08:41:35 +09:00
Yusuke Kuoka	bdbcf66569	chore: Add signrel command for sigining arc release assets (#1426 ) * chore: Add signrel command for sigining arc release assets I used this command to sign assets for the recent releases to comply with the recommendation of `5758364c82/docs/checks.md (signed-releases)` Ref #1298 * Implement signrel subcommands for listing tags and signing assets, with docs	2022-05-16 08:40:41 +09:00
Yusuke Kuoka	0e15a78541	Create SECURITY.md (#1424 ) * Create SECURITY.md According to `5758364c82/docs/checks.md (security-policy)` Ref #1298 * Update SECURITY.md	2022-05-16 08:40:16 +09:00
renovate[bot]	f85c3d06d9	chore(deps): update docker/setup-qemu-action action to v2 (#1450 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-14 16:07:23 +01:00
Callum Tait	51ba7d7160	chore: more initialisation info to help debug (#1276 ) * chore: more initialisation info to help debug * chore: clearer flag description * chore: use actual english Co-authored-by: toast-gear <toast-gear@users.noreply.github.com>	2022-05-14 17:11:20 +09:00
Yusuke Kuoka	759349de11	fix: force restartPolicy "Never" to prevent runner pods from stucking in Terminating when the container disappeared (#1395 ) Ref #1369	2022-05-14 09:07:17 +01:00
renovate[bot]	3014e98681	chore(deps): update helm/chart-releaser-action digest to a3454e4 (#1441 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-13 07:47:19 +09:00
renovate[bot]	5f4be6a883	fix(deps): update module github.com/go-logr/logr to v1.2.3 (#1241 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-13 07:42:58 +09:00
Yusuke Kuoka	b98f470a70	ci: enable CodeQL Alerts following the OpenSSF Security Scorecards recommendation (#1421 ) Ref #1298	2022-05-12 10:55:11 +01:00
Yusuke Kuoka	e46b90f758	fix: runner pods managed by RunnerSet to not stuck in Terminating (#1420 ) This is intended to fix #1369 mostly for RunnerSet-managed runner pods. It is "mostly" because this fix might work well for RunnerDeployment in cases that #1395 does not work, like in a case that the user explicitly set the runner pod restart policy to anything other than "Never". Ref #1369	2022-05-12 09:34:27 +01:00
Yusuke Kuoka	3a7e8c844b	feat: Support arbitrarily setting `privileged: true` for runner container (#1383 ) Resolves #1282	2022-05-12 09:25:51 +01:00
renovate[bot]	65a67ee61c	chore(deps): update docker/setup-qemu-action digest to 0522dcd (#1440 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 11:09:43 +09:00
renovate[bot]	215ba36fd1	chore(deps): update docker/setup-buildx-action digest to 91cb32d (#1439 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 11:08:57 +09:00
renovate[bot]	27774b47bd	fix(deps): update golang.org/x/oauth2 digest to 9780585 (#1329 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 10:50:29 +09:00
renovate[bot]	fbde2b9a41	chore(deps): update docker/login-action digest to d398f07 (#1438 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 10:37:34 +09:00
renovate[bot]	212098183a	chore(deps): update docker/build-push-action digest to c5e6528 (#1437 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 10:37:09 +09:00
renovate[bot]	4a5097d8cf	chore(deps): update actions/setup-go digest to 193b404 (#1431 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 10:36:53 +09:00
renovate[bot]	9c57d085f8	chore(deps): update actions/stale digest to 65d24b7 (#1433 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 09:21:08 +09:00
renovate[bot]	d6622f9369	chore(deps): update actions/setup-python digest to c57f793 (#1432 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 09:20:57 +09:00
Yusuke Kuoka	3b67ee727f	e2e: Fix wrong scale trigger configuration used in test (#1434 )	2022-05-12 09:19:58 +09:00
Yusuke Kuoka	e6bddcd238	Fix certain runnerset name in E2E and acceptance (#1435 )	2022-05-12 09:19:47 +09:00
Callum Tait	f60e57d789	docs: improve troubleshooting (#1428 ) * docs: runner cleanup instructions * docs: add delay in job allocation * docs: fix broken link * docs: reorganise into categories * docs: align the format across the doc * docs: remove code comment * docs: add a tools section * docs: add a short description to each section Co-authored-by: toast-gear <toast-gear@users.noreply.github.com>	2022-05-12 08:45:10 +09:00
renovate[bot]	3ca1152420	chore(deps): update actions/checkout digest to 2541b12 (#1430 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 08:43:35 +09:00
renovate[bot]	e94fa19843	chore(deps): update actions/cache digest to 95f200e (#1429 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-12 08:43:27 +09:00
renovate[bot]	99832d7104	chore(deps): update docker/setup-buildx-action action to v2 (#1416 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-11 16:34:53 +01:00
renovate[bot]	289bcd8b64	chore(deps): update docker/login-action action to v2 (#1415 ) Co-authored-by: Renovate Bot <bot@renovateapp.com>	2022-05-11 16:34:40 +01:00
Jacob Gadikian	5e8cba82c2	docs: simplify wording (#1427 ) clarify docs	2022-05-11 11:44:07 +01:00
Yusuke Kuoka	dabbc99c78	refactor(controller): stop auto-setting RUNNER_FEATURE_FLAG_EPHEMERAL (#1385 ) This feature flag was provided from ARC to runner container automatically to let it use `--ephemeral` instead of `--once` by default. As the support for `--once` is being dropped from the runner image via #1384, we no longer need that. Ref #1196	2022-05-11 11:42:55 +01:00
Yusuke Kuoka	d01595cfbc	ci: pin GitHub Actions workflow actions by hash (#1422 ) as recommended in `5758364c82/docs/checks.md (pinned-dependencies)` Ref #1298	2022-05-11 11:41:30 +01:00
Yusuke Kuoka	c1e5829b03	refactor(runner): ability to opt-out of using --ephemeral / opt-in to legacy --once for GHES older than 3.3 (#1384 ) * runner: Remove the ability to use the deprecated `--once` flag Ref #1196 * runner: Ability to opt-out of using --ephemeral Although we are going to eventually remove the ability to use the legacy --once flag as proposed in #1196, there might be folks still using legacy GHES versions 3.2 or earlier. This commit removes the existing feature flag to opt-in for --ephemeral, while adding another feature flag RUNNER_FEATURE_FLAG_ONCE to opt-in for --once so that folks stuck in legacy GHES versions can still use ARC. Since this change every user starts using --ephemeral by default. If they see any issues on legacy GHES instance, RUNNER_FEATURE_FLAG_ONCE=true can be set to opt-in to keep using --once, which gives one more ARC release until they upgrade their GHES instance. But beware, we won't support legacy GHES instances forever as it's going to be a maintenance nightmare. Please upgrade! Ref #1196	2022-05-11 09:55:33 +01:00
Renovate Bot	800d6bd586	chore(deps): update dependency actions/runner to v2.291.1	2022-04-29 19:05:31 +00:00