Bump chart version to 0.20.0 for ARC 0.25.0 (#1600)

We'll be merging this immediately after ARC 0.25.0 gets released.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -133,6 +133,8 @@ body:
     description: "NEVER EVER OMIT THIS! Include logs from `actions-runner-controller`'s controller-manager pod"
     render: shell
     placeholder: |
+      PROVIDE THE LOGS VIA A GIST LINK (https://gist.github.com/), NOT DIRECTLY IN THIS TEXT AREA
+      
       To grab controller logs:
 
       # Set NS according to your setup
@@ -142,8 +144,6 @@ body:
       kubectl -n $NS get po
 
       kubectl -n $NS logs $POD_NAME > arc.log
-
-      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
   validations:
     required: true
 - type: textarea
@@ -153,6 +153,8 @@ body:
     description: "Include logs from runner pod(s)"
     render: shell
     placeholder: |
+      PROVIDE THE LOGS VIA A GIST LINK (https://gist.github.com/), NOT DIRECTLY IN THIS TEXT AREA
+      
       To grab the runner pod logs:
 
       # Set NS according to your setup. It should match your RunnerDeployment's metadata.namespace.
@@ -163,8 +165,6 @@ body:
 
       kubectl -n $NS logs $POD_NAME -c runner > runnerpod_runner.log
       kubectl -n $NS logs $POD_NAME -c docker > runnerpod_docker.log
-
-      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
   validations:
     required: true
 - type: textarea

.github/actions/setup-docker-environment/action.yaml

@@ -37,12 +37,14 @@ runs:
         version: latest
 
     - name: Login to DockerHub
+      if: ${{ github.event_name == 'release' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
       uses: docker/login-action@v2
       with:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
 
     - name: Login to GitHub Container Registry
+      if: ${{ github.event_name == 'release' || github.event_name == 'push' && github.ref == 'refs/heads/master' }}
       uses: docker/login-action@v2
       with:
         registry: ghcr.io

.github/workflows/publish-arc.yaml

@@ -5,6 +5,11 @@ on:
     types:
       - published
 
+# https://docs.github.com/en/rest/overview/permissions-required-for-github-apps
+permissions:
+ contents: write 
+ packages: write
+
 jobs:
   release-controller:
     name: Release
@@ -58,6 +63,8 @@ jobs:
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
+            ghcr.io/actions-runner-controller/actions-runner-controller:latest
+            ghcr.io/actions-runner-controller/actions-runner-controller:${{ env.VERSION }}
+            ghcr.io/actions-runner-controller/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-

.github/workflows/publish-chart.yaml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.1
+        uses: azure/setup-helm@v3.0
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -73,7 +73,7 @@ jobs:
 
       - name: Create kind cluster
         if: steps.list-changed.outputs.changed == 'true'
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.3.0
 
       # We need cert-manager already installed in the cluster because we assume the CRDs exist
       - name: Install cert-manager

.github/workflows/runners.yaml

@@ -6,7 +6,16 @@ on:
       - opened
       - synchronize
       - reopened
-      - closed
+    branches:
+      - 'master'
+    paths:
+      - 'runner/**'
+      - '!runner/Makefile'
+      - '.github/workflows/runners.yaml'
+      - '!**.md'
+  # We must do a trigger on a push: instead of a types: closed so GitHub Secrets 
+  # are available to the workflow run
+  push:
     branches:
       - 'master'
     paths:
@@ -16,9 +25,10 @@ on:
       - '!**.md'
 
 env:
-  RUNNER_VERSION: 2.293.0
+  RUNNER_VERSION: 2.294.0
   DOCKER_VERSION: 20.10.12
-  DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
+  RUNNER_CONTAINER_HOOKS_VERSION: 0.1.2
+  DOCKERHUB_USERNAME: summerwind
 
 jobs:
   build-runners:
@@ -57,10 +67,11 @@ jobs:
           context: ./runner
           file: ./runner/${{ matrix.name }}.dockerfile
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
           build-args: |
             RUNNER_VERSION=${{ env.RUNNER_VERSION }}
             DOCKER_VERSION=${{ env.DOCKER_VERSION }}
+            RUNNER_CONTAINER_HOOKS_VERSION=${{ env.RUNNER_CONTAINER_HOOKS_VERSION }}
           tags: |
             ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
             ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}

.github/workflows/validate-chart.yaml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.1
+        uses: azure/setup-helm@v3.0
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -67,7 +67,7 @@ jobs:
           ct lint --config charts/.ci/ct-config.yaml
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.2.0
+        uses: helm/kind-action@v1.3.0
         if: steps.list-changed.outputs.changed == 'true'
 
       # We need cert-manager already installed in the cluster because we assume the CRDs exist

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.18.2 as builder
+FROM --platform=$BUILDPLATFORM golang:1.18.3 as builder
 
 WORKDIR /workspace
 

Makefile

@@ -5,7 +5,7 @@ else
 endif
 DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
 VERSION ?= latest
-RUNNER_VERSION ?= 2.293.0
+RUNNER_VERSION ?= 2.294.0
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}

README.md

@@ -30,6 +30,7 @@ ToC:
     - [Autoscaling to/from 0](#autoscaling-tofrom-0)
     - [Scheduled Overrides](#scheduled-overrides)
   - [Runner with DinD](#runner-with-dind)
+  - [Runner with k8s jobs](#runner-with-k8s-jobs)
   - [Additional Tweaks](#additional-tweaks)
   - [Custom Volume mounts](#custom-volume-mounts)
   - [Runner Labels](#runner-labels)
@@ -418,7 +419,7 @@ spec:
         app: example
 ```
 
-As it is based on `StatefulSet`, `selector` and `template.medatada.labels` it needs to be defined and have the exact same set of labels. `serviceName` must be set to some non-empty string as it is also required by `StatefulSet`.
+As it is based on `StatefulSet`, `selector` and `template.metadata.labels` it needs to be defined and have the exact same set of labels. `serviceName` must be set to some non-empty string as it is also required by `StatefulSet`.
 
 Runner-related fields like `ephemeral`, `repository`, `organization`, `enterprise`, and so on should be written directly under `spec`.
 
@@ -718,7 +719,7 @@ With the above example, the webhook server scales `example-runners` by `1` repli
 
 Of note is the `HRA.spec.scaleUpTriggers[].duration` attribute. This attribute is used to calculate if the replica number added via the trigger is expired or not. On each reconciliation loop, the controller sums up all the non-expiring replica numbers from previous scale-up triggers. It then compares the summed desired replica number against the current replica number. If the summed desired replica number > the current number then it means the replica count needs to scale up.
 
-As mentioned previously, the `scaleDownDelaySecondsAfterScaleOut` property has the final say still. If the latest scale-up time + the anti-flapping duration is later than the current time, it doesn’t immediately scale up and instead retries the calculation again later to see if it needs to scale yet.
+As mentioned previously, the `scaleDownDelaySecondsAfterScaleOut` property has the final say still. If the latest scale-up time + the anti-flapping duration is later than the current time, it doesn’t immediately scale down and instead retries the calculation again later to see if it needs to scale yet.
 
 ---
 
@@ -726,6 +727,8 @@ The primary benefit of autoscaling on Webhooks compared to the pull driven scali
 
 > You can learn the implementation details in [#282](https://github.com/actions-runner-controller/actions-runner-controller/pull/282)
 
+##### Install with Helm
+
 To enable this feature, you first need to install the GitHub webhook server. To install via our Helm chart,
 _[see the values documentation for all configuration options](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/charts/actions-runner-controller/README.md)_
 
@@ -839,12 +842,49 @@ alongside your webhook on the Settings -> Webhooks page.
 Once you were able to confirm that the Webhook server is ready and running from GitHub create or update your
 `HorizontalRunnerAutoscaler` resources by learning the following configuration examples.
 
+##### Install with Kustomize
+
+To install this feature using Kustomize, add `github-webhook-server` resources to your `kustomization.yaml` file as in the example below:
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+# You should already have this
+- github.com/actions-runner-controller/actions-runner-controller/config//default?ref=v0.22.2
+# Add the below!
+- github.com/actions-runner-controller/actions-runner-controller/config//github-webhook-server?ref=v0.22.2
+
+Finally, you will have to configure an ingress so that you may configure the webhook in github. An example of such ingress can be find below:
+
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: actions-runners-webhook-server
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /
+        backend:
+          service:
+            name: github-webhook-server
+            port:
+              number: 80
+        pathType: Exact
+
+```
+
+##### Examples
+
 - [Example 1: Scale on each `workflow_job` event](#example-1-scale-on-each-workflow_job-event)
 - [Example 2: Scale up on each `check_run` event](#example-2-scale-up-on-each-check_run-event)
 - [Example 3: Scale on each `pull_request` event against a given set of branches](#example-3-scale-on-each-pull_request-event-against-a-given-set-of-branches)
 - [Example 4: Scale on each `push` event](#example-4-scale-on-each-push-event)
 
-##### Example 1: Scale on each `workflow_job` event
+###### Example 1: Scale on each `workflow_job` event
 
 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
 
@@ -888,7 +928,7 @@ You can configure your GitHub webhook settings to only include `Workflows Job` e
 
 Each kind has a `status` of `queued`, `in_progress` and `completed`. With the above configuration, `actions-runner-controller` adds one runner for a `workflow_job` event whose `status` is `queued`. Similarly, it removes one runner for a `workflow_job` event whose `status` is `completed`. The caveat to this to remember is that this scale-down is within the bounds of your `scaleDownDelaySecondsAfterScaleOut` configuration, if this time hasn't passed the scale down will be deferred.
 
-##### Example 2: Scale up on each `check_run` event
+###### Example 2: Scale up on each `check_run` event
 
 > Note: This should work almost like https://github.com/philips-labs/terraform-aws-github-runner
 
@@ -950,7 +990,7 @@ spec:
     duration: "5m"
 ```
 
-##### Example 3: Scale on each `pull_request` event against a given set of branches
+###### Example 3: Scale on each `pull_request` event against a given set of branches
 
 To scale up replicas of the runners for `example/myrepo` by 1 for 5 minutes on each `pull_request` against the `main` or `develop` branch you write manifests like the below:
 
@@ -1125,6 +1165,36 @@ spec:
 
 This also helps with resources, as you don't need to give resources separately to docker and runner.
 
+### Runner with K8s Jobs
+
+When using the default runner, jobs that use a container will run in docker. This necessitates privileged mode, either on the runner pod or the sidecar container
+
+By setting the container mode, you can instead invoke these jobs using a [kubernetes implementation](https://github.com/actions/runner-container-hooks/tree/main/packages/k8s) while not executing in privileged mode.
+
+The runner will dynamically spin up pods and k8s jobs in the runner's namespace to run the workflow, so a `workVolumeClaimTemplate` is required for the runner's working directory, and a service account with the [appropriate permissions.](https://github.com/actions/runner-container-hooks/tree/main/packages/k8s#pre-requisites)
+
+There are some [limitations](https://github.com/actions/runner-container-hooks/tree/main/packages/k8s#limitations) to this approach, mainly [job containers](https://docs.github.com/en/actions/using-jobs/running-jobs-in-a-container) are required on all workflows.
+
+```yaml
+# runner.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: Runner
+metadata:
+  name: example-runner
+spec:
+  repository: example/myrepo
+  containerMode: kubernetes
+  serviceAccountName: my-service-account
+  workVolumeClaimTemplate:
+    storageClassName: "my-dynamic-storage-class"
+    accessModes:
+    - ReadWriteOnce
+    resources:
+      requests:
+        storage: 10Gi
+  env: []
+```
+
 ### Additional Tweaks
 
 You can pass details through the spec selector. Here's an eg. of what you may like to do:
@@ -1142,6 +1212,7 @@ spec:
       annotations:
         cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
     spec:
+      priorityClassName: "high"
       nodeSelector:
         node-role.kubernetes.io/test: ""
 
@@ -1579,12 +1650,6 @@ spec:
         # Disables automatic runner updates
         - name: DISABLE_RUNNER_UPDATE
           value: "true"
-        # Configure runner with legacy --once instead of --ephemeral flag
-        # WARNING | THIS ENV VAR IS DEPRECATED AND WILL BE REMOVED
-        # THIS ENV VAR WILL BE REMOVED SOON.
-        # SEE ISSUE #1196 FOR DETAILS
-        - name: RUNNER_FEATURE_FLAG_ONCE
-          value: "true"
 ```
 
 ### Using IRSA (IAM Roles for Service Accounts) in EKS

TROUBLESHOOTING.md

@@ -2,8 +2,8 @@
 
 * [Tools](#tools)
 * [Installation](#installation)
+  * [InternalError when calling webhook: context deadline exceeded](#internalerror-when-calling-webhook-context-deadline-exceeded)
   * [Invalid header field value](#invalid-header-field-value)
-  * [Deployment fails on GKE due to webhooks](#deployment-fails-on-gke-due-to-webhooks)
 * [Operations](#operations)
   * [Stuck runner kind or backing pod](#stuck-runner-kind-or-backing-pod)
   * [Delay in jobs being allocated to runners](#delay-in-jobs-being-allocated-to-runners)
@@ -22,39 +22,37 @@ A list of tools which are helpful for troubleshooting
 
 Troubeshooting runbooks that relate to ARC installation problems
 
-### Invalid header field value
+### InternalError when calling webhook: context deadline exceeded
 
 **Problem**
 
-```json
-2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	
-{
-  "controller": "runner",
-  "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9",
-  "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"
-}
+This issue can come up for various reasons like leftovers from previous installations or not being able to access the K8s service's clusterIP associated with the admission webhook server (of ARC).
+
+```
+Internal error occurred: failed calling webhook "mutate.runnerdeployment.actions.summerwind.dev":
+Post "https://actions-runner-controller-webhook.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runnerdeployment?timeout=10s": context deadline exceeded
 ```
 
 **Solution**
 
-Your base64'ed PAT token has a new line at the end, it needs to be created without a `\n` added, either:
-* `echo -n $TOKEN | base64`
-* Create the secret as described in the docs using the shell and documented flags
+First we will try the common solution of checking webhook leftovers from previous installations:
+
+1.  ```bash
+    kubectl get validatingwebhookconfiguration -A
+    kubectl get mutatingwebhookconfiguration -A
+    ```
+2.  If you see any webhooks related to actions-runner-controller, delete them:
+    ```bash
+    kubectl delete mutatingwebhookconfiguration actions-runner-controller-mutating-webhook-configuration
+    kubectl delete validatingwebhookconfiguration actions-runner-controller-validating-webhook-configuration
+    ```
+
+If that didn't work then probably your K8s control-plane is somehow unable to access the K8s service's clusterIP associated with the admission webhook server:
+1. You're running apiserver as a binary and you didn't make service cluster IPs available to the host network. 
+2. You're running the apiserver in the pod but your pod network (i.e. CNI plugin installation and config) is not good so your pods(like kube-apiserver) in the K8s control-plane nodes can't access ARC's admission webhook server pod(s) in probably data-plane nodes.
 
 
-### Deployment fails on GKE due to webhooks
-
-**Problem**
-
-Due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster:
-
-```
-Internal error occurred: failed calling webhook "mutate.runner.actions.summerwind.dev": 
-Post https://webhook-service.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runner?timeout=10s: 
-context deadline exceeded
-```
-
-**Solution**<br />
+Another reason could be due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster: 
 
 To fix this, you may either:
 
@@ -88,6 +86,26 @@ To fix this, you may either:
    gcloud compute firewall-rules create k8s-cert-manager --source-ranges $SOURCE --target-tags $WORKER_NODES_TAG  --allow TCP:9443 --network $NETWORK
    ```
 
+### Invalid header field value
+
+**Problem**
+
+```json
+2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	
+{
+  "controller": "runner",
+  "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9",
+  "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"
+}
+```
+
+**Solution**
+
+Your base64'ed PAT token has a new line at the end, it needs to be created without a `\n` added, either:
+* `echo -n $TOKEN | base64`
+* Create the secret as described in the docs using the shell and documented flags
+
+
 ## Operations
 
 Troubeshooting runbooks that relate to ARC operational problems

acceptance/deploy.sh

@@ -76,56 +76,3 @@ kubectl -n actions-runner-system wait deploy/actions-runner-controller --for con
 
 # Adhocly wait for some time until actions-runner-controller's admission webhook gets ready
 sleep 20
-
-RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
-
-if [ -n "${TEST_REPO}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl apply -f -
-  else
-    echo 'Deploying runnerdeployment and hra. Set USE_RUNNERSET if you want to deploy runnerset instead.'
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerdeploy envsubst | kubectl apply -f -
-  fi
-else
-  echo 'Skipped deploying runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy.'
-fi
-
-if [ -n "${TEST_ORG}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerset envsubst | kubectl apply -f -
-  else
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerdeploy envsubst | kubectl apply -f -
-  fi
-
-  if [ -n "${TEST_ORG_GROUP}" ]; then
-    if [ "${USE_RUNNERSET}" != "false" ]; then
-      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerset envsubst | kubectl apply -f -
-    else
-      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl apply -f -
-    fi
-  else
-    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ORG_GROUP to deploy.'
-  fi
-else
-  echo 'Skipped deploying organizational runnerdeployment. Set TEST_ORG to deploy.'
-fi
-
-if [ -n "${TEST_ENTERPRISE}" ]; then
-  if [ "${USE_RUNNERSET}" != "false" ]; then
-    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerset envsubst | kubectl apply -f -
-  else
-    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerdeploy envsubst | kubectl apply -f -
-  fi
-
-  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
-    if [ "${USE_RUNNERSET}" != "false" ]; then
-      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerset envsubst | kubectl apply -f -
-    else
-      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl apply -f -
-    fi
-  else
-    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to deploy.'
-  fi
-else
-  echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE to deploy.'
-fi

acceptance/deploy_runners.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -e
+
+OP=${OP:-apply}
+
+RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
+
+if [ -n "${TEST_REPO}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl ${OP} -f -
+  else
+    echo "Running ${OP} runnerdeployment and hra. Set USE_RUNNERSET if you want to deploy runnerset instead."
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+else
+  echo "Skipped ${OP} for runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy."
+fi
+
+if [ -n "${TEST_ORG}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerset envsubst | kubectl ${OP} -f -
+  else
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+
+  if [ -n "${TEST_ORG_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerset envsubst | kubectl ${OP} -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl ${OP} -f -
+    fi
+  else
+    echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ORG_GROUP to ${OP}."
+  fi
+else
+  echo "Skipped ${OP} on organizational runnerdeployment. Set TEST_ORG to ${OP}."
+fi
+
+if [ -n "${TEST_ENTERPRISE}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerset envsubst | kubectl ${OP} -f -
+  else
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerdeploy envsubst | kubectl ${OP} -f -
+  fi
+
+  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerset envsubst | kubectl ${OP} -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl ${OP} -f -
+    fi
+  else
+    echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to ${OP}."
+  fi
+else
+  echo "Skipped ${OP} on enterprise runnerdeployment. Set TEST_ENTERPRISE to ${OP}."
+fi

acceptance/testdata/runnerset.envsubst.yaml

@@ -122,8 +122,10 @@ spec:
           value: "/home/runner/.cache/go-mod"
         # PV-backed runner work dir
         volumeMounts:
-        - name: work
-          mountPath: /runner/_work
+        # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
+        # The volume and mount with the same names will be created by workVolumeClaimTemplate and the kubernetes container mode support.
+        # - name: work
+        #   mountPath: /runner/_work
         # Cache docker image layers, in case dockerdWithinRunnerContainer=true
         - name: var-lib-docker
           mountPath: /var/lib/docker
@@ -163,17 +165,18 @@ spec:
         # For buildx cache
         - name: cache
           mountPath: "/home/runner/.cache"
-      volumes:
-      - name: work
-        ephemeral:
-          volumeClaimTemplate:
-            spec:
-              accessModes:
-              - ReadWriteOnce
-              storageClassName: "${NAME}-runner-work-dir"
-              resources:
-                requests:
-                  storage: 10Gi
+        # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
+      # volumes:
+      # - name: work
+      #   ephemeral:
+      #     volumeClaimTemplate:
+      #       spec:
+      #         accessModes:
+      #         - ReadWriteOnce
+      #         storageClassName: "${NAME}-runner-work-dir"
+      #         resources:
+      #           requests:
+      #             storage: 10Gi
   volumeClaimTemplates:
   - metadata:
       name: vol1
@@ -251,3 +254,10 @@ spec:
   minReplicas: ${RUNNER_MIN_REPLICAS}
   maxReplicas: 10
   scaleDownDelaySecondsAfterScaleOut: ${RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT}
+  # Comment out the whole metrics if you'd like to solely test webhook-based scaling
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'
+    scaleDownThreshold: '0.25'
+    scaleUpFactor: '2'
+    scaleDownFactor: '0.5'

api/v1alpha1/runner_types.go

@@ -18,8 +18,10 @@ package v1alpha1
 
 import (
 	"errors"
+	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -71,6 +73,9 @@ type RunnerConfig struct {
 	VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"`
 	// +optional
 	VolumeStorageMedium *string `json:"volumeStorageMedium,omitempty"`
+
+	// +optional
+	ContainerMode string `json:"containerMode,omitempty"`
 }
 
 // RunnerPodSpec defines the desired pod spec fields of the runner pod
@@ -135,6 +140,9 @@ type RunnerPodSpec struct {
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 
+	// +optional
+	PriorityClassName string `json:"priorityClassName,omitempty"`
+
 	// +optional
 	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
 
@@ -154,10 +162,37 @@ type RunnerPodSpec struct {
 
 	// +optional
 	DnsConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"`
+
+	// +optional
+	WorkVolumeClaimTemplate *WorkVolumeClaimTemplate `json:"workVolumeClaimTemplate,omitempty"`
+}
+
+func (rs *RunnerSpec) Validate(rootPath *field.Path) field.ErrorList {
+	var (
+		errList field.ErrorList
+		err     error
+	)
+
+	err = rs.validateRepository()
+	if err != nil {
+		errList = append(errList, field.Invalid(rootPath.Child("repository"), rs.Repository, err.Error()))
+	}
+
+	err = rs.validateWorkVolumeClaimTemplate()
+	if err != nil {
+		errList = append(errList, field.Invalid(rootPath.Child("workVolumeClaimTemplate"), rs.WorkVolumeClaimTemplate, err.Error()))
+	}
+
+	err = rs.validateIsServiceAccountNameSet()
+	if err != nil {
+		errList = append(errList, field.Invalid(rootPath.Child("serviceAccountName"), rs.ServiceAccountName, err.Error()))
+	}
+
+	return errList
 }
 
 // ValidateRepository validates repository field.
-func (rs *RunnerSpec) ValidateRepository() error {
+func (rs *RunnerSpec) validateRepository() error {
 	// Enterprise, Organization and repository are both exclusive.
 	foundCount := 0
 	if len(rs.Organization) > 0 {
@@ -179,6 +214,29 @@ func (rs *RunnerSpec) ValidateRepository() error {
 	return nil
 }
 
+func (rs *RunnerSpec) validateWorkVolumeClaimTemplate() error {
+	if rs.ContainerMode != "kubernetes" {
+		return nil
+	}
+
+	if rs.WorkVolumeClaimTemplate == nil {
+		return errors.New("Spec.ContainerMode: kubernetes must have workVolumeClaimTemplate field specified")
+	}
+
+	return rs.WorkVolumeClaimTemplate.validate()
+}
+
+func (rs *RunnerSpec) validateIsServiceAccountNameSet() error {
+	if rs.ContainerMode != "kubernetes" {
+		return nil
+	}
+
+	if rs.ServiceAccountName == "" {
+		return errors.New("service account name is required if container mode is kubernetes")
+	}
+	return nil
+}
+
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
 	// Turns true only if the runner pod is ready.
@@ -207,6 +265,51 @@ type RunnerStatusRegistration struct {
 	ExpiresAt    metav1.Time `json:"expiresAt"`
 }
 
+type WorkVolumeClaimTemplate struct {
+	StorageClassName string                              `json:"storageClassName"`
+	AccessModes      []corev1.PersistentVolumeAccessMode `json:"accessModes"`
+	Resources        corev1.ResourceRequirements         `json:"resources"`
+}
+
+func (w *WorkVolumeClaimTemplate) validate() error {
+	if w.AccessModes == nil || len(w.AccessModes) == 0 {
+		return errors.New("Access mode should have at least one mode specified")
+	}
+
+	for _, accessMode := range w.AccessModes {
+		switch accessMode {
+		case corev1.ReadWriteOnce, corev1.ReadWriteMany:
+		default:
+			return fmt.Errorf("Access mode %v is not supported", accessMode)
+		}
+	}
+	return nil
+}
+
+func (w *WorkVolumeClaimTemplate) V1Volume() corev1.Volume {
+	return corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes:      w.AccessModes,
+						StorageClassName: &w.StorageClassName,
+						Resources:        w.Resources,
+					},
+				},
+			},
+		},
+	}
+}
+
+func (w *WorkVolumeClaimTemplate) V1VolumeMount(mountPath string) corev1.VolumeMount {
+	return corev1.VolumeMount{
+		MountPath: mountPath,
+		Name:      "work",
+	}
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:JSONPath=".spec.enterprise",name=Enterprise,type=string

api/v1alpha1/runner_webhook.go

@@ -66,15 +66,7 @@ func (r *Runner) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *Runner) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "repository"), r.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Validate(field.NewPath("spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

api/v1alpha1/runnerdeployment_webhook.go

@@ -66,15 +66,7 @@ func (r *RunnerDeployment) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *RunnerDeployment) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.Template.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

api/v1alpha1/runnerreplicaset_webhook.go

@@ -66,15 +66,7 @@ func (r *RunnerReplicaSet) ValidateDelete() error {
 
 // Validate validates resource spec.
 func (r *RunnerReplicaSet) Validate() error {
-	var (
-		errList field.ErrorList
-		err     error
-	)
-
-	err = r.Spec.Template.Spec.ValidateRepository()
-	if err != nil {
-		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
-	}
+	errList := r.Spec.Template.Spec.Validate(field.NewPath("spec", "template", "spec"))
 
 	if len(errList) > 0 {
 		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)

api/v1alpha1/runnerset_types.go

@@ -33,6 +33,12 @@ type RunnerSetSpec struct {
 	// +nullable
 	EffectiveTime *metav1.Time `json:"effectiveTime,omitempty"`
 
+	// +optional
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
+	// +optional
+	WorkVolumeClaimTemplate *WorkVolumeClaimTemplate `json:"workVolumeClaimTemplate,omitempty"`
+
 	appsv1.StatefulSetSpec `json:",inline"`
 }
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -741,6 +741,11 @@ func (in *RunnerPodSpec) DeepCopyInto(out *RunnerPodSpec) {
 		*out = new(v1.PodDNSConfig)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.WorkVolumeClaimTemplate != nil {
+		in, out := &in.WorkVolumeClaimTemplate, &out.WorkVolumeClaimTemplate
+		*out = new(WorkVolumeClaimTemplate)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerPodSpec.
@@ -939,6 +944,11 @@ func (in *RunnerSetSpec) DeepCopyInto(out *RunnerSetSpec) {
 		in, out := &in.EffectiveTime, &out.EffectiveTime
 		*out = (*in).DeepCopy()
 	}
+	if in.WorkVolumeClaimTemplate != nil {
+		in, out := &in.WorkVolumeClaimTemplate, &out.WorkVolumeClaimTemplate
+		*out = new(WorkVolumeClaimTemplate)
+		(*in).DeepCopyInto(*out)
+	}
 	in.StatefulSetSpec.DeepCopyInto(&out.StatefulSetSpec)
 }
 
@@ -1126,6 +1136,27 @@ func (in *ScheduledOverride) DeepCopy() *ScheduledOverride {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WorkVolumeClaimTemplate) DeepCopyInto(out *WorkVolumeClaimTemplate) {
+	*out = *in
+	if in.AccessModes != nil {
+		in, out := &in.AccessModes, &out.AccessModes
+		*out = make([]v1.PersistentVolumeAccessMode, len(*in))
+		copy(*out, *in)
+	}
+	in.Resources.DeepCopyInto(&out.Resources)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkVolumeClaimTemplate.
+func (in *WorkVolumeClaimTemplate) DeepCopy() *WorkVolumeClaimTemplate {
+	if in == nil {
+		return nil
+	}
+	out := new(WorkVolumeClaimTemplate)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WorkflowJobSpec) DeepCopyInto(out *WorkflowJobSpec) {
 	*out = *in

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.19.1
+version: 0.20.0
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.24.1
+appVersion: 0.25.0
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/templates/githubwebhook.ingress.yaml

@@ -1,11 +1,11 @@
 {{- if .Values.githubWebhookServer.ingress.enabled -}}
 {{- $fullName := include "actions-runner-controller-github-webhook-server.fullname" . -}}
 {{- $svcPort := (index .Values.githubWebhookServer.service.ports 0).port -}}
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }}
 apiVersion: networking.k8s.io/v1
-{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
+{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/Ingress" }}
 apiVersion: networking.k8s.io/v1beta1
-{{- else if .Capabilities.APIVersions.Has "extensions/v1beta1" }}
+{{- else if .Capabilities.APIVersions.Has "extensions/v1beta1/Ingress" }}
 apiVersion: extensions/v1beta1
 {{- end }}
 kind: Ingress
@@ -42,11 +42,11 @@ spec:
           {{- end }}
           {{- range .paths }}
           - path: {{ .path }}
-            {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+            {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }}
             pathType: {{ .pathType }}
             {{- end }}
             backend:
-              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }}
               service:
                name: {{ $fullName }}
                port:

charts/actions-runner-controller/templates/githubwebhook.secrets.yaml

@@ -12,5 +12,17 @@ data:
 {{- if .Values.githubWebhookServer.secret.github_webhook_secret_token }}
   github_webhook_secret_token: {{ .Values.githubWebhookServer.secret.github_webhook_secret_token | toString | b64enc }}
 {{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_id }}
+  github_app_id: {{ .Values.githubWebhookServer.secret.github_app_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_installation_id }}
+  github_app_installation_id: {{ .Values.githubWebhookServer.secret.github_app_installation_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_app_private_key }}
+  github_app_private_key: {{ .Values.githubWebhookServer.secret.github_app_private_key | toString | b64enc }}
+{{- end }}
+{{- if .Values.githubWebhookServer.secret.github_token }}
+  github_token: {{ .Values.githubWebhookServer.secret.github_token | toString | b64enc }}
+{{- end }}
 {{- end }}
 {{- end }}

charts/actions-runner-controller/templates/manager_role.yaml

@@ -250,3 +250,11 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch

charts/actions-runner-controller/values.yaml

@@ -109,7 +109,7 @@ metrics:
     enabled: true
     image:
       repository: quay.io/brancz/kube-rbac-proxy
-      tag: v0.12.0
+      tag: v0.13.0
 
 resources:
   {}
@@ -183,6 +183,13 @@ githubWebhookServer:
     name: "github-webhook-server"
     ### GitHub Webhook Configuration
     github_webhook_secret_token: ""
+    ### GitHub Apps Configuration
+    ## NOTE: IDs MUST be strings, use quotes
+    #github_app_id: ""
+    #github_app_installation_id: ""
+    #github_app_private_key: |
+    ### GitHub PAT Configuration
+    #github_token: ""
   imagePullSecrets: []
   nameOverride: ""
   fullnameOverride: ""

cmd/githubwebhookserver/main.go

@@ -72,6 +72,7 @@ func main() {
 		enableLeaderElection bool
 		syncPeriod           time.Duration
 		logLevel             string
+		queueLimit           int
 
 		ghClient *github.Client
 	)
@@ -92,6 +93,7 @@ func main() {
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
+	flag.IntVar(&queueLimit, "queue-limit", controllers.DefaultQueueLimit, `The maximum length of the scale operation queue. The scale opration is enqueued per every matching webhook event, and the server returns a 500 HTTP status when the queue was already full on enqueue attempt.`)
 	flag.StringVar(&webhookSecretToken, "github-webhook-secret-token", "", "The personal access token of GitHub.")
 	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
 	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
@@ -164,6 +166,7 @@ func main() {
 		SecretKeyBytes: []byte(webhookSecretToken),
 		Namespace:      watchNamespace,
 		GitHubClient:   ghClient,
+		QueueLimit:     queueLimit,
 	}
 
 	if err = hraGitHubWebhook.SetupWithManager(mgr); err != nil {

config/default/kustomization.yaml

@@ -22,8 +22,6 @@ bases:
 - ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
-# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
-#- ../github-webhook-server
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.
@@ -46,10 +44,6 @@ patchesStrategicMerge:
 # 'CERTMANAGER' needs to be enabled to use ca injection
 - webhookcainjection_patch.yaml
 
-# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
-# Protect the GitHub webhook server metrics endpoint by putting it behind auth.
-# - gh-webhook-server-auth-proxy-patch.yaml
-
 # the following config is for teaching kustomize how to do var substitution
 vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.

config/github-webhook-server/kustomization.yaml

@@ -2,11 +2,14 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 images:
-  - name: controller
-    newName: summerwind/actions-runner-controller
-    newTag: latest
+- name: controller
+  newName: summerwind/actions-runner-controller
+  newTag: latest
 
 resources:
-  - deployment.yaml
-  - rbac.yaml
-  - service.yaml
+- deployment.yaml
+- rbac.yaml
+- service.yaml
+
+patchesStrategicMerge:
+- gh-webhook-server-auth-proxy-patch.yaml

config/rbac/role.yaml

@@ -249,3 +249,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - delete
+  - get
+  - list
+  - watch

controllers/autoscaling.go

@@ -9,7 +9,9 @@ import (
 	"strings"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -314,22 +316,52 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 		numRunners           int
 		numRunnersRegistered int
 		numRunnersBusy       int
+		numTerminatingBusy   int
 	)
 
 	numRunners = len(runnerMap)
 
+	busyTerminatingRunnerPods := map[string]struct{}{}
+
+	kindLabel := LabelKeyRunnerDeploymentName
+	if hra.Spec.ScaleTargetRef.Kind == "RunnerSet" {
+		kindLabel = LabelKeyRunnerSetName
+	}
+
+	var runnerPodList corev1.PodList
+	if err := r.Client.List(ctx, &runnerPodList, client.InNamespace(hra.Namespace), client.MatchingLabels(map[string]string{
+		kindLabel: hra.Spec.ScaleTargetRef.Name,
+	})); err != nil {
+		return nil, err
+	}
+
+	for _, p := range runnerPodList.Items {
+		if p.Annotations[AnnotationKeyUnregistrationFailureMessage] != "" {
+			busyTerminatingRunnerPods[p.Name] = struct{}{}
+		}
+	}
+
 	for _, runner := range runners {
 		if _, ok := runnerMap[*runner.Name]; ok {
 			numRunnersRegistered++
 
 			if runner.GetBusy() {
 				numRunnersBusy++
+			} else if _, ok := busyTerminatingRunnerPods[*runner.Name]; ok {
+				numTerminatingBusy++
 			}
+
+			delete(busyTerminatingRunnerPods, *runner.Name)
 		}
 	}
 
+	// Remaining busyTerminatingRunnerPods are runners that were not on the ListRunners API response yet
+	for range busyTerminatingRunnerPods {
+		numTerminatingBusy++
+	}
+
 	var desiredReplicas int
-	fractionBusy := float64(numRunnersBusy) / float64(desiredReplicasBefore)
+	fractionBusy := float64(numRunnersBusy+numTerminatingBusy) / float64(desiredReplicasBefore)
 	if fractionBusy >= scaleUpThreshold {
 		if scaleUpAdjustment > 0 {
 			desiredReplicas = desiredReplicasBefore + scaleUpAdjustment
@@ -358,6 +390,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 		"num_runners", numRunners,
 		"num_runners_registered", numRunnersRegistered,
 		"num_runners_busy", numRunnersBusy,
+		"num_terminating_busy", numTerminatingBusy,
 		"namespace", hra.Namespace,
 		"kind", st.kind,
 		"name", st.st,

controllers/constants.go

@@ -4,17 +4,22 @@ import "time"
 
 const (
 	LabelKeyRunnerSetName = "runnerset-name"
+	LabelKeyRunner        = "actions-runner"
 )
 
 const (
 	// This names requires at least one slash to work.
 	// See https://github.com/google/knative-gcp/issues/378
-	runnerPodFinalizerName = "actions.summerwind.dev/runner-pod"
+	runnerPodFinalizerName             = "actions.summerwind.dev/runner-pod"
+	runnerLinkedResourcesFinalizerName = "actions.summerwind.dev/linked-resources"
 
 	annotationKeyPrefix = "actions-runner/"
 
 	AnnotationKeyLastRegistrationCheckTime = "actions-runner-controller/last-registration-check-time"
 
+	// AnnotationKeyUnregistrationFailureMessage is the annotation that is added onto the pod once it failed to be unregistered from GitHub due to e.g. 422 error
+	AnnotationKeyUnregistrationFailureMessage = annotationKeyPrefix + "unregistration-failure-message"
+
 	// AnnotationKeyUnregistrationCompleteTimestamp is the annotation that is added onto the pod once the previously started unregistration process has been completed.
 	AnnotationKeyUnregistrationCompleteTimestamp = annotationKeyPrefix + "unregistration-complete-timestamp"
 
@@ -61,4 +66,7 @@ const (
 
 	EnvVarRunnerName  = "RUNNER_NAME"
 	EnvVarRunnerToken = "RUNNER_TOKEN"
+
+	// defaultHookPath is path to the hook script used when the "containerMode: kubernetes" is specified
+	defaultRunnerHookPath = "/runner/k8s/index.js"
 )

controllers/horizontal_runner_autoscaler_batch_scale.go

@@ -0,0 +1,207 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/go-logr/logr"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type batchScaler struct {
+	Ctx      context.Context
+	Client   client.Client
+	Log      logr.Logger
+	interval time.Duration
+
+	queue       chan *ScaleTarget
+	workerStart sync.Once
+}
+
+func newBatchScaler(ctx context.Context, client client.Client, log logr.Logger) *batchScaler {
+	return &batchScaler{
+		Ctx:      ctx,
+		Client:   client,
+		Log:      log,
+		interval: 3 * time.Second,
+	}
+}
+
+type batchScaleOperation struct {
+	namespacedName types.NamespacedName
+	scaleOps       []scaleOperation
+}
+
+type scaleOperation struct {
+	trigger v1alpha1.ScaleUpTrigger
+	log     logr.Logger
+}
+
+// Add the scale target to the unbounded queue, blocking until the target is successfully added to the queue.
+// All the targets in the queue are dequeued every 3 seconds, grouped by the HRA, and applied.
+// In a happy path, batchScaler update each HRA only once, even though the HRA had two or more associated webhook events in the 3 seconds interval,
+// which results in less K8s API calls and less HRA update conflicts in case your ARC installation receives a lot of webhook events
+func (s *batchScaler) Add(st *ScaleTarget) {
+	if st == nil {
+		return
+	}
+
+	s.workerStart.Do(func() {
+		var expBackoff = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second}
+
+		s.queue = make(chan *ScaleTarget)
+
+		log := s.Log
+
+		go func() {
+			log.Info("Starting batch worker")
+			defer log.Info("Stopped batch worker")
+
+			for {
+				select {
+				case <-s.Ctx.Done():
+					return
+				default:
+				}
+
+				log.V(2).Info("Batch worker is dequeueing operations")
+
+				batches := map[types.NamespacedName]batchScaleOperation{}
+				after := time.After(s.interval)
+				var ops uint
+
+			batch:
+				for {
+					select {
+					case <-after:
+						after = nil
+						break batch
+					case st := <-s.queue:
+						nsName := types.NamespacedName{
+							Namespace: st.HorizontalRunnerAutoscaler.Namespace,
+							Name:      st.HorizontalRunnerAutoscaler.Name,
+						}
+						b, ok := batches[nsName]
+						if !ok {
+							b = batchScaleOperation{
+								namespacedName: nsName,
+							}
+						}
+						b.scaleOps = append(b.scaleOps, scaleOperation{
+							log:     *st.log,
+							trigger: st.ScaleUpTrigger,
+						})
+						batches[nsName] = b
+						ops++
+					}
+				}
+
+				log.V(2).Info("Batch worker dequeued operations", "ops", ops, "batches", len(batches))
+
+			retry:
+				for i := 0; ; i++ {
+					failed := map[types.NamespacedName]batchScaleOperation{}
+
+					for nsName, b := range batches {
+						b := b
+						if err := s.batchScale(context.Background(), b); err != nil {
+							log.V(2).Info("Failed to scale due to error", "error", err)
+							failed[nsName] = b
+						} else {
+							log.V(2).Info("Successfully ran batch scale", "hra", b.namespacedName)
+						}
+					}
+
+					if len(failed) == 0 {
+						break retry
+					}
+
+					batches = failed
+
+					delay := 16 * time.Second
+					if i < len(expBackoff) {
+						delay = expBackoff[i]
+					}
+					time.Sleep(delay)
+				}
+			}
+		}()
+	})
+
+	s.queue <- st
+}
+
+func (s *batchScaler) batchScale(ctx context.Context, batch batchScaleOperation) error {
+	var hra v1alpha1.HorizontalRunnerAutoscaler
+
+	if err := s.Client.Get(ctx, batch.namespacedName, &hra); err != nil {
+		return err
+	}
+
+	copy := hra.DeepCopy()
+
+	copy.Spec.CapacityReservations = getValidCapacityReservations(copy)
+
+	var added, completed int
+
+	for _, scale := range batch.scaleOps {
+		amount := 1
+
+		if scale.trigger.Amount != 0 {
+			amount = scale.trigger.Amount
+		}
+
+		scale.log.V(2).Info("Adding capacity reservation", "amount", amount)
+
+		if amount > 0 {
+			now := time.Now()
+			copy.Spec.CapacityReservations = append(copy.Spec.CapacityReservations, v1alpha1.CapacityReservation{
+				EffectiveTime:  metav1.Time{Time: now},
+				ExpirationTime: metav1.Time{Time: now.Add(scale.trigger.Duration.Duration)},
+				Replicas:       amount,
+			})
+
+			added += amount
+		} else if amount < 0 {
+			var reservations []v1alpha1.CapacityReservation
+
+			var found bool
+
+			for _, r := range copy.Spec.CapacityReservations {
+				if !found && r.Replicas+amount == 0 {
+					found = true
+				} else {
+					reservations = append(reservations, r)
+				}
+			}
+
+			copy.Spec.CapacityReservations = reservations
+
+			completed += amount
+		}
+	}
+
+	before := len(hra.Spec.CapacityReservations)
+	expired := before - len(copy.Spec.CapacityReservations)
+	after := len(copy.Spec.CapacityReservations)
+
+	s.Log.V(1).Info(
+		fmt.Sprintf("Updating hra %s for capacityReservations update", hra.Name),
+		"before", before,
+		"expired", expired,
+		"added", added,
+		"completed", completed,
+		"after", after,
+	)
+
+	if err := s.Client.Update(ctx, copy); err != nil {
+		return fmt.Errorf("updating horizontalrunnerautoscaler to add capacity reservation: %w", err)
+	}
+
+	return nil
+}

controllers/horizontal_runner_autoscaler_webhook.go

@@ -23,14 +23,14 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/go-logr/logr"
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v45/github"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -46,6 +46,8 @@ const (
 
 	keyPrefixEnterprise = "enterprises/"
 	keyRunnerGroup      = "/group/"
+
+	DefaultQueueLimit = 100
 )
 
 // HorizontalRunnerAutoscalerGitHubWebhook autoscales a HorizontalRunnerAutoscaler and the RunnerDeployment on each
@@ -68,6 +70,15 @@ type HorizontalRunnerAutoscalerGitHubWebhook struct {
 	// Set to empty for letting it watch for all namespaces.
 	Namespace string
 	Name      string
+
+	// QueueLimit is the maximum length of the bounded queue of scale targets and their associated operations
+	// A scale target is enqueued on each retrieval of each eligible webhook event, so that it is processed asynchronously.
+	QueueLimit int
+
+	worker      *worker
+	workerInit  sync.Once
+	workerStart sync.Once
+	batchCh     chan *ScaleTarget
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) {
@@ -312,9 +323,19 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.Respons
 		return
 	}
 
-	if err := autoscaler.tryScale(context.TODO(), target); err != nil {
-		log.Error(err, "could not scale up")
+	autoscaler.workerInit.Do(func() {
+		batchScaler := newBatchScaler(context.Background(), autoscaler.Client, autoscaler.Log)
 
+		queueLimit := autoscaler.QueueLimit
+		if queueLimit == 0 {
+			queueLimit = DefaultQueueLimit
+		}
+		autoscaler.worker = newWorker(context.Background(), queueLimit, batchScaler.Add)
+	})
+
+	target.log = &log
+	if ok := autoscaler.worker.Add(target); !ok {
+		log.Error(err, "Could not scale up due to queue full")
 		return
 	}
 
@@ -383,6 +404,8 @@ func matchTriggerConditionAgainstEvent(types []string, eventAction *string) bool
 type ScaleTarget struct {
 	v1alpha1.HorizontalRunnerAutoscaler
 	v1alpha1.ScaleUpTrigger
+
+	log *logr.Logger
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) searchScaleTargets(hras []v1alpha1.HorizontalRunnerAutoscaler, f func(v1alpha1.ScaleUpTrigger) bool) []ScaleTarget {
@@ -501,6 +524,7 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTargetWithF
 	if autoscaler.GitHubClient != nil {
 		simu := &simulator.Simulator{
 			Client: autoscaler.GitHubClient,
+			Log:    log,
 		}
 		// Get available organization runner groups and enterprise runner groups for a repository
 		// These are the sum of runner groups with repository access = All repositories and runner groups
@@ -770,63 +794,6 @@ HRA:
 	return nil, nil
 }
 
-func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScale(ctx context.Context, target *ScaleTarget) error {
-	if target == nil {
-		return nil
-	}
-
-	copy := target.HorizontalRunnerAutoscaler.DeepCopy()
-
-	amount := 1
-
-	if target.ScaleUpTrigger.Amount != 0 {
-		amount = target.ScaleUpTrigger.Amount
-	}
-
-	capacityReservations := getValidCapacityReservations(copy)
-
-	if amount > 0 {
-		now := time.Now()
-		copy.Spec.CapacityReservations = append(capacityReservations, v1alpha1.CapacityReservation{
-			EffectiveTime:  metav1.Time{Time: now},
-			ExpirationTime: metav1.Time{Time: now.Add(target.ScaleUpTrigger.Duration.Duration)},
-			Replicas:       amount,
-		})
-	} else if amount < 0 {
-		var reservations []v1alpha1.CapacityReservation
-
-		var found bool
-
-		for _, r := range capacityReservations {
-			if !found && r.Replicas+amount == 0 {
-				found = true
-			} else {
-				reservations = append(reservations, r)
-			}
-		}
-
-		copy.Spec.CapacityReservations = reservations
-	}
-
-	before := len(target.HorizontalRunnerAutoscaler.Spec.CapacityReservations)
-	expired := before - len(capacityReservations)
-	after := len(copy.Spec.CapacityReservations)
-
-	autoscaler.Log.V(1).Info(
-		fmt.Sprintf("Patching hra %s for capacityReservations update", target.HorizontalRunnerAutoscaler.Name),
-		"before", before,
-		"expired", expired,
-		"amount", amount,
-		"after", after,
-	)
-
-	if err := autoscaler.Client.Patch(ctx, copy, client.MergeFrom(&target.HorizontalRunnerAutoscaler)); err != nil {
-		return fmt.Errorf("patching horizontalrunnerautoscaler to add capacity reservation: %w", err)
-	}
-
-	return nil
-}
-
 func getValidCapacityReservations(autoscaler *v1alpha1.HorizontalRunnerAutoscaler) []v1alpha1.CapacityReservation {
 	var capacityReservations []v1alpha1.CapacityReservation
 

controllers/horizontal_runner_autoscaler_webhook_on_check_run.go

@@ -3,7 +3,7 @@ package controllers
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/pkg/actionsglob"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchCheckRunEvent(event *github.CheckRunEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPullRequestEvent(event *github.PullRequestEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_push.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event *github.PushEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_test.go

@@ -15,7 +15,7 @@ import (
 
 	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"

controllers/horizontal_runner_autoscaler_webhook_worker.go

@@ -0,0 +1,55 @@
+package controllers
+
+import (
+	"context"
+)
+
+// worker is a worker that has a non-blocking bounded queue of scale targets, dequeues scale target and executes the scale operation one by one.
+type worker struct {
+	scaleTargetQueue chan *ScaleTarget
+	work             func(*ScaleTarget)
+	done             chan struct{}
+}
+
+func newWorker(ctx context.Context, queueLimit int, work func(*ScaleTarget)) *worker {
+	w := &worker{
+		scaleTargetQueue: make(chan *ScaleTarget, queueLimit),
+		work:             work,
+		done:             make(chan struct{}),
+	}
+
+	go func() {
+		defer close(w.done)
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case t := <-w.scaleTargetQueue:
+				work(t)
+			}
+		}
+	}()
+
+	return w
+}
+
+// Add the scale target to the bounded queue, returning the result as a bool value. It returns true on successful enqueue, and returns false otherwise.
+// When returned false, the queue is already full so the enqueue operation must be retried later.
+// If the enqueue was triggered by an external source and there's no intermediate queue that we can use,
+// you must instruct the source to resend the original request later.
+// In case you're building a webhook server around this worker, this means that you must return a http error to the webhook server,
+// so that (hopefully) the sender can resend the webhook event later, or at least the human operator can notice or be notified about the
+// webhook develiery failure so that a manual retry can be done later.
+func (w *worker) Add(st *ScaleTarget) bool {
+	select {
+	case w.scaleTargetQueue <- st:
+		return true
+	default:
+		return false
+	}
+}
+
+func (w *worker) Done() chan struct{} {
+	return w.done
+}

controllers/horizontal_runner_autoscaler_webhook_worker_test.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWorker_Add(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	w := newWorker(ctx, 2, func(st *ScaleTarget) {})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+}
+
+func TestWorker_Work(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var count int
+
+	w := newWorker(ctx, 1, func(st *ScaleTarget) {
+		count++
+		cancel()
+	})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+
+	<-w.Done()
+
+	require.Equal(t, count, 1)
+}

controllers/integration_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	github2 "github.com/actions-runner-controller/actions-runner-controller/github"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
 
@@ -1367,7 +1367,7 @@ func (env *testEnvironment) ExpectRegisteredNumberCountEventuallyEquals(want int
 
 			return len(rs)
 		},
-		time.Second*5, time.Millisecond*500).Should(Equal(want), optionalDescriptions...)
+		time.Second*10, time.Millisecond*500).Should(Equal(want), optionalDescriptions...)
 }
 
 func (env *testEnvironment) SendOrgPullRequestEvent(org, repo, branch, action string) {

controllers/new_runner_pod_test.go

@@ -56,7 +56,7 @@ func TestNewRunnerPod(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
-				"runnerset-name": "runner",
+				"actions-runner": "",
 			},
 		},
 		Spec: corev1.PodSpec{
@@ -198,7 +198,7 @@ func TestNewRunnerPod(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
-				"runnerset-name": "runner",
+				"actions-runner": "",
 			},
 		},
 		Spec: corev1.PodSpec{
@@ -276,7 +276,7 @@ func TestNewRunnerPod(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
-				"runnerset-name": "runner",
+				"actions-runner": "",
 			},
 		},
 		Spec: corev1.PodSpec{
@@ -515,7 +515,7 @@ func TestNewRunnerPod(t *testing.T) {
 	for i := range testcases {
 		tc := testcases[i]
 		t.Run(tc.description, func(t *testing.T) {
-			got, err := newRunnerPod("runner", tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
+			got, err := newRunnerPod(tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
 			require.NoError(t, err)
 			require.Equal(t, tc.want, got)
 		})
@@ -546,7 +546,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
 				"pod-template-hash": "8857b86c7",
-				"runnerset-name":    "runner",
+				"actions-runner":    "",
 			},
 			OwnerReferences: []metav1.OwnerReference{
 				{
@@ -703,7 +703,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
 				"pod-template-hash": "8857b86c7",
-				"runnerset-name":    "runner",
+				"actions-runner":    "",
 			},
 			OwnerReferences: []metav1.OwnerReference{
 				{
@@ -800,7 +800,7 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 			Labels: map[string]string{
 				"actions-runner-controller/inject-registration-token": "true",
 				"pod-template-hash": "8857b86c7",
-				"runnerset-name":    "runner",
+				"actions-runner":    "",
 			},
 			OwnerReferences: []metav1.OwnerReference{
 				{

controllers/runner_controller.go

@@ -18,7 +18,9 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
@@ -76,6 +78,7 @@ type RunnerReconciler struct {
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;delete
 // +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
@@ -112,6 +115,7 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			// Pod was not found
 			return r.processRunnerDeletion(runner, ctx, log, nil)
 		}
+
 		return r.processRunnerDeletion(runner, ctx, log, &pod)
 	}
 
@@ -412,7 +416,17 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	template.Spec.SecurityContext = runner.Spec.SecurityContext
 	template.Spec.EnableServiceLinks = runner.Spec.EnableServiceLinks
 
-	pod, err := newRunnerPod(runner.Name, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL)
+	if runner.Spec.ContainerMode == "kubernetes" {
+		workDir := runner.Spec.WorkDir
+		if workDir == "" {
+			workDir = "/runner/_work"
+		}
+		if err := applyWorkVolumeClaimTemplateToPod(&template, runner.Spec.WorkVolumeClaimTemplate, workDir); err != nil {
+			return corev1.Pod{}, err
+		}
+	}
+
+	pod, err := newRunnerPodWithContainerMode(runner.Spec.ContainerMode, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL)
 	if err != nil {
 		return pod, err
 	}
@@ -424,6 +438,9 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		// if operater provides a work volume mount, use that
 		isPresent, _ := workVolumeMountPresent(runnerSpec.VolumeMounts)
 		if isPresent {
+			if runnerSpec.ContainerMode == "kubernetes" {
+				return pod, errors.New("volume mount \"work\" should be specified by workVolumeClaimTemplate in container mode kubernetes")
+			}
 			// remove work volume since it will be provided from runnerSpec.Volumes
 			// if we don't remove it here we would get a duplicate key error, i.e. two volumes named work
 			_, index := workVolumeMountPresent(pod.Spec.Containers[0].VolumeMounts)
@@ -437,6 +454,9 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		// if operator provides a work volume. use that
 		isPresent, _ := workVolumePresent(runnerSpec.Volumes)
 		if isPresent {
+			if runnerSpec.ContainerMode == "kubernetes" {
+				return pod, errors.New("volume \"work\" should be specified by workVolumeClaimTemplate in container mode kubernetes")
+			}
 			_, index := workVolumePresent(pod.Spec.Volumes)
 
 			// remove work volume since it will be provided from runnerSpec.Volumes
@@ -446,6 +466,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 
 		pod.Spec.Volumes = append(pod.Spec.Volumes, runnerSpec.Volumes...)
 	}
+
 	if len(runnerSpec.InitContainers) != 0 {
 		pod.Spec.InitContainers = append(pod.Spec.InitContainers, runnerSpec.InitContainers...)
 	}
@@ -476,6 +497,10 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		pod.Spec.Tolerations = runnerSpec.Tolerations
 	}
 
+	if runnerSpec.PriorityClassName != "" {
+		pod.Spec.PriorityClassName = runnerSpec.PriorityClassName
+	}
+
 	if len(runnerSpec.TopologySpreadConstraints) != 0 {
 		pod.Spec.TopologySpreadConstraints = runnerSpec.TopologySpreadConstraints
 	}
@@ -526,7 +551,45 @@ func mutatePod(pod *corev1.Pod, token string) *corev1.Pod {
 	return updated
 }
 
-func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
+func runnerHookEnvs(pod *corev1.Pod) ([]corev1.EnvVar, error) {
+	isRequireSameNode, err := isRequireSameNode(pod)
+	if err != nil {
+		return nil, err
+	}
+
+	return []corev1.EnvVar{
+		{
+			Name:  "ACTIONS_RUNNER_CONTAINER_HOOKS",
+			Value: defaultRunnerHookPath,
+		},
+		{
+			Name:  "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER",
+			Value: "true",
+		},
+		{
+			Name: "ACTIONS_RUNNER_POD_NAME",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "metadata.name",
+				},
+			},
+		},
+		{
+			Name: "ACTIONS_RUNNER_JOB_NAMESPACE",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "metadata.namespace",
+				},
+			},
+		},
+		corev1.EnvVar{
+			Name:  "ACTIONS_RUNNER_REQUIRE_SAME_NODE",
+			Value: strconv.FormatBool(isRequireSameNode),
+		},
+	}, nil
+}
+
+func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
 	var (
 		privileged                bool = true
 		dockerdInRunner           bool = runnerSpec.DockerdWithinRunnerContainer != nil && *runnerSpec.DockerdWithinRunnerContainer
@@ -535,10 +598,16 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 		dockerdInRunnerPrivileged bool = dockerdInRunner
 	)
 
+	if containerMode == "kubernetes" {
+		dockerdInRunner = false
+		dockerEnabled = false
+		dockerdInRunnerPrivileged = false
+	}
+
 	template = *template.DeepCopy()
 
 	// This label selector is used by default when rd.Spec.Selector is empty.
-	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerName)
+	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunner, "")
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyPodMutation, LabelValuePodMutation)
 
 	workDir := runnerSpec.WorkDir
@@ -621,6 +690,17 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 		}
 	}
 
+	if containerMode == "kubernetes" {
+		if dockerdContainer != nil {
+			template.Spec.Containers = append(template.Spec.Containers[:dockerdContainerIndex], template.Spec.Containers[dockerdContainerIndex+1:]...)
+		}
+		if runnerContainerIndex < runnerContainerIndex {
+			runnerContainerIndex--
+		}
+		dockerdContainer = nil
+		dockerdContainerIndex = -1
+	}
+
 	if runnerContainer == nil {
 		runnerContainerIndex = -1
 		runnerContainer = &corev1.Container{
@@ -651,6 +731,13 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 	}
 
 	runnerContainer.Env = append(runnerContainer.Env, env...)
+	if containerMode == "kubernetes" {
+		hookEnvs, err := runnerHookEnvs(&template)
+		if err != nil {
+			return corev1.Pod{}, err
+		}
+		runnerContainer.Env = append(runnerContainer.Env, hookEnvs...)
+	}
 
 	if runnerContainer.SecurityContext == nil {
 		runnerContainer.SecurityContext = &corev1.SecurityContext{}
@@ -875,6 +962,10 @@ func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.Ru
 	return *pod, nil
 }
 
+func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
+	return newRunnerPodWithContainerMode("", template, runnerSpec, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
+}
+
 func (r *RunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	name := "runner-controller"
 	if r.Name != "" {
@@ -937,3 +1028,71 @@ func workVolumeMountPresent(items []corev1.VolumeMount) (bool, int) {
 	}
 	return false, 0
 }
+
+func applyWorkVolumeClaimTemplateToPod(pod *corev1.Pod, workVolumeClaimTemplate *v1alpha1.WorkVolumeClaimTemplate, workDir string) error {
+	if workVolumeClaimTemplate == nil {
+		return errors.New("work volume claim template must be specified in container mode kubernetes")
+	}
+	for i := range pod.Spec.Volumes {
+		if pod.Spec.Volumes[i].Name == "work" {
+			return fmt.Errorf("Work volume should not be specified in container mode kubernetes. workVolumeClaimTemplate field should be used instead.")
+		}
+	}
+	pod.Spec.Volumes = append(pod.Spec.Volumes, workVolumeClaimTemplate.V1Volume())
+
+	var runnerContainer *corev1.Container
+	for i := range pod.Spec.Containers {
+		if pod.Spec.Containers[i].Name == "runner" {
+			runnerContainer = &pod.Spec.Containers[i]
+			break
+		}
+	}
+
+	if runnerContainer == nil {
+		return fmt.Errorf("runner container is not present when applying work volume claim template")
+	}
+
+	if isPresent, _ := workVolumeMountPresent(runnerContainer.VolumeMounts); isPresent {
+		return fmt.Errorf("volume mount \"work\" should not be present on the runner container in container mode kubernetes")
+	}
+
+	runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts, workVolumeClaimTemplate.V1VolumeMount(workDir))
+
+	return nil
+}
+
+// isRequireSameNode specifies for the runner in kubernetes mode wether it should
+// schedule jobs to the same node where the runner is
+//
+// This function should only be called in containerMode: kubernetes
+func isRequireSameNode(pod *corev1.Pod) (bool, error) {
+	isPresent, index := workVolumePresent(pod.Spec.Volumes)
+	if !isPresent {
+		return true, errors.New("internal error: work volume mount must exist in containerMode: kubernetes")
+	}
+
+	if pod.Spec.Volumes[index].Ephemeral == nil || pod.Spec.Volumes[index].Ephemeral.VolumeClaimTemplate == nil {
+		return true, errors.New("containerMode: kubernetes should have pod.Spec.Volumes[].Ephemeral.VolumeClaimTemplate set")
+	}
+
+	for _, accessMode := range pod.Spec.Volumes[index].Ephemeral.VolumeClaimTemplate.Spec.AccessModes {
+		switch accessMode {
+		case corev1.ReadWriteOnce:
+			return true, nil
+		case corev1.ReadWriteMany:
+		default:
+			return true, errors.New("actions-runner-controller supports ReadWriteOnce and ReadWriteMany modes only")
+		}
+	}
+	return false, nil
+}
+
+func overwriteRunnerEnv(runner *v1alpha1.Runner, key string, value string) {
+	for i := range runner.Spec.Env {
+		if runner.Spec.Env[i].Name == key {
+			runner.Spec.Env[i].Value = value
+			return
+		}
+	}
+	runner.Spec.Env = append(runner.Spec.Env, corev1.EnvVar{Name: key, Value: value})
+}

controllers/runner_graceful_stop.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v45/github"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -67,6 +67,25 @@ func annotatePodOnce(ctx context.Context, c client.Client, log logr.Logger, pod
 
 	return updated, nil
 }
+func labelPod(ctx context.Context, c client.Client, log logr.Logger, pod *corev1.Pod, k, v string) (*corev1.Pod, error) {
+	if pod == nil {
+		return nil, nil
+	}
+
+	updated := pod.DeepCopy()
+	if updated.Labels == nil {
+		updated.Labels = map[string]string{}
+	}
+	updated.Labels[k] = v
+	if err := c.Patch(ctx, updated, client.MergeFrom(pod)); err != nil {
+		log.Error(err, fmt.Sprintf("Failed to patch pod to have %s annotation", k))
+		return nil, err
+	}
+
+	log.V(2).Info("Labeled pod", "key", k, "value", v)
+
+	return updated, nil
+}
 
 // If the first return value is nil, it's safe to delete the runner pod.
 func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*ctrl.Result, error) {
@@ -151,7 +170,10 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 
 		log.V(1).Info("Failed to unregister runner before deleting the pod.", "error", err)
 
-		var runnerBusy bool
+		var (
+			runnerBusy                         bool
+			runnerUnregistrationFailureMessage string
+		)
 
 		errRes := &gogithub.ErrorResponse{}
 		if errors.As(err, &errRes) {
@@ -173,6 +195,7 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 			}
 
 			runnerBusy = errRes.Response.StatusCode == 422
+			runnerUnregistrationFailureMessage = errRes.Message
 
 			if runnerBusy && code != nil {
 				log.V(2).Info("Runner container has already stopped but the unregistration attempt failed. "+
@@ -187,6 +210,11 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 		}
 
 		if runnerBusy {
+			_, err := labelPod(ctx, c, log, pod, AnnotationKeyUnregistrationFailureMessage, runnerUnregistrationFailureMessage)
+			if err != nil {
+				return &ctrl.Result{}, err
+			}
+
 			// We want to prevent spamming the deletion attemps but returning ctrl.Result with RequeueAfter doesn't
 			// work as the reconcilation can happen earlier due to pod status update.
 			// For ephemeral runners, we can expect it to stop and unregister itself on completion.

controllers/runner_pod_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -50,6 +51,7 @@ type RunnerPodReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -60,8 +62,11 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	_, isRunnerPod := runnerPod.Labels[LabelKeyRunnerSetName]
-	if !isRunnerPod {
+	_, isRunnerPod := runnerPod.Labels[LabelKeyRunner]
+	_, isRunnerSetPod := runnerPod.Labels[LabelKeyRunnerSetName]
+	_, isRunnerDeploymentPod := runnerPod.Labels[LabelKeyRunnerDeploymentName]
+
+	if !isRunnerPod && !isRunnerSetPod && !isRunnerDeploymentPod {
 		return ctrl.Result{}, nil
 	}
 
@@ -77,6 +82,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	var enterprise, org, repo string
+	var isContainerMode bool
 
 	for _, e := range envvars {
 		switch e.Name {
@@ -86,13 +92,20 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			org = e.Value
 		case EnvVarRepo:
 			repo = e.Value
+		case "ACTIONS_RUNNER_CONTAINER_HOOKS":
+			isContainerMode = true
 		}
 	}
 
 	if runnerPod.ObjectMeta.DeletionTimestamp.IsZero() {
 		finalizers, added := addFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
 
-		if added {
+		var cleanupFinalizersAdded bool
+		if isContainerMode {
+			finalizers, cleanupFinalizersAdded = addFinalizer(finalizers, runnerLinkedResourcesFinalizerName)
+		}
+
+		if added || cleanupFinalizersAdded {
 			newRunner := runnerPod.DeepCopy()
 			newRunner.ObjectMeta.Finalizers = finalizers
 
@@ -108,6 +121,27 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	} else {
 		log.V(2).Info("Seen deletion-timestamp is already set")
 
+		if finalizers, removed := removeFinalizer(runnerPod.ObjectMeta.Finalizers, runnerLinkedResourcesFinalizerName); removed {
+			if err := r.cleanupRunnerLinkedPods(ctx, &runnerPod, log); err != nil {
+				log.Info("Runner-linked pods clean up that has failed due to an error. If this persists, please manually remove the runner-linked pods to unblock ARC", "err", err.Error())
+				return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+			}
+			if err := r.cleanupRunnerLinkedSecrets(ctx, &runnerPod, log); err != nil {
+				log.Info("Runner-linked secrets clean up that has failed due to an error. If this persists, please manually remove the runner-linked secrets to unblock ARC", "err", err.Error())
+				return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+			}
+			patchedPod := runnerPod.DeepCopy()
+			patchedPod.ObjectMeta.Finalizers = finalizers
+
+			if err := r.Patch(ctx, patchedPod, client.MergeFrom(&runnerPod)); err != nil {
+				log.Error(err, "Failed to update runner for finalizer linked resources removal")
+				return ctrl.Result{}, err
+			}
+
+			// Otherwise the subsequent patch request can revive the removed finalizer and it will trigger a unnecessary reconcilation
+			runnerPod = *patchedPod
+		}
+
 		finalizers, removed := removeFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
 
 		if removed {
@@ -222,3 +256,93 @@ func (r *RunnerPodReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Named(name).
 		Complete(r)
 }
+
+func (r *RunnerPodReconciler) cleanupRunnerLinkedPods(ctx context.Context, pod *corev1.Pod, log logr.Logger) error {
+	var runnerLinkedPodList corev1.PodList
+	if err := r.List(ctx, &runnerLinkedPodList, client.InNamespace(pod.Namespace), client.MatchingLabels(
+		map[string]string{
+			"runner-pod": pod.ObjectMeta.Name,
+		},
+	)); err != nil {
+		return fmt.Errorf("failed to list runner-linked pods: %w", err)
+	}
+
+	var (
+		wg   sync.WaitGroup
+		errs []error
+	)
+	for _, p := range runnerLinkedPodList.Items {
+		if !p.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		p := p
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := r.Delete(ctx, &p); err != nil {
+				if kerrors.IsNotFound(err) || kerrors.IsGone(err) {
+					return
+				}
+				errs = append(errs, fmt.Errorf("delete pod %q error: %v", p.ObjectMeta.Name, err))
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			log.Error(err, "failed to remove runner-linked pod")
+		}
+		return errors.New("failed to remove some runner linked pods")
+	}
+
+	return nil
+}
+
+func (r *RunnerPodReconciler) cleanupRunnerLinkedSecrets(ctx context.Context, pod *corev1.Pod, log logr.Logger) error {
+	log.V(2).Info("Listing runner-linked secrets to be deleted", "ns", pod.Namespace)
+
+	var runnerLinkedSecretList corev1.SecretList
+	if err := r.List(ctx, &runnerLinkedSecretList, client.InNamespace(pod.Namespace), client.MatchingLabels(
+		map[string]string{
+			"runner-pod": pod.ObjectMeta.Name,
+		},
+	)); err != nil {
+		return fmt.Errorf("failed to list runner-linked secrets: %w", err)
+	}
+
+	var (
+		wg   sync.WaitGroup
+		errs []error
+	)
+	for _, s := range runnerLinkedSecretList.Items {
+		if !s.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		s := s
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := r.Delete(ctx, &s); err != nil {
+				if kerrors.IsNotFound(err) || kerrors.IsGone(err) {
+					return
+				}
+				errs = append(errs, fmt.Errorf("delete secret %q error: %v", s.ObjectMeta.Name, err))
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			log.Error(err, "failed to remove runner-linked secret")
+		}
+		return errors.New("failed to remove some runner linked secrets")
+	}
+
+	return nil
+}

controllers/runnerdeployment_controller.go

@@ -179,7 +179,10 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	newDesiredReplicas := getIntOrDefault(desiredRS.Spec.Replicas, defaultReplicas)
 
 	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
-	if currentDesiredReplicas != newDesiredReplicas {
+	//
+	// If we missed taking the EffectiveTime diff into account, you might end up experiencing scale-ups being delayed scale-down.
+	// See https://github.com/actions-runner-controller/actions-runner-controller/pull/1477#issuecomment-1164154496
+	if currentDesiredReplicas != newDesiredReplicas || newestSet.Spec.EffectiveTime != rd.Spec.EffectiveTime {
 		newestSet.Spec.Replicas = &newDesiredReplicas
 		newestSet.Spec.EffectiveTime = rd.Spec.EffectiveTime
 

controllers/runnerset_controller.go

@@ -195,7 +195,33 @@ func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*ap
 		Spec:       runnerSetWithOverrides.StatefulSetSpec.Template.Spec,
 	}
 
-	pod, err := newRunnerPod(runnerSet.Name, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL)
+	if runnerSet.Spec.RunnerConfig.ContainerMode == "kubernetes" {
+		found := false
+		for i := range template.Spec.Containers {
+			if template.Spec.Containers[i].Name == containerName {
+				found = true
+			}
+		}
+		if !found {
+			template.Spec.Containers = append(template.Spec.Containers, corev1.Container{
+				Name: "runner",
+			})
+		}
+
+		workDir := runnerSet.Spec.RunnerConfig.WorkDir
+		if workDir == "" {
+			workDir = "/runner/_work"
+		}
+		if err := applyWorkVolumeClaimTemplateToPod(&template, runnerSet.Spec.WorkVolumeClaimTemplate, workDir); err != nil {
+			return nil, err
+		}
+
+		template.Spec.ServiceAccountName = runnerSet.Spec.ServiceAccountName
+	}
+
+	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerSet.Name)
+
+	pod, err := newRunnerPodWithContainerMode(runnerSet.Spec.RunnerConfig.ContainerMode, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL)
 	if err != nil {
 		return nil, err
 	}

controllers/utils_test.go

@@ -3,6 +3,9 @@ package controllers
 import (
 	"reflect"
 	"testing"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 )
 
 func Test_filterLabels(t *testing.T) {
@@ -32,3 +35,94 @@ func Test_filterLabels(t *testing.T) {
 		})
 	}
 }
+
+func Test_workVolumeClaimTemplateVolumeV1VolumeTransformation(t *testing.T) {
+	storageClassName := "local-storage"
+	workVolumeClaimTemplate := v1alpha1.WorkVolumeClaimTemplate{
+		StorageClassName: storageClassName,
+		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+		Resources:        corev1.ResourceRequirements{},
+	}
+	want := corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+						StorageClassName: &storageClassName,
+						Resources:        corev1.ResourceRequirements{},
+					},
+				},
+			},
+		},
+	}
+
+	got := workVolumeClaimTemplate.V1Volume()
+
+	if got.Name != want.Name {
+		t.Errorf("want name %q, got %q\n", want.Name, got.Name)
+	}
+
+	if got.VolumeSource.Ephemeral == nil {
+		t.Fatal("work volume claim template should transform itself into Ephemeral volume source\n")
+	}
+
+	if got.VolumeSource.Ephemeral.VolumeClaimTemplate == nil {
+		t.Fatal("work volume claim template should have ephemeral volume claim template set\n")
+	}
+
+	gotClassName := *got.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName
+	wantClassName := *want.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName
+	if gotClassName != wantClassName {
+		t.Errorf("expected storage class name %q, got %q\n", wantClassName, gotClassName)
+	}
+
+	gotAccessModes := got.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes
+	wantAccessModes := want.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes
+	if len(gotAccessModes) != len(wantAccessModes) {
+		t.Fatalf("access modes lengths missmatch: got %v, expected %v\n", gotAccessModes, wantAccessModes)
+	}
+
+	diff := make(map[corev1.PersistentVolumeAccessMode]int, len(wantAccessModes))
+	for _, am := range wantAccessModes {
+		diff[am]++
+	}
+
+	for _, am := range gotAccessModes {
+		_, ok := diff[am]
+		if !ok {
+			t.Errorf("got access mode %v that is not in the wanted access modes\n", am)
+		}
+
+		diff[am]--
+		if diff[am] == 0 {
+			delete(diff, am)
+		}
+	}
+
+	if len(diff) != 0 {
+		t.Fatalf("got access modes did not take every access mode into account\nactual: %v expected: %v\n", gotAccessModes, wantAccessModes)
+	}
+}
+
+func Test_workVolumeClaimTemplateV1VolumeMount(t *testing.T) {
+
+	workVolumeClaimTemplate := v1alpha1.WorkVolumeClaimTemplate{
+		StorageClassName: "local-storage",
+		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+		Resources:        corev1.ResourceRequirements{},
+	}
+
+	mountPath := "/test/_work"
+	want := corev1.VolumeMount{
+		MountPath: mountPath,
+		Name:      "work",
+	}
+
+	got := workVolumeClaimTemplate.V1VolumeMount(mountPath)
+
+	if want != got {
+		t.Fatalf("expected volume mount %+v, actual %+v\n", want, got)
+	}
+}

docs/releasenotes/0.25.md

@@ -0,0 +1,43 @@
+# actions-runner-controller v0.25.0
+
+All planned changes in this release can be found in the milestone https://github.com/actions-runner-controller/actions-runner-controller/milestone/8.
+
+Also see https://github.com/actions-runner-controller/actions-runner-controller/compare/v0.24.1...v0.25.0 for full changelog.
+
+This log documents breaking changes and major enhancements
+
+## Upgrading
+
+In case you're using our Helm chart to deploy ARC, use the chart 0.20.0 or greater. Don't miss upgrading CRDs as usual! Helm doesn't upgrade CRDs.
+
+## BREAKING CHANGE : Support for `--once` has been dropped
+
+In case you're still on ARC v0.23.0 or earlier, please also read [the relevant part of v0.24.0 release note for more information](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/docs/releasenotes/0.24.md#breaking-change--support-for---once-is-being-dropped).
+
+Relevant PR(s): #1580, #1590
+
+## ENHANCEMENT : Support for the new Kubernetes container mode of Actions runner
+
+The GitHub Actions team has recently added `actions/runner` an ability to use [runner container hooks](https://github.com/actions/runner-container-hooks) to run job steps on Kubernetes pods instead of docker containers created by the `docker` command. It allows us to avoid the use of privileged containers while still being able to run container-backed job steps.
+
+To use the new container mode, you set `.spec.template.spec.containerMode` in `RunnerDeployment` to `"kubernetes"`, while defining `.spec.template.spec.workVolumeClaimTemplate`. The volume claim template is used for provisioning and assigning persistent volumes mounted across the runner pod and the job pods for sharing the job workspace.
+
+Before using this feature, we highly recommend you to read [the detailed explanation in the original pull request](https://github.com/actions-runner-controller/actions-runner-controller/pull/1546), and [the new section in ARC's documentation](https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs).
+
+Big kudos to @thboop and the GitHub Actions team for implementing and contributing this feature!
+
+Relevant PR(s): #1546
+
+## FIX : Webhook-based scaling is even more reliable
+
+We fixed a race condition in the webhook-based autoscaler that resulted in not adding a runner when necessary.
+
+The race condition had been happening when it received a webhook event while processing another webhook event and both ended up scaling up the same horizontal runner autoscaler at the same time.
+
+To mitigate that, ARC now uses Kubernetes' Update API instead of Patch to update `HRA.spec.capacityReservations` which is the underlying data structure that makes the webhook-based scaler to add replicas to RunnerDeployment or RunnerSet on demand.
+
+We were also worried about stressing the Kubernetes apiserver when your ARC webhook-based autoscaler received a lot of concurrent webhook events, we also enhanced it to batch the Update API calls for 3 seconds, which basically means it will call the Update API at most once every 3 seconds per webhook-based autoscaler instance.
+
+Lastly, we fixed a bug in the autoscaler that resulted in it to stop adding replicas for newly received webhook events when the desired replicas reached `maxReplicas`.
+
+Relevant PR(s): #1477, #1568

github/fake/runners.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 	"github.com/gorilla/mux"
 )
 

github/github.go

@@ -14,7 +14,7 @@ import (
 	"github.com/actions-runner-controller/actions-runner-controller/logging"
 	"github.com/bradleyfalzon/ghinstallation/v2"
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 	"github.com/gregjones/httpcache"
 	"golang.org/x/oauth2"
 )
@@ -248,7 +248,8 @@ func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string)
 func (c *Client) ListOrganizationRunnerGroups(ctx context.Context, org string) ([]*github.RunnerGroup, error) {
 	var runnerGroups []*github.RunnerGroup
 
-	opts := github.ListOptions{PerPage: 100}
+	opts := github.ListOrgRunnerGroupOptions{}
+	opts.PerPage = 100
 	for {
 		list, res, err := c.Client.Actions.ListOrganizationRunnerGroups(ctx, org, &opts)
 		if err != nil {
@@ -271,9 +272,21 @@ func (c *Client) ListOrganizationRunnerGroups(ctx context.Context, org string) (
 func (c *Client) ListOrganizationRunnerGroupsForRepository(ctx context.Context, org, repo string) ([]*github.RunnerGroup, error) {
 	var runnerGroups []*github.RunnerGroup
 
-	opts := github.ListOptions{PerPage: 100}
+	var opts github.ListOrgRunnerGroupOptions
+
+	opts.PerPage = 100
+
+	repoName := repo
+	parts := strings.Split(repo, "/")
+	if len(parts) == 2 {
+		repoName = parts[1]
+	}
+	// This must be the repo name without the owner part, so in case the repo is "myorg/myrepo" the repo name
+	// passed to visible_to_repository must be "myrepo".
+	opts.VisibleToRepository = repoName
+
 	for {
-		list, res, err := c.listOrganizationRunnerGroupsVisibleToRepo(ctx, org, repo, &opts)
+		list, res, err := c.Actions.ListOrganizationRunnerGroups(ctx, org, &opts)
 		if err != nil {
 			return runnerGroups, fmt.Errorf("failed to list organization runner groups: %w", err)
 		}
@@ -309,42 +322,6 @@ func (c *Client) ListRunnerGroupRepositoryAccesses(ctx context.Context, org stri
 	return repos, nil
 }
 
-// listOrganizationRunnerGroupsVisibleToRepo lists all self-hosted runner groups configured in an organization which can be used by the repository.
-//
-// GitHub API docs: https://docs.github.com/en/rest/reference/actions#list-self-hosted-runner-groups-for-an-organization
-func (c *Client) listOrganizationRunnerGroupsVisibleToRepo(ctx context.Context, org, repo string, opts *github.ListOptions) (*github.RunnerGroups, *github.Response, error) {
-	repoName := repo
-	parts := strings.Split(repo, "/")
-	if len(parts) == 2 {
-		repoName = parts[1]
-	}
-
-	u := fmt.Sprintf("orgs/%v/actions/runner-groups?visible_to_repository=%v", org, repoName)
-
-	if opts != nil {
-		if opts.PerPage > 0 {
-			u = fmt.Sprintf("%v&per_page=%v", u, opts.PerPage)
-		}
-
-		if opts.Page > 0 {
-			u = fmt.Sprintf("%v&page=%v", u, opts.Page)
-		}
-	}
-
-	req, err := c.Client.NewRequest("GET", u, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	groups := &github.RunnerGroups{}
-	resp, err := c.Client.Do(ctx, req, &groups)
-	if err != nil {
-		return nil, resp, err
-	}
-
-	return groups, resp, nil
-}
-
 // cleanup removes expired registration tokens.
 func (c *Client) cleanup() {
 	c.mu.Lock()

github/github_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 )
 
 var server *httptest.Server

go.mod

@@ -7,46 +7,56 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v1.2.3
 	github.com/google/go-cmp v0.5.8
-	github.com/google/go-github/v39 v39.2.0
+	github.com/google/go-github/v45 v45.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
 	github.com/prometheus/client_golang v1.12.2
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.8.0
 	github.com/teambition/rrule-go v1.8.0
 	go.uber.org/zap v1.21.0
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
+	golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0
 	gomodules.xyz/jsonpatch/v2 v2.2.0
-	k8s.io/api v0.23.5
-	k8s.io/apimachinery v0.23.5
-	k8s.io/client-go v0.23.5
-	sigs.k8s.io/controller-runtime v0.11.2
+	k8s.io/api v0.24.2
+	k8s.io/apimachinery v0.24.2
+	k8s.io/client-go v0.24.2
+	sigs.k8s.io/controller-runtime v0.12.2
 	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
 	cloud.google.com/go v0.81.0 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-github/v41 v41.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -56,24 +66,24 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
+	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.23.5 // indirect
-	k8s.io/component-base v0.23.5 // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
-	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.24.2 // indirect
+	k8s.io/component-base v0.24.2 // indirect
+	k8s.io/klog/v2 v2.60.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect
+	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
+	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 )
 

go.sum

@@ -52,7 +52,9 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/actions-runner-controller/httpcache v0.2.0 h1:hCNvYuVPJ2xxYBymqBvH0hSiQpqz4PHF/LbU3XghGNI=
 github.com/actions-runner-controller/httpcache v0.2.0/go.mod h1:JLu9/2M/btPz1Zu/vTZ71XzukQHn2YeISPmJoM5exBI=
@@ -66,6 +68,7 @@ github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.m
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@@ -78,6 +81,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 h1:tXKVfhE7FcSkhkv0UwkLvPDeZ4kz6OXd0PKPlFqf81M=
 github.com/bradleyfalzon/ghinstallation/v2 v2.0.4/go.mod h1:B40qPqJxWE0jDZgOR1JmaMy+4AY1eBP+IByOvqyAKp0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -106,6 +110,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -117,6 +122,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -157,10 +163,13 @@ github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk=
 github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
 github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
@@ -210,7 +219,10 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
+github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
 github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA=
+github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
+github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -225,10 +237,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ=
-github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
 github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg=
 github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
+github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI=
+github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -295,6 +307,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -327,6 +340,7 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -345,6 +359,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
+github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -353,6 +368,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -394,6 +410,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
 github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -421,6 +438,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -441,6 +459,7 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
+github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -451,6 +470,7 @@ github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -459,6 +479,10 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q=
+github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/teambition/rrule-go v1.8.0 h1:a/IX5s56hGkFF+nRlJUooZU/45OTeeldBGL29nDKIHw=
 github.com/teambition/rrule-go v1.8.0/go.mod h1:Ieq5AbrKGciP1V//Wq8ktsTXwSwJHDD5mD/wLBGl3p4=
@@ -471,12 +495,16 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
 go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q=
 go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
 go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
 go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
@@ -524,6 +552,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -559,6 +590,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -605,10 +637,14 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -623,8 +659,13 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 h1:uBgVQYJLi/m8M0wzp+aGwBWt90gMRoOVf+aWTW10QHI=
+golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
+golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 h1:VnGaRqoLmqZH/3TMLJwYCEWkR4j1nuIU1U9TvbqsDUw=
+golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -701,9 +742,14 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
@@ -724,6 +770,8 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
+golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -783,6 +831,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
+golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -864,6 +913,7 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -900,6 +950,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -931,6 +983,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -942,34 +996,57 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
 k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
+k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI=
+k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg=
 k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI=
 k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ=
+k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k=
+k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ=
 k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0=
 k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
+k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM=
+k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
 k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw=
+k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI=
 k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8=
 k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
+k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA=
+k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30=
 k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
+k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w=
 k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE=
 k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0=
+k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU=
+k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
+k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
+k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU=
+k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
+k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
 sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA=
 sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4=
+sigs.k8s.io/controller-runtime v0.12.2 h1:nqV02cvhbAj7tbt21bpPpTByrXGn2INHRsi39lXy9sE=
+sigs.k8s.io/controller-runtime v0.12.2/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
+sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=
+sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=

pkg/githubwebhookdeliveryforwarder/githubwebhookdelivery.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v45/github"
 )
 
 type server struct {

pkg/hookdeliveryforwarder/forwarder.go

@@ -12,7 +12,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v45/github"
 )
 
 type Forwarder struct {

pkg/hookdeliveryforwarder/hooks.go

@@ -3,7 +3,7 @@ package hookdeliveryforwarder
 import (
 	"context"
 
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v45/github"
 )
 
 type hooksAPI struct {

pkg/hookdeliveryforwarder/hooks_deliveries.go

@@ -3,7 +3,7 @@ package hookdeliveryforwarder
 import (
 	"context"
 
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v45/github"
 )
 
 type hookDeliveriesAPI struct {

pkg/hookdeliveryforwarder/multiforwarder.go

@@ -8,7 +8,7 @@ import (
 	"sync"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v39/github"
+	gogithub "github.com/google/go-github/v45/github"
 )
 
 type MultiForwarder struct {

runner/Makefile

@@ -4,7 +4,8 @@ DIND_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind
 TAG ?= latest
 TARGETPLATFORM ?= $(shell arch)
 
-RUNNER_VERSION ?= 2.293.0
+RUNNER_VERSION ?= 2.294.0
+RUNNER_CONTAINER_HOOKS_VERSION ?= 0.1.2
 DOCKER_VERSION ?= 20.10.12
 
 # default list of platforms for which multiarch image is built
@@ -28,6 +29,7 @@ docker-build-ubuntu:
 	docker build \
 	  --build-arg TARGETPLATFORM=${TARGETPLATFORM} \
 	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+	  --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \
 	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
 	  -f actions-runner.dockerfile \
 	  -t ${NAME}:${TAG} .
@@ -50,12 +52,14 @@ docker-buildx-ubuntu:
 	fi
 	docker buildx build --platform ${PLATFORMS} \
 	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+	  --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \
 	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
 	  -f actions-runner.dockerfile \
 	  -t "${NAME}:${TAG}" \
 	  . ${PUSH_ARG}
 	docker buildx build --platform ${PLATFORMS} \
 	  --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+	  --build-arg RUNNER_CONTAINER_HOOKS_VERSION=${RUNNER_CONTAINER_HOOKS_VERSION} \
 	  --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
 	  -f actions-runner-dind.dockerfile \
 	  -t "${DIND_RUNNER_NAME}:${TAG}" \

runner/actions-runner-dind.dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.293.0
+ARG RUNNER_VERSION=2.294.0
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
@@ -74,8 +74,6 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 	dockerd --version; \
 	docker --version
 
-ENV HOME=/home/runner
-
 # Runner download supports amd64 as x64
 #
 # libyaml-dev is required for ruby/setup-ruby action.
@@ -113,6 +111,7 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 
 VOLUME /var/lib/docker
 
+ENV HOME=/home/runner
 # Add the Python "User Script Directory" to the PATH
 ENV PATH="${PATH}:${HOME}/.local/bin"
 ENV ImageOS=ubuntu20

runner/actions-runner.dockerfile

@@ -1,7 +1,8 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.293.0
+ARG RUNNER_VERSION=2.294.0
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.1.2
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
@@ -66,8 +67,6 @@ RUN set -vx; \
     && usermod -aG docker runner \
     && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
 
-ENV HOME=/home/runner
-
 # Uncomment the below COPY to use your own custom build of actions-runner.
 #
 # To build a custom runner:
@@ -105,6 +104,11 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
     && apt-get install -y libyaml-dev \
     && rm -rf /var/lib/apt/lists/*
 
+RUN cd "$RUNNER_ASSETS_DIR" \
+    && curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-container-hooks/releases/download/v${RUNNER_CONTAINER_HOOKS_VERSION}/actions-runner-hooks-k8s-${RUNNER_CONTAINER_HOOKS_VERSION}.zip \
+    && unzip ./runner-container-hooks.zip -d ./k8s \
+    && rm runner-container-hooks.zip
+
 ENV RUNNER_TOOL_CACHE=/opt/hostedtoolcache
 RUN mkdir /opt/hostedtoolcache \
     && chgrp docker /opt/hostedtoolcache \
@@ -114,6 +118,7 @@ RUN mkdir /opt/hostedtoolcache \
 # override them with scripts of the same name placed in `/usr/local/bin`.
 COPY entrypoint.sh logger.bash /usr/bin/
 
+ENV HOME=/home/runner
 # Add the Python "User Script Directory" to the PATH
 ENV PATH="${PATH}:${HOME}/.local/bin"
 ENV ImageOS=ubuntu20

runner/entrypoint.sh

@@ -9,14 +9,6 @@ if [ ! -z "${STARTUP_DELAY_IN_SECONDS}" ]; then
   sleep ${STARTUP_DELAY_IN_SECONDS}
 fi
 
-if [[ "${DISABLE_WAIT_FOR_DOCKER}" != "true" ]] && [[ "${DOCKER_ENABLED}" == "true" ]]; then
-    log.debug 'Docker enabled runner detected and Docker daemon wait is enabled'
-    log.debug 'Waiting until Docker is available or the timeout is reached'
-    timeout 120s bash -c 'until docker ps ;do sleep 1; done'
-else
-  log.notice 'Docker wait check skipped. Either Docker is disabled or the wait is disabled, continuing with entrypoint'
-fi
-
 if [ -z "${GITHUB_URL}" ]; then
   log.debug 'Working with public GitHub'
   GITHUB_URL="https://github.com/"
@@ -140,13 +132,12 @@ if [ -z "${UNITTEST:-}" ] && [ -e ./externalstmp ]; then
   mv ./externalstmp/* ./externals/
 fi
 
-args=()
-if [ "${RUNNER_FEATURE_FLAG_ONCE:-}" == "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
-  args+=(--once)
-  log.warning 'Passing --once is deprecated and will be removed as an option' \
-    'from the image and actions-runner-controller at the release of 0.25.0.' \
-    'Upgrade to GHES => 3.3 to continue using actions-runner-controller. If' \
-    'you are using github.com ignore this warning.'
+if [[ "${DISABLE_WAIT_FOR_DOCKER}" != "true" ]] && [[ "${DOCKER_ENABLED}" == "true" ]]; then
+    log.debug 'Docker enabled runner detected and Docker daemon wait is enabled'
+    log.debug 'Waiting until Docker is available or the timeout is reached'
+    timeout 120s bash -c 'until docker ps ;do sleep 1; done'
+else
+  log.notice 'Docker wait check skipped. Either Docker is disabled or the wait is disabled, continuing with entrypoint'
 fi
 
 # Unset entrypoint environment variables so they don't leak into the runner environment
@@ -164,4 +155,4 @@ unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN STARTUP_DELAY_IN_SECONDS DISABLE_WAIT
 if [ -z "${UNITTEST:-}" ]; then
   mapfile -t env </etc/environment
 fi
-exec env -- "${env[@]}" ./run.sh "${args[@]}"
+exec env -- "${env[@]}" ./run.sh

runner/startup.sh

@@ -20,7 +20,9 @@ function wait_for_process () {
 sudo /bin/bash <<SCRIPT
 mkdir -p /etc/docker
 
-echo "{}" > /etc/docker/daemon.json
+if [ ! -f /etc/docker/daemon.json ]; then
+  echo "{}" > /etc/docker/daemon.json
+fi
 
 if [ -n "${MTU}" ]; then
 jq ".\"mtu\" = ${MTU}" /etc/docker/daemon.json > /tmp/.daemon.json && mv /tmp/.daemon.json /etc/docker/daemon.json

simulator/runnergroup_visibility.go

@@ -5,10 +5,12 @@ import (
 	"fmt"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/go-logr/logr"
 )
 
 type Simulator struct {
 	Client *github.Client
+	Log    logr.Logger
 }
 
 func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org, repo string, managed *VisibleRunnerGroups) (*VisibleRunnerGroups, error) {
@@ -24,6 +26,10 @@ func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org,
 			return visible, err
 		}
 
+		if c.Log.V(3).Enabled() {
+			c.Log.V(3).Info("ListOrganizationRunnerGroupsForRepository succeeded", "runerGroups", runnerGroups)
+		}
+
 		for _, runnerGroup := range runnerGroups {
 			ref := NewRunnerGroupFromGitHub(runnerGroup)
 

simulator/runnergroups.go

@@ -5,7 +5,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/google/go-github/v39/github"
+	"github.com/google/go-github/v45/github"
 )
 
 type RunnerGroupScope int

test/e2e/e2e_test.go

@@ -13,6 +13,13 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+type DeployKind int
+
+const (
+	RunnerSets DeployKind = iota
+	RunnerDeployments
+)
+
 var (
 	controllerImageRepo = "actionsrunnercontrollere2e/actions-runner-controller"
 	controllerImageTag  = "e2e"
@@ -41,7 +48,7 @@ var (
 			Args: []testing.BuildArg{
 				{
 					Name:  "RUNNER_VERSION",
-					Value: "2.291.1",
+					Value: "2.294.0",
 				},
 			},
 			Image:        runnerImage,
@@ -52,7 +59,7 @@ var (
 			Args: []testing.BuildArg{
 				{
 					Name:  "RUNNER_VERSION",
-					Value: "2.291.1",
+					Value: "2.294.0",
 				},
 			},
 			Image:        runnerDindImage,
@@ -71,9 +78,7 @@ var (
 	}
 
 	commonScriptEnv = []string{
-		"SYNC_PERIOD=" + "30m",
-		"NAME=" + controllerImageRepo,
-		"VERSION=" + controllerImageTag,
+		"SYNC_PERIOD=" + "30s",
 		"RUNNER_TAG=" + runnerImageTag,
 	}
 
@@ -101,13 +106,27 @@ var (
 // whenever the whole test failed, so that you can immediately start fixing issues and rerun inidividual tests.
 // See the below link for how terratest handles this:
 // https://terratest.gruntwork.io/docs/testing-best-practices/iterating-locally-using-test-stages/
+//
+// This functions leaves PVs undeleted. To delete PVs, run:
+//   kubectl get pv -ojson | jq -rMc '.items[] | select(.status.phase == "Available") | {name:.metadata.name, status:.status.phase} | .name' | xargs kubectl delete pv
+//
+// If you disk full after dozens of test runs, try:
+//   docker system prune
+// and
+//   kind delete cluster --name teste2e
+//
+// The former tend to release 200MB-3GB and the latter can result in releasing like 100GB due to kind node contains loaded container images and
+// (in case you use it) local provisioners disk image(which is implemented as a directory within the kind node).
 func TestE2E(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipped as -short is set")
 	}
 
+	skipRunnerCleanUp := os.Getenv("ARC_E2E_SKIP_RUNNER_CLEANUP") != ""
+	retainCluster := os.Getenv("ARC_E2E_RETAIN_CLUSTER") != ""
+	skipTestIDCleanUp := os.Getenv("ARC_E2E_SKIP_TEST_ID_CLEANUP") != ""
+
 	env := initTestEnv(t)
-	env.useRunnerSet = true
 
 	t.Run("build and load images", func(t *testing.T) {
 		env.buildAndLoadImages(t)
@@ -121,72 +140,107 @@ func TestE2E(t *testing.T) {
 		return
 	}
 
-	t.Run("install actions-runner-controller and runners", func(t *testing.T) {
-		env.installActionsRunnerController(t)
+	t.Run("RunnerSets", func(t *testing.T) {
+		var (
+			testID string
+		)
+
+		t.Run("get or generate test ID", func(t *testing.T) {
+			testID = env.GetOrGenerateTestID(t)
+		})
+
+		if !skipTestIDCleanUp {
+			t.Cleanup(func() {
+				env.DeleteTestID(t)
+			})
+		}
+
+		t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
+			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
+		})
+
+		t.Run("deploy runners", func(t *testing.T) {
+			env.deploy(t, RunnerSets, testID)
+		})
+
+		if !skipRunnerCleanUp {
+			t.Cleanup(func() {
+				env.undeploy(t, RunnerSets, testID)
+			})
+		}
+
+		t.Run("install edge actions-runner-controller", func(t *testing.T) {
+			env.installActionsRunnerController(t, controllerImageRepo, controllerImageTag, testID)
+		})
+
+		if t.Failed() {
+			return
+		}
+
+		t.Run("Install workflow", func(t *testing.T) {
+			env.installActionsWorkflow(t, RunnerSets, testID)
+		})
+
+		if t.Failed() {
+			return
+		}
+
+		t.Run("Verify workflow run result", func(t *testing.T) {
+			env.verifyActionsWorkflowRun(t, testID)
+		})
 	})
 
-	if t.Failed() {
-		return
-	}
+	t.Run("RunnerDeployments", func(t *testing.T) {
+		var (
+			testID string
+		)
 
-	t.Run("Install workflow", func(t *testing.T) {
-		env.installActionsWorkflow(t)
+		t.Run("get or generate test ID", func(t *testing.T) {
+			testID = env.GetOrGenerateTestID(t)
+		})
+
+		if !skipTestIDCleanUp {
+			t.Cleanup(func() {
+				env.DeleteTestID(t)
+			})
+		}
+
+		t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
+			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
+		})
+
+		t.Run("deploy runners", func(t *testing.T) {
+			env.deploy(t, RunnerDeployments, testID)
+		})
+
+		if !skipRunnerCleanUp {
+			t.Cleanup(func() {
+				env.undeploy(t, RunnerDeployments, testID)
+			})
+		}
+
+		t.Run("install edge actions-runner-controller", func(t *testing.T) {
+			env.installActionsRunnerController(t, controllerImageRepo, controllerImageTag, testID)
+		})
+
+		if t.Failed() {
+			return
+		}
+
+		t.Run("Install workflow", func(t *testing.T) {
+			env.installActionsWorkflow(t, RunnerDeployments, testID)
+		})
+
+		if t.Failed() {
+			return
+		}
+
+		t.Run("Verify workflow run result", func(t *testing.T) {
+			env.verifyActionsWorkflowRun(t, testID)
+		})
 	})
 
-	if t.Failed() {
-		return
-	}
-
-	t.Run("Verify workflow run result", func(t *testing.T) {
-		env.verifyActionsWorkflowRun(t)
-	})
-
-	if os.Getenv("ARC_E2E_NO_CLEANUP") != "" {
-		t.FailNow()
-	}
-}
-
-func TestE2ERunnerDeploy(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipped as -short is set")
-	}
-
-	env := initTestEnv(t)
-	env.useApp = true
-
-	t.Run("build and load images", func(t *testing.T) {
-		env.buildAndLoadImages(t)
-	})
-
-	t.Run("install cert-manager", func(t *testing.T) {
-		env.installCertManager(t)
-	})
-
-	if t.Failed() {
-		return
-	}
-
-	t.Run("install actions-runner-controller and runners", func(t *testing.T) {
-		env.installActionsRunnerController(t)
-	})
-
-	if t.Failed() {
-		return
-	}
-
-	t.Run("Install workflow", func(t *testing.T) {
-		env.installActionsWorkflow(t)
-	})
-
-	if t.Failed() {
-		return
-	}
-
-	t.Run("Verify workflow run result", func(t *testing.T) {
-		env.verifyActionsWorkflowRun(t)
-	})
-
-	if os.Getenv("ARC_E2E_NO_CLEANUP") != "" {
+	if retainCluster {
 		t.FailNow()
 	}
 }
@@ -194,23 +248,20 @@ func TestE2ERunnerDeploy(t *testing.T) {
 type env struct {
 	*testing.Env
 
-	useRunnerSet bool
 	// Uses GITHUB_APP_ID, GITHUB_APP_INSTALLATION_ID, and GITHUB_APP_PRIVATE_KEY
 	// to let ARC authenticate as a GitHub App
 	useApp bool
 
-	testID                                                   string
-	testName                                                 string
-	repoToCommit                                             string
-	appID, appInstallationID, appPrivateKeyFile              string
-	runnerLabel, githubToken, testRepo, testOrg, testOrgRepo string
-	githubTokenWebhook                                       string
-	testEnterprise                                           string
-	testEphemeral                                            string
-	scaleDownDelaySecondsAfterScaleOut                       int64
-	minReplicas                                              int64
-	dockerdWithinRunnerContainer                             bool
-	testJobs                                                 []job
+	testName                                    string
+	repoToCommit                                string
+	appID, appInstallationID, appPrivateKeyFile string
+	githubToken, testRepo, testOrg, testOrgRepo string
+	githubTokenWebhook                          string
+	testEnterprise                              string
+	testEphemeral                               string
+	scaleDownDelaySecondsAfterScaleOut          int64
+	minReplicas                                 int64
+	dockerdWithinRunnerContainer                bool
 }
 
 func initTestEnv(t *testing.T) *env {
@@ -220,15 +271,11 @@ func initTestEnv(t *testing.T) *env {
 
 	e := &env{Env: testingEnv}
 
-	id := e.ID()
-
-	testName := t.Name() + " " + id
+	testName := t.Name()
 
 	t.Logf("Initializing test with name %s", testName)
 
-	e.testID = id
 	e.testName = testName
-	e.runnerLabel = "test-" + id
 	e.githubToken = testing.Getenv(t, "GITHUB_TOKEN")
 	e.appID = testing.Getenv(t, "GITHUB_APP_ID")
 	e.appInstallationID = testing.Getenv(t, "GITHUB_APP_INSTALLATION_ID")
@@ -240,7 +287,6 @@ func initTestEnv(t *testing.T) *env {
 	e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO", "")
 	e.testEnterprise = testing.Getenv(t, "TEST_ENTERPRISE", "")
 	e.testEphemeral = testing.Getenv(t, "TEST_EPHEMERAL", "")
-	e.testJobs = createTestJobs(id, testResultCMNamePrefix, 6)
 
 	e.scaleDownDelaySecondsAfterScaleOut, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT", "10"), 10, 32)
 	e.minReplicas, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_MIN_REPLICAS", "1"), 10, 32)
@@ -281,7 +327,7 @@ func (e *env) installCertManager(t *testing.T) {
 	e.KubectlWaitUntilDeployAvailable(t, "cert-manager", waitCfg.WithTimeout(60*time.Second))
 }
 
-func (e *env) installActionsRunnerController(t *testing.T) {
+func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID string) {
 	t.Helper()
 
 	e.createControllerNamespaceAndServiceAccount(t)
@@ -291,25 +337,11 @@ func (e *env) installActionsRunnerController(t *testing.T) {
 		"ACCEPTANCE_TEST_DEPLOYMENT_TOOL=" + "helm",
 	}
 
-	if e.useRunnerSet {
-		scriptEnv = append(scriptEnv, "USE_RUNNERSET=1")
-	} else {
-		scriptEnv = append(scriptEnv, "USE_RUNNERSET=false")
-	}
-
 	varEnv := []string{
-		"TEST_ENTERPRISE=" + e.testEnterprise,
-		"TEST_REPO=" + e.testRepo,
-		"TEST_ORG=" + e.testOrg,
-		"TEST_ORG_REPO=" + e.testOrgRepo,
 		"WEBHOOK_GITHUB_TOKEN=" + e.githubTokenWebhook,
-		"RUNNER_LABEL=" + e.runnerLabel,
-		"TEST_ID=" + e.testID,
-		"TEST_EPHEMERAL=" + e.testEphemeral,
-		fmt.Sprintf("RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT=%d", e.scaleDownDelaySecondsAfterScaleOut),
-		fmt.Sprintf("REPO_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
-		fmt.Sprintf("ORG_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
-		fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
+		"TEST_ID=" + testID,
+		"NAME=" + repo,
+		"VERSION=" + tag,
 	}
 
 	if e.useApp {
@@ -326,6 +358,54 @@ func (e *env) installActionsRunnerController(t *testing.T) {
 		)
 	}
 
+	scriptEnv = append(scriptEnv, varEnv...)
+	scriptEnv = append(scriptEnv, commonScriptEnv...)
+
+	e.RunScript(t, "../../acceptance/deploy.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
+}
+
+func (e *env) deploy(t *testing.T, kind DeployKind, testID string) {
+	t.Helper()
+	e.do(t, "apply", kind, testID)
+}
+
+func (e *env) undeploy(t *testing.T, kind DeployKind, testID string) {
+	t.Helper()
+	e.do(t, "delete", kind, testID)
+}
+
+func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
+	t.Helper()
+
+	e.createControllerNamespaceAndServiceAccount(t)
+
+	scriptEnv := []string{
+		"KUBECONFIG=" + e.Kubeconfig(),
+		"OP=" + op,
+	}
+
+	switch kind {
+	case RunnerSets:
+		scriptEnv = append(scriptEnv, "USE_RUNNERSET=1")
+	case RunnerDeployments:
+		scriptEnv = append(scriptEnv, "USE_RUNNERSET=false")
+	default:
+		t.Fatalf("Invalid deploy kind %v", kind)
+	}
+
+	varEnv := []string{
+		"TEST_ENTERPRISE=" + e.testEnterprise,
+		"TEST_REPO=" + e.testRepo,
+		"TEST_ORG=" + e.testOrg,
+		"TEST_ORG_REPO=" + e.testOrgRepo,
+		"RUNNER_LABEL=" + e.runnerLabel(testID),
+		"TEST_EPHEMERAL=" + e.testEphemeral,
+		fmt.Sprintf("RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT=%d", e.scaleDownDelaySecondsAfterScaleOut),
+		fmt.Sprintf("REPO_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
+		fmt.Sprintf("ORG_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
+		fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
+	}
+
 	if e.dockerdWithinRunnerContainer {
 		varEnv = append(varEnv,
 			"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=true",
@@ -341,7 +421,11 @@ func (e *env) installActionsRunnerController(t *testing.T) {
 	scriptEnv = append(scriptEnv, varEnv...)
 	scriptEnv = append(scriptEnv, commonScriptEnv...)
 
-	e.RunScript(t, "../../acceptance/deploy.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
+	e.RunScript(t, "../../acceptance/deploy_runners.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
+}
+
+func (e *env) runnerLabel(testID string) string {
+	return "test-" + testID
 }
 
 func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
@@ -351,16 +435,20 @@ func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
 	e.KubectlEnsureClusterRoleBindingServiceAccount(t, "default-admin", "cluster-admin", "default:default", testing.KubectlConfig{})
 }
 
-func (e *env) installActionsWorkflow(t *testing.T) {
+func (e *env) installActionsWorkflow(t *testing.T, kind DeployKind, testID string) {
 	t.Helper()
 
-	installActionsWorkflow(t, e.testName, e.runnerLabel, testResultCMNamePrefix, e.repoToCommit, e.testJobs)
+	installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID))
 }
 
-func (e *env) verifyActionsWorkflowRun(t *testing.T) {
+func (e *env) testJobs(testID string) []job {
+	return createTestJobs(testID, testResultCMNamePrefix, 6)
+}
+
+func (e *env) verifyActionsWorkflowRun(t *testing.T, testID string) {
 	t.Helper()
 
-	verifyActionsWorkflowRun(t, e.Env, e.testJobs)
+	verifyActionsWorkflowRun(t, e.Env, e.testJobs(testID))
 }
 
 type job struct {
@@ -383,7 +471,7 @@ func createTestJobs(id, testResultCMNamePrefix string, numJobs int) []job {
 
 const Branch = "main"
 
-func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, testJobs []job) {
+func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job) {
 	t.Helper()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
@@ -400,45 +488,70 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 		Jobs: map[string]testing.Job{},
 	}
 
+	kubernetesContainerMode := os.Getenv("TEST_CONTAINER_MODE") == "kubernetes"
+
+	var container string
+	if kubernetesContainerMode {
+		container = "golang:1.18"
+	}
+
 	for _, j := range testJobs {
-		wf.Jobs[j.name] = testing.Job{
-			RunsOn: runnerLabel,
-			Steps: []testing.Step{
-				{
-					Uses: testing.ActionsCheckoutV2,
-				},
-				{
+		steps := []testing.Step{
+			{
+				Uses: testing.ActionsCheckout,
+			},
+		}
+
+		if !kubernetesContainerMode {
+			if kind == RunnerDeployments {
+				steps = append(steps,
+					testing.Step{
+						Run: "sudo mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
+					},
+				)
+			}
+
+			steps = append(steps,
+				testing.Step{
 					// This might be the easiest way to handle permissions without use of securityContext
 					// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
 					Run: "sudo chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
 				},
-				{
+				testing.Step{
 					// This might be the easiest way to handle permissions without use of securityContext
 					// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
 					Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
 				},
-				{
+				testing.Step{
 					Uses: "actions/setup-go@v3",
 					With: &testing.With{
 						GoVersion: "1.18.2",
 					},
 				},
-				{
-					Run: "go version",
-				},
-				{
-					Run: "go build .",
-				},
-				{
+			)
+		}
+
+		steps = append(steps,
+			testing.Step{
+				Run: "go version",
+			},
+			testing.Step{
+				Run: "go build .",
+			},
+		)
+
+		if !kubernetesContainerMode {
+			steps = append(steps,
+				testing.Step{
 					// https://github.com/docker/buildx/issues/413#issuecomment-710660155
 					// To prevent setup-buildx-action from failing with:
 					//   error: could not create a builder instance with TLS data loaded from environment. Please use `docker context create <context-name>` to create a context for current environment and then create a builder instance with `docker buildx create <context-name>`
 					Run: "docker context create mycontext",
 				},
-				{
+				testing.Step{
 					Run: "docker context use mycontext",
 				},
-				{
+				testing.Step{
 					Name: "Set up Docker Buildx",
 					Uses: "docker/setup-buildx-action@v1",
 					With: &testing.With{
@@ -449,30 +562,36 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 						Install: false,
 					},
 				},
-				{
+				testing.Step{
 					Run: "docker buildx build --platform=linux/amd64 " +
 						"--cache-from=type=local,src=/home/runner/.cache/buildx " +
 						"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max " +
 						".",
 				},
-				{
+				testing.Step{
 					// https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
 					// See https://github.com/moby/buildkit/issues/1896 for why this is needed
 					Run: "rm -rf /home/runner/.cache/buildx && mv /home/runner/.cache/buildx-new /home/runner/.cache/buildx",
 				},
-				{
+				testing.Step{
 					Run: "ls -lah /home/runner/.cache/*",
 				},
-				{
+				testing.Step{
 					Uses: "azure/setup-kubectl@v1",
 					With: &testing.With{
 						Version: "v1.20.2",
 					},
 				},
-				{
+				testing.Step{
 					Run: fmt.Sprintf("./test.sh %s %s", t.Name(), j.testArg),
 				},
-			},
+			)
+		}
+
+		wf.Jobs[j.name] = testing.Job{
+			RunsOn:    runnerLabel,
+			Container: container,
+			Steps:     steps,
 		}
 	}
 
@@ -556,5 +675,5 @@ func verifyActionsWorkflowRun(t *testing.T, env *testing.Env, testJobs []job) {
 		}
 
 		return results, err
-	}, 3*60*time.Second, 10*time.Second).Should(gomega.Equal(expected))
+	}, 8*60*time.Second, 30*time.Second).Should(gomega.Equal(expected))
 }

testing/kubectl.go

@@ -86,6 +86,16 @@ func (k *Kubectl) CreateCMLiterals(ctx context.Context, name string, literals ma
 	return nil
 }
 
+func (k *Kubectl) DeleteCM(ctx context.Context, name string, cfg KubectlConfig) error {
+	args := []string{"cm", name}
+
+	if _, err := k.CombinedOutput(k.kubectlCmd(ctx, "delete", args, cfg)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (k *Kubectl) Apply(ctx context.Context, path string, cfg KubectlConfig) error {
 	if _, err := k.CombinedOutput(k.kubectlCmd(ctx, "apply", []string{"-f", path}, cfg)); err != nil {
 		return err

testing/testing.go

@@ -32,7 +32,6 @@ type Env struct {
 	docker  *Docker
 	Kubectl *Kubectl
 	bash    *Bash
-	id      string
 }
 
 func Start(t *testing.T, opts ...Option) *Env {
@@ -56,7 +55,11 @@ func Start(t *testing.T, opts ...Option) *Env {
 
 	env.bash = bash
 
-	//
+	return &env
+}
+
+func (e *Env) GetOrGenerateTestID(t *testing.T) string {
+	k, kctl := e.kind, e.Kubectl
 
 	cmKey := "id"
 
@@ -82,13 +85,27 @@ func Start(t *testing.T, opts ...Option) *Env {
 		}
 	}
 
-	env.id = m[cmKey]
-
-	return &env
+	return m[cmKey]
 }
 
-func (e *Env) ID() string {
-	return e.id
+func (e *Env) DeleteTestID(t *testing.T) {
+	k, kctl := e.kind, e.Kubectl
+
+	kubectlEnv := []string{
+		"KUBECONFIG=" + k.Kubeconfig(),
+	}
+
+	cmCfg := KubectlConfig{
+		Env: kubectlEnv,
+	}
+	testInfoName := "test-info"
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
+	defer cancel()
+
+	if err := kctl.DeleteCM(ctx, testInfoName, cmCfg); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func (e *Env) DockerBuild(t *testing.T, builds []DockerBuild) {

testing/workflow.go

@@ -1,7 +1,7 @@
 package testing
 
 const (
-	ActionsCheckoutV2 = "actions/checkout@v2"
+	ActionsCheckout = "actions/checkout@v3"
 )
 
 type Workflow struct {
@@ -30,8 +30,9 @@ type InputSpec struct {
 }
 
 type Job struct {
-	RunsOn string `json:"runs-on"`
-	Steps  []Step `json:"steps"`
+	RunsOn    string `json:"runs-on"`
+	Container string `json:"container,omitempty"`
+	Steps     []Step `json:"steps"`
 }
 
 type Step struct {