Add releasenote for 0.26.0 (#1796)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.github/renovate.json5

@@ -31,7 +31,8 @@
     {
       "fileMatch": [
         "runner/actions-runner.dockerfile",
-        "runner/actions-runner-dind.dockerfile"
+        "runner/actions-runner-dind.dockerfile",
+        "runner/actions-runner-dind-rootless.dockerfile"
       ],
       "matchStrings": ["RUNNER_VERSION=+(?<currentValue>.*?)\\n"],
       "depNameTemplate": "actions/runner",

.github/workflows/publish-arc.yaml

@@ -58,6 +58,7 @@ jobs:
         with:
           file: Dockerfile
           platforms: linux/amd64,linux/arm64
+          build-args: VERSION=${{ env.VERSION }}
           push: true
           tags: |
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest

.github/workflows/publish-canary.yaml

@@ -50,9 +50,10 @@ jobs:
         with:
           file: Dockerfile
           platforms: linux/amd64,linux/arm64
+          build-args: VERSION=canary-${{ github.sha }}
           push: true
           tags: |
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary
-            ghcr.io/actions-runner-controller/actions-runner-controller:canary
+            ghcr.io/${{ github.repository }}:canary
           cache-from: type=gha,scope=arc-canary
           cache-to: type=gha,mode=max,scope=arc-canary

.github/workflows/publish-chart.yaml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3.0
+        uses: azure/setup-helm@v3.3
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -57,7 +57,7 @@ jobs:
           python-version: '3.7'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.2.1
+        uses: helm/chart-testing-action@v2.3.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed

.github/workflows/runners.yaml

@@ -25,7 +25,7 @@ on:
       - '!**.md'
 
 env:
-  RUNNER_VERSION: 2.294.0
+  RUNNER_VERSION: 2.296.2
   DOCKER_VERSION: 20.10.12
   RUNNER_CONTAINER_HOOKS_VERSION: 0.1.2
   DOCKERHUB_USERNAME: summerwind
@@ -47,6 +47,9 @@ jobs:
           - name: actions-runner-dind
             os-name: ubuntu
             os-version: 20.04
+          - name: actions-runner-dind-rootless
+            os-name: ubuntu
+            os-version: 20.04
 
     steps:
       - name: Checkout

.github/workflows/validate-chart.yaml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3.0
+        uses: azure/setup-helm@v3.3
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -52,7 +52,7 @@ jobs:
           python-version: '3.7'
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.2.1
+        uses: helm/chart-testing-action@v2.3.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed

Actions-Runner-Controller-Overview.md

@@ -0,0 +1,132 @@
+## Introduction
+This document provides a high level overview of Actions Runner Controller (ARC). ARC enables running Github Actions Runners on Kubernetes (K8s) clusters.
+
+This document provides a background of Github Actions, self-hosted runners and ARC overview. By the end of the doc, the reader should have a foundation with basic scenarios and be capable of reviewing other advanced topics.
+
+## GitHub Actions
+[GitHub Actions](https://github.com/features/actions) is a continuous integration and continuous delivery (CI/CD) platform to automate your build, test, and deployment pipeline. 
+
+You can create workflows that build and test every pull request to your repository, or deploy merged pull requests to production. Your workflow contains one or more jobs which can run in sequential order or in parallel. Each job will run inside its own runner and has one or more steps that either run a script that you define or run an action, which is a reusable extension that can simplify your workflow. To learn more about about Actions - see "[Learn Github Actions](https://docs.github.com/en/actions/learn-github-actions)".
+
+## Runners
+Runners execute the job that is assigned to them by Github Actions workflow. There are two types of Runners:
+
+- [Github-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners) - GitHub provides Linux, Windows, and macOS virtual machines to run your workflows. These virtual machines are hosted in the cloud by Github.
+- [Self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) - you can host your own self-hosted runners in your own data center or cloud infrastructure. ARC deploys self-hosted runners.
+
+## Self hosted runners
+Self-hosted runners offer more control of hardware, operating system, and software tools than GitHub-hosted runners. With self-hosted runners, you can create custom hardware configurations that meet your needs with processing power or memory to run larger jobs, install software available on your local network, and choose an operating system not offered by GitHub-hosted runners.
+
+### Types of Self hosted runners
+Self-hosted runners can be physical, virtual, in a container, on-premises, or in a cloud.
+- Traditional Deployment is having a physical machine, with OS and apps on it. The runner runs on this machine and executes any jobs. It comes with the cost of owning and operating the hardware 24/7 even if it isn't in use that entire time. 
+- Virtualized deployments are simpler to manage. Each runner runs on a virtual machine (VM) that runs on a host. There could be multiple such VMs running on the same host. VMs are complete OS’s and might take time to bring up everytime a clean environment is needed to run workflows.
+- Containerized deployments are similar to VMs, but instead of bringing up entire VM’s, a container gets deployed.Kubernetes (K8s) provides a scalable and reproducible environment for containerized workloads. They are lightweight, loosely coupled, highly efficient and can be managed centrally. There are advantages to using Kubernetes (outlined "[here](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/)."), but it is more complicated and less widely-understood than the other options. A managed provider makes this much simpler to run at scale.
+
+*Actions Runner Controller(ARC) makes it simpler to run self hosted runners on K8s managed containers.*
+
+## Actions Runner Controller (ARC)
+ARC  is a K8s controller to create self-hosted runners on your K8s cluster. With few commands, you can set up self hosted runners that can scale up and down based on demand. And since these could be ephemeral and based on containers, new instances of the runner can be brought up rapidly and cleanly.
+
+### Deploying ARC
+We have a quick start guide that demonstrates how to easily deploy ARC into your K8s environment. For more details, see "[QuickStart Guide](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/QuickStartGuide.md)."
+
+## ARC components
+ARC basically consists of a set of custom resources. An ARC deployment is applying these custom resources onto a K8s cluster. Once applied, it creates a set of Pods, with the Github Actions runner running within them. Github is now able to treat these Pods as self hosted runners and allocate jobs to them.
+
+### Custom resources 
+ARC consists of several custom resource definitions (Runner, Runner Set, Runner Deployment, Runner Replica Set and Horizontal Runner AutoScaler). For more information on CRDs, refer "[Kubernetes Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/)."
+
+The helm command (in the QuickStart guide) installs the custom resources into the actions-runner-system namespace.
+```console
+helm install -f custom-values.yaml --wait --namespace actions-runner-system \
+  --create-namespace actions-runner-controller \
+  actions-runner-controller/actions-runner-controller
+ ```
+
+### Runner deployment
+Once the custom resources are installed, another command deploys ARC into your K8s cluster.
+
+![actions-runner-controller architecture](https://user-images.githubusercontent.com/53718047/183928236-ddf72c15-1d11-4304-ad6f-0a0ff251ca55.jpg)
+
+
+
+The `Deployment and Configure ARC` section in the `Quick Start guide` lists the steps to deploy ARC using a `runnerdeployment.yaml` file. Here, we will explain the details
+For more details, see "[QuickStart Guide](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/QuickStartGuide.md)."
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeploy
+spec:
+  replicas: 1
+  template:
+    spec:
+      repository: mumoshu/actions-runner-controller-ci
+```
+
+- `kind: RunnerDeployment`: indicates its a kind of custom resource RunnerDeployment.
+- `replicas: 1` : will deploy one replica. Multiple replicas can also be deployed ( more on that later).
+- `repository: mumoshu/actions-runner-controller-ci` : is the repository to link to when the pod comes up with the Actions runner (Note, this can be configured to link at the Enterprise or Organization level also).
+
+When this configuration is applied with `kubectl apply -f runnerdeployment.yaml` , ARC creates one pod `example-runnerdeploy-[**]` with 2 containers `runner` and `docker`.
+`runner` container has the github runner component installed, `docker` container has docker installed.
+
+
+### The Runner container image
+The GitHub hosted runners include a large amount of pre-installed software packages. For complete list, see "[Runner images](https://github.com/actions/virtual-environments/tree/main/images/linux)."
+
+ARC maintains a few runner images with `latest` aligning with GitHub's Ubuntu version. These images do not contain all of the software installed on the GitHub runners. They contain subset of packages from the GitHub runners: Basic CLI packages, git, docker and build-essentials. To install additional software, it is recommended to use the corresponding setup actions. For instance, `actions/setup-java` for Java or `actions/setup-node` for Node.
+
+## Executing workflows
+Now, all the setup and configuration is done. A workflow can be created in the same repository that could target the self hosted runner created from ARC. The workflow needs to have `runs-on: self-hosted` so it can target the self host pool. For more information on targeting workflows to run on self hosted runners, see "[Using Self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow)."
+
+## Scaling runners - statically with replicas count
+With a small tweak to the replicas count (for eg - `replicas: 2`) in the `runnerdeployment.yaml` file, more runners can be created. Depending on the count of replicas, those many sets of pods would be created. As before, Each pod contains the two containers.
+
+
+## Scaling runners - dynamically with Pull Driven Scaling
+ARC also allows for scaling the runners dynamically. There are two mechanisms for dynamically scaling - (1) Webhook driven scaling and (2) Pull Driven scaling, This document describes the Pull Driven scaling model.
+
+![actions-runner-controller architecture_2](https://user-images.githubusercontent.com/53718047/183928429-7000329d-38eb-4054-9879-41ae44e1ff85.jpg)
+
+
+
+You can enable scaling with 3 steps
+1) Enable `HorizontalRunnerAutoscaler` - Create a `deployment.yaml` file of type `HorizontalRunnerAutoscaler`. The schema for this file is defined below.
+2) Scaling parameters - `minReplicas` and `maxReplicas` indicates the min and max number of replicas to scale to.
+3) Scaling metrics - ARC currently supports `PercentageRunnersBusy` as a metric type. The `PercentageRunnersBusy` will poll GitHub for the number of runners in the `busy` state in the RunnerDeployment's namespace, it will then scale depending on how you have configured the scale factors.
+
+### Pull Driven Scaling Schema
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    # Your RunnerDeployment Here
+    name: example-runnerdeploy
+    kind: RunnerDeployment
+  minReplicas: 1
+  maxReplicas: 5
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'
+    scaleDownThreshold: '0.25'
+    scaleUpFactor: '2'
+    scaleDownFactor: '0.5'
+  ```
+
+For more details - please see "[Pull Driven Scaling](https://github.com/actions-runner-controller/actions-runner-controller#pull-driven-scaling)."
+
+*The period between polls is defined by the controller's `--sync-period` flag. If this flag isn't provided then the controller defaults to a sync period of `1m`, this can be configured in seconds or minutes.*
+
+## Other Configurations
+ARC supports several different advanced configuration. 
+- support for alternate runners : Setting up runner pods with Docker-In-Docker configuration.
+- managing runner groups : Managing a set of running with runner groups thus making it easy to manage different groups within enterprise
+- Webhook driven scaling.
+
+Please refer to the documentation in this repo for further details.

Dockerfile

@@ -1,11 +1,10 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.18.3 as builder
+FROM --platform=$BUILDPLATFORM golang:1.19.1 as builder
 
 WORKDIR /workspace
 
 # Make it runnable on a distroless image/without libc
 ENV CGO_ENABLED=0
-
 # Copy the Go Modules manifests
 COPY go.mod go.sum ./
 
@@ -25,7 +24,7 @@ RUN go mod download
 # With the above commmand,
 # TARGETOS can be "linux", TARGETARCH can be "amd64", "arm64", and "arm", TARGETVARIANT can be "v7".
 
-ARG TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT
+ARG TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT VERSION=dev
 
 # We intentionally avoid `--mount=type=cache,mode=0777,target=/go/pkg/mod` in the `go mod download` and the `go build` runs
 # to avoid https://github.com/moby/buildkit/issues/2334
@@ -37,7 +36,7 @@ env GOCACHE /build/${TARGETPLATFORM}/root/.cache/go-build
 RUN --mount=target=. \
   --mount=type=cache,mode=0777,target=${GOCACHE} \
   export GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOARM=${TARGETVARIANT#v} && \
-  go build -o /out/manager main.go && \
+  go build -ldflags="-X 'github.com/actions-runner-controller/actions-runner-controller/build.Version=${VERSION}'" -o /out/manager main.go && \
   go build -o /out/github-webhook-server ./cmd/githubwebhookserver
 
 # Use distroless as minimal base image to package the manager binary

Makefile

@@ -4,8 +4,8 @@ else
 	NAME ?= summerwind/actions-runner-controller
 endif
 DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
-VERSION ?= latest
-RUNNER_VERSION ?= 2.294.0
+VERSION ?= dev
+RUNNER_VERSION ?= 2.296.2
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}
@@ -92,7 +92,7 @@ manifests: manifests-gen-crds chart-crds
 manifests-gen-crds: controller-gen yq
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 	for YAMLFILE in config/crd/bases/actions*.yaml; do \
-		$(YQ) write --inplace "$$YAMLFILE" spec.preserveUnknownFields false; \
+		$(YQ) '.spec.preserveUnknownFields = false' --inplace "$$YAMLFILE" ; \
 	done
 
 chart-crds:
@@ -119,6 +119,7 @@ docker-buildx:
 	docker buildx build --platform ${PLATFORMS} \
 		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		--build-arg VERSION=${VERSION} \
 		-t "${NAME}:${VERSION}" \
 		-f Dockerfile \
 		. ${PUSH_ARG}
@@ -242,7 +243,7 @@ ifeq (, $(wildcard $(GOBIN)/yq))
 	YQ_TMP_DIR=$$(mktemp -d) ;\
 	cd $$YQ_TMP_DIR ;\
 	go mod init tmp ;\
-	go install github.com/mikefarah/yq/v3@3.4.0 ;\
+	go install github.com/mikefarah/yq/v4@v4.25.3 ;\
 	rm -rf $$YQ_TMP_DIR ;\
 	}
 endif

QuickStartGuide.md

@@ -0,0 +1,136 @@
+## Introduction
+
+GitHub Actions can be run in GitHub-hosted cloud or self hosted environments. Self-hosted runners offer more control of hardware, operating system, and software tools than GitHub-hosted runners provide.
+
+With just a few steps, you can set up your kubernetes (K8s) cluster to be a self-hosted environment.
+In this guide, we will setup prerequistes, deploy Actions Runner controller (ARC) and then target that cluster to run GitHub Action workflows.
+
+<p align="center">
+  <img src="https://user-images.githubusercontent.com/53718047/181159115-dbf41416-89a7-408c-b575-bb0d059a1a36.png" />
+</p>
+
+
+
+## Setup your K8s cluster
+
+<details><summary><sub>Create a K8s cluster, if not available.</sub></summary>
+   <sub>
+If you don't have a K8s cluster, you can install a local environment using minikube. For more information, see "[Installing minikube](https://minikube.sigs.k8s.io/docs/start/)."
+     
+     "[Using workflows](/actions/using-workflows)."
+   </sub>
+</details>
+
+:one: Install cert-manager in your cluster. For more information, see "[cert-manager](https://cert-manager.io/docs/installation/)."
+
+```shell
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.2/cert-manager.yaml
+```
+<sub> *note:- This command uses v1.8.2. Please replace with a later version, if available.</sub>
+
+
+>You may also install cert-manager using Helm. For instructions, see "[Installing with Helm](https://cert-manager.io/docs/installation/helm/#installing-with-helm)."
+
+
+:two: Next, Generate a Personal Access Token (PAT) for ARC to authenticate with GitHub.
+   - Login to GitHub account and Navigate to https://github.com/settings/tokens/new.
+   - Select  **repo**.
+   - Click **Generate Token** and then copy the token locally ( we’ll need it later).
+
+
+
+
+## Deploy and Configure ARC
+1️⃣ Deploy  and configure ARC on your K8s cluster. You may use Helm or Kubectl.
+
+
+<details><summary>Helm deployment</summary>
+
+##### Add repository
+```shell
+helm repo add actions-runner-controller https://actions-runner-controller.github.io/actions-runner-controller
+```
+
+##### Install Helm chart
+```shell
+helm upgrade --install --namespace actions-runner-system --create-namespace\
+  --set=authSecret.create=true\
+  --set=authSecret.github_token="REPLACE_YOUR_TOKEN_HERE"\
+  --wait actions-runner-controller actions-runner-controller/actions-runner-controller
+```
+<sub> *note:- Replace REPLACE_YOUR_TOKEN_HERE with your PAT that was generated in Step 1 </sub>
+</details>
+
+<details><summary>Kubectl deployment</summary>
+
+##### Deploy ARC
+```shell
+kubectl apply -f \
+https://github.com/actions-runner-controller/actions-runner-controller/\
+releases/download/v0.22.0/actions-runner-controller.yaml
+```
+<sub> *note:- Replace "v0.22.0" with the version you wish to deploy </sub>
+ 
+
+##### Configure Personal Access Token
+```shell
+kubectl create secret generic controller-manager \
+    -n actions-runner-system \
+    --from-literal=github_token=REPLACE_YOUR_TOKEN_HERE
+````
+<sub> *note:- Replace REPLACE_YOUR_TOKEN_HERE with your PAT that was generated in Step 1. </sub>
+  
+  </details>
+
+2️⃣ Create the GitHub self hosted runners and configure to run against your repository.
+
+Create a `runnerdeployment.yaml` file containing..
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeploy
+spec:
+  replicas: 1
+  template:
+    spec:
+      repository: mumoshu/actions-runner-controller-ci
+````
+<sub> *note:- Replace mumoshu/actions-runner-controller-ci with the full path to your github repository. </sub>
+
+Apply this file to your K8s cluster.
+```shell
+kubectl apply -f runnerdeployment.yaml
+````
+ 
+
+>
+>🎉 We are done - now we should have self hosted runners running in K8s configured to your repository.  🎉
+> 
+> Up Next - lets verify and execute some workflows.
+ 
+## Verify and execute workflows
+:one: Verify your setup is successful with.. 
+```shell
+$ kubectl get runners
+NAME                             REPOSITORY                             STATUS
+example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
+
+$ kubectl get pods
+NAME                           READY   STATUS    RESTARTS   AGE
+example-runnerdeploy2475ht2qbr 2/2     Running   0          1m
+````
+Also, this runner has been registered directly to the specified repository, you can see it in repository settings. For more information, see "[settings](https://docs.github.com/en/actions/hosting-your-own-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner)."
+
+:two: You are ready to execute workflows against this self hosted runner. 
+GitHub documentation lists the steps to target Actions against self hosted runners. For more information, see "[Using self-hosted runners in a workflow - GitHub Docs](https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow#using-self-hosted-runners-in-a-workflow)."
+
+There's also has a quick start guide to get started on Actions, For more information, see "[Quick start Guide to GitHub Actions](https://docs.github.com/en/actions/quickstart)."
+
+## Next steps
+ARC provides several interesting features and capabilities. For more information, see "[readme](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/README.md)."
+
+
+
+ 

README.md

@@ -29,7 +29,9 @@ ToC:
     - [Webhook Driven Scaling](#webhook-driven-scaling)
     - [Autoscaling to/from 0](#autoscaling-tofrom-0)
     - [Scheduled Overrides](#scheduled-overrides)
+  - [Alternative Runners](#alternative-runners)
     - [Runner with DinD](#runner-with-dind)
+    - [Runner with rootless DinD](#runner-with-rootless-dind)
     - [Runner with k8s jobs](#runner-with-k8s-jobs)
   - [Additional Tweaks](#additional-tweaks)
   - [Custom Volume mounts](#custom-volume-mounts)
@@ -39,6 +41,8 @@ ToC:
   - [Using IRSA (IAM Roles for Service Accounts) in EKS](#using-irsa-iam-roles-for-service-accounts-in-eks)
   - [Software Installed in the Runner Image](#software-installed-in-the-runner-image)
   - [Using without cert-manager](#using-without-cert-manager)
+  - [Windows Runners](#setting-up-windows-runners)
+  - [Multitenancy](#multitenancy)
 - [Troubleshooting](#troubleshooting)
 - [Contributing](#contributing)
 
@@ -81,8 +85,8 @@ After installing cert-manager, install the custom resource definitions and actio
 **Kubectl Deployment:**
 
 ```shell
-# REPLACE "v0.22.0" with the version you wish to deploy
-kubectl apply -f https://github.com/actions-runner-controller/actions-runner-controller/releases/download/v0.22.0/actions-runner-controller.yaml
+# REPLACE "v0.25.2" with the version you wish to deploy
+kubectl create -f https://github.com/actions-runner-controller/actions-runner-controller/releases/download/v0.25.2/actions-runner-controller.yaml
 ```
 
 **Helm Deployment:**
@@ -99,7 +103,7 @@ helm upgrade --install --namespace actions-runner-system --create-namespace \
 
 The solution supports both GHEC (GitHub Enterprise Cloud) and GHES (GitHub Enterprise Server) editions as well as regular GitHub. Both PAT (personal access token) and GitHub App authentication works for installations that will be deploying either repository level and / or organization level runners. If you need to deploy enterprise level runners then you are restricted to PAT based authentication as GitHub doesn't support GitHub App based authentication for enterprise runners currently.
 
-If you are deploying this solution into a GHES environment then you will need to be running version >= [3.3.0](https://docs.github.com/en/enterprise-server@3.3/admin/release-notes).
+If you are deploying this solution into a GHES environment then you will need to be running version >= [3.6.0](https://docs.github.com/en/enterprise-server@3.6/admin/release-notes).
 
 When deploying the solution for a GHES environment you need to provide an additional environment variable as part of the controller deployment:
 
@@ -141,16 +145,14 @@ _Note: Links are provided further down to create an app for your logged in user
 * Metadata (read)
 
 **Organization Permissions**
+
 * Self-hosted runners (read / write)
 
 _Note: All API routes mapped to their permissions can be found [here](https://docs.github.com/en/rest/reference/permissions-required-for-github-apps) if you wish to review_
 
 **Subscribe to events**
 
-At this point you have a choice of configuring a webhook, a webhook is needed if you are going to use [webhook driven scaling](#webhook-driven-scaling). The webhook can be configured centrally in the GitHub app itself or separately. In either case the event details are:
-
-* Check run (required for all webhook driven scaling events)
-* Workflow job (optionally) (required for [webhook driven scaling with workflow_job events](https://github.com/actions-runner-controller/actions-runner-controller#example-1-scale-on-each-workflow_job-event)
+At this point you have a choice of configuring a webhook, a webhook is needed if you are going to use [webhook driven scaling](#webhook-driven-scaling). The webhook can be configured centrally in the GitHub app itself or separately. In either case you need to subscribe to the `Workflow Job` event.
 
 ---
 
@@ -256,7 +258,7 @@ You can deploy multiple controllers either in a single shared namespace, or in a
 
 If you plan on installing all instances of the controller stack into a single namespace there are a few things you need to do for this to work.
 
-1. All resources per stack must have a unique, in the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
+1. All resources per stack must have a unique name, in the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
 2. `authSecret.name` needs to be unique per stack when each stack is tied to runners in different GitHub organizations and repositories AND you want your GitHub credentials to be narrowly scoped.
 3. `leaderElectionId` needs to be unique per stack. If this is not unique to the stack the controller tries to race onto the leader election lock resulting in only one stack working concurrently. Your controller will be stuck with a log message something like this `attempting to acquire leader lease arc-controllers/actions-runner-controller...`
 4. The MutatingWebhookConfiguration in each stack must include a namespace selector for that stack's corresponding runner namespace, this is already configured in the helm chart.
@@ -270,52 +272,50 @@ Alternatively, you can install each controller stack into a unique namespace (re
 - The organization level
 - The enterprise level
 
-There are two ways to use this controller:
+Runners can be deployed as 1 of 2 abstractions:
 
-- Manage runners one by one with `Runner`.
-- Manage a set of runners with `RunnerDeployment`.
+- A `RunnerDeployment` (similar to k8s's `Deployments`, based on `Pods`)
+- A `RunnerSet` (based on k8s's `StatefulSets`)
+
+We go into details about the differences between the 2 later, initially lets look at how to deploy a basic `RunnerDeployment` at the 3 possible management hierarchies.
 
 ### Repository Runners
 
-To launch a single self-hosted runner, you need to create a manifest file that includes a `Runner` resource as follows. This example launches a self-hosted runner with name *example-runner* for the *actions-runner-controller/actions-runner-controller* repository.
+To launch a single self-hosted runner, you need to create a manifest file that includes a `RunnerDeployment` resource as follows. This example launches a self-hosted runner with name *example-runnerdeploy* for the *actions-runner-controller/actions-runner-controller* repository.
 
 ```yaml
-# runner.yaml
+# runnerdeployment.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
-kind: Runner
+kind: RunnerDeployment
 metadata:
-  name: example-runner
+  name: example-runnerdeploy
 spec:
-  repository: example/myrepo
-  env: []
+  replicas: 1
+  template:
+    spec:
+      repository: mumoshu/actions-runner-controller-ci
 ```
 
 Apply the created manifest file to your Kubernetes.
 
 ```shell
-$ kubectl apply -f runner.yaml
-runner.actions.summerwind.dev/example-runner created
+$ kubectl apply -f runnerdeployment.yaml
+runnerdeployment.actions.summerwind.dev/example-runnerdeploy created
 ```
 
-You can see that the Runner resource has been created.
+You can see that 1 runner and its underlying pod has been created as specified by `replicas: 1` attribute:
 
 ```shell
 $ kubectl get runners
 NAME                             REPOSITORY                             STATUS
-example-runner   actions-runner-controller/actions-runner-controller   Running
-```
+example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
 
-You can also see that the runner pod has been running.
-
-```shell
 $ kubectl get pods
 NAME                           READY   STATUS    RESTARTS   AGE
-example-runner 2/2     Running   0          1m
+example-runnerdeploy2475ht2qbr 2/2     Running   0          1m
 ```
 
-The runner you created has been registered to your repository.
-
-<img width="756" alt="Actions tab in your repository settings" src="https://user-images.githubusercontent.com/230145/73618667-8cbf9700-466c-11ea-80b6-c67e6d3f70e7.png">
+The runner you created has been registered directly to the defined repository, you should be able to see it in the settings of the repository.
 
 Now you can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
 
@@ -324,11 +324,13 @@ Now you can use your self-hosted runner. See the [official documentation](https:
 To add the runner to an organization, you only need to replace the `repository` field with `organization`, so the runner will register itself to the organization.
 
 ```yaml
-# runner.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
-kind: Runner
+kind: RunnerDeployment
 metadata:
-  name: example-org-runner
+  name: example-runnerdeploy
+spec:
+  replicas: 1
+  template:
     spec:
       organization: your-organization-name
 ```
@@ -340,11 +342,13 @@ Now you can see the runner on the organization level (if you have organization o
 To add the runner to an enterprise, you only need to replace the `repository` field with `enterprise`, so the runner will register itself to the enterprise.
 
 ```yaml
-# runner.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
-kind: Runner
+kind: RunnerDeployment
 metadata:
-  name: example-enterprise-runner
+  name: example-runnerdeploy
+spec:
+  replicas: 1
+  template:
     spec:
       enterprise: your-enterprise-name
 ```
@@ -353,11 +357,7 @@ Now you can see the runner on the enterprise level (if you have enterprise acces
 
 ### RunnerDeployments
 
-You can manage sets of runners instead of individually through the `RunnerDeployment` kind and its `replicas:` attribute. This kind is required for many of the advanced features.
-
-There are `RunnerReplicaSet` and `RunnerDeployment` kinds that corresponds to the `ReplicaSet` and `Deployment` kinds but for the `Runner` kind.
-
-You typically only need `RunnerDeployment` rather than `RunnerReplicaSet` as the former is for managing the latter.
+In our previous examples we were deploying a single runner via the `RunnerDeployment` kind, the amount of runners deployed can be statically set via the `replicas:` field, we can increase this value to deploy additional sets of runners instead:
 
 ```yaml
 # runnerdeployment.yaml
@@ -366,11 +366,11 @@ kind: RunnerDeployment
 metadata:
   name: example-runnerdeploy
 spec:
+  # This will deploy 2 runners now
   replicas: 2
   template:
     spec:
       repository: mumoshu/actions-runner-controller-ci
-      env: []
 ```
 
 Apply the manifest file to your cluster:
@@ -393,11 +393,7 @@ example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 
 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
 
-_Ensure you see the limitations before using this kind!!!!!_
-
-For scenarios where you require the advantages of a `StatefulSet`, for example persistent storage, ARC implements a runner based on Kubernetes' `StatefulSets`, the `RunnerSet`.
-
-A basic `RunnerSet` would look like this:
+We can also deploy sets of RunnerSets the same way, a basic `RunnerSet` would look like this:
 
 ```yaml
 apiVersion: actions.summerwind.dev/v1alpha1
@@ -405,8 +401,7 @@ kind: RunnerSet
 metadata:
   name: example
 spec:
-  ephemeral: false
-  replicas: 2
+  replicas: 1
   repository: mumoshu/actions-runner-controller-ci
   # Other mandatory fields from StatefulSet
   selector:
@@ -437,8 +432,7 @@ kind: RunnerSet
 metadata:
   name: example
 spec:
-  ephemeral: false
-  replicas: 2
+  replicas: 1
   repository: mumoshu/actions-runner-controller-ci
   dockerdWithinRunnerContainer: true
   template:
@@ -486,10 +480,6 @@ You can also read the design and usage documentation written in the original pul
 
 Under the hood, `RunnerSet` relies on Kubernetes's `StatefulSet` and Mutating Webhook. A `statefulset` is used to create a number of pods that has stable names and dynamically provisioned persistent volumes, so that each `statefulset-managed` pod gets the same persistent volume even after restarting. A mutating webhook is used to dynamically inject a runner's "registration token" which is used to call GitHub's "Create Runner" API.
 
-**Limitations**
-
-* For autoscaling the `RunnerSet` kind only supports pull driven scaling or the `workflow_job` event for webhook driven scaling.
-
 ### Persistent Runners
 
 Every runner managed by ARC is "ephemeral" by default. The life of an ephemeral runner managed by ARC looks like this- ARC creates a runner pod for the runner. As it's an ephemeral runner, the `--ephemeral` flag is passed to the `actions/runner` agent that runs within the `runner` container of the runner pod.
@@ -506,11 +496,9 @@ Persistent runners are available as an option for some edge cases however they a
 
 ### Autoscaling
 
-> Since the release of GitHub's [`workflow_job` webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job), webhook driven scaling is the preferred way of autoscaling as it enables targeted scaling of your `RunnerDeployment` / `RunnerSet` as it includes the `runs-on` information needed to scale the appropriate runners for that workflow run. More broadly, webhook driven scaling is the preferred scaling option as it is far quicker compared to the pull driven scaling and is easy to set up.
+> If you are using controller version < [v0.22.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.22.0) and you are not using GHES, and so you can't set your rate limit budget, it is recommended that you use 100 replicas or fewer to prevent being rate limited.
 
-> If you are using controller version < [v0.22.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.22.0) and you are not using GHES, and so can't set your rate limit budget, it is recommended that you use 100 replicas or fewer to prevent being rate limited.
-
-A `RunnerDeployment` or `RunnerSet` can scale the number of runners between `minReplicas` and `maxReplicas` fields driven by either pull based scaling metrics or via a webhook event (see limitations section of [RunnerSets](#runnersets) for caveats of this kind). Whether the autoscaling is driven from a webhook event or pull based metrics it is implemented by backing a `RunnerDeployment` or `RunnerSet` kind with a `HorizontalRunnerAutoscaler` kind.
+A `RunnerDeployment` or `RunnerSet` can scale the number of runners between `minReplicas` and `maxReplicas` fields driven by either pull based scaling metrics or via a webhook event. Whether the autoscaling is driven from a webhook event or pull based metrics it is implemented by backing a `RunnerDeployment` or `RunnerSet` kind with a `HorizontalRunnerAutoscaler` kind.
 
 **_Important!!! If you opt to configure autoscaling, ensure you remove the `replicas:` attribute in the `RunnerDeployment` / `RunnerSet` kinds that are configured for autoscaling [#206](https://github.com/actions-runner-controller/actions-runner-controller/issues/206#issuecomment-748601907)_**
 
@@ -546,9 +534,10 @@ spec:
   # for 5 minutes instead of the default 10 minutes now
   scaleDownDelaySecondsAfterScaleOut: 300
   scaleTargetRef:
-    name: example-runner-deployment
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runner-deployment
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -574,10 +563,10 @@ metadata:
   name: example-runner-deployment-autoscaler
 spec:
   scaleTargetRef:
-    # Your RunnerDeployment Here
-    name: example-runner-deployment
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runner-deployment
   minReplicas: 1
   maxReplicas: 5
   # Your chosen scaling metrics here
@@ -618,9 +607,10 @@ metadata:
   name: example-runner-deployment-autoscaler
 spec:
   scaleTargetRef:
-    name: example-runner-deployment
-    # IMPORTANT : If your HRA is targeting a RunnerSet you must specify the kind in the scaleTargetRef:, uncomment the below
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runner-deployment
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -653,9 +643,10 @@ metadata:
   name: example-runner-deployment-autoscaler
 spec:
   scaleTargetRef:
-    name: example-runner-deployment
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runner-deployment
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -674,9 +665,10 @@ metadata:
   name: example-runner-deployment-autoscaler
 spec:
   scaleTargetRef:
-    name: example-runner-deployment
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runner-deployment
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -689,43 +681,72 @@ spec:
 
 #### Webhook Driven Scaling
 
+> This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
+
 > To configure pull driven scaling see the [Pull Driven Scaling](#pull-driven-scaling) section
 
-Webhooks are processed by a separate webhook server. The webhook server receives GitHub Webhook events and scales
-[`RunnerDeployments`](#runnerdeployments) by updating corresponding [`HorizontalRunnerAutoscalers`](#autoscaling).
+Alternatively ARC can be configured to scale based on the `workflow_job` webhook event. The primary benefit of autoscaling on webhooks compared to the pull driven scaling is that ARC is immediately notified of the scaling need.
 
-Today, the Webhook server can be configured to respond to GitHub's `check_run`, `workflow_job`, `pull_request`, and `push`  events
-by scaling up the matching `HorizontalRunnerAutoscaler` by N replica(s), where `N` is configurable within `HorizontalRunnerAutoscaler`'s `spec:`.
-
-More concretely, you can configure the targeted GitHub event types and the `N` in `scaleUpTriggers`:
+Webhooks are processed by a separate webhook server. The webhook server receives `workflow_job` webhook events and scales RunnerDeployments / RunnerSets by updating HRAs configured for the webhook trigger. Below is an example set-up where a HRA has been configured to scale a `RunnerDeployment` from a `workflow_job` event:
 
 ```yaml
-kind: HorizontalRunnerAutoscaler
-spec:
-  scaleTargetRef:
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
   name: example-runners
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+spec:
+  template:
+    spec:
+      repository: example/myrepo
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runners
+spec:
+  minReplicas: 1
+  maxReplicas: 10
+  scaleTargetRef:
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runners
   scaleUpTriggers:
     - githubEvent:
-      checkRun:
-        types: ["created"]
-        status: "queued"
-    amount: 1
-    duration: "5m"
+        workflowJob: {}
+      duration: "30m"
 ```
 
-With the above example, the webhook server scales `example-runners` by `1` replica for 5 minutes on each `check_run` event with the type of `created` and the status of `queued` received.
+The lifecycle of a runner provisioned from a webhook is different to a runner provisioned from the pull based scaling method:
 
-Of note is the `HRA.spec.scaleUpTriggers[].duration` attribute. This attribute is used to calculate if the replica number added via the trigger is expired or not. On each reconciliation loop, the controller sums up all the non-expiring replica numbers from previous scale-up triggers. It then compares the summed desired replica number against the current replica number. If the summed desired replica number > the current number then it means the replica count needs to scale up.
+1. GitHub sends a `workflow_job` event to ARC with `status=queued`
+2. ARC finds a HRA with a `workflow_job` webhook scale trigger that backs a RunnerDeployment / RunnerSet with matching runner labels
+3. The matched HRA adds a unit to its `capacityReservations` list
+4. ARC adds a replica and sets the EffectiveTime of that replica to current + `HRA.spec.scaleUpTriggers[].duration`
 
-As mentioned previously, the `scaleDownDelaySecondsAfterScaleOut` property has the final say still. If the latest scale-up time + the anti-flapping duration is later than the current time, it doesn’t immediately scale down and instead retries the calculation again later to see if it needs to scale yet.
+At this point there are a few things that can happen, either the job gets allocated to the runner or the runner is left dangling due to it not being used, if the runner gets assigned the job that triggered the scale up the lifecycle looks like this:
 
----
+1. The new runner gets allocated the job and processes it
+2. Upon the job ending GitHub sends another `workflow_job` event to ARC but with `status=completed`
+3. The HRA removes the oldest capacity reservation from its `capacityReservations` and picks a runner to terminate ensuring it isn't busy via the GitHub API beforehand
 
-The primary benefit of autoscaling on Webhooks compared to the pull driven scaling is that it is far quicker as it allows you to immediately add runner resources rather than waiting for the next sync period.
+If the job is cancelled before it is allocated to a runner then the lifecycle looks like this:
 
-> You can learn the implementation details in [#282](https://github.com/actions-runner-controller/actions-runner-controller/pull/282)
+1. Upon the job cancellation GitHub sends another `workflow_job` event to ARC but with `status=cancelled`
+2. The HRA removes the oldest capacity reservation from its `capacityReservations` and picks a runner to terminate ensuring it isn't busy via the GitHub API beforehand
+
+If runner is never used due to other runners matching needed runner group and required runner labels are allocated the job then the lifecycle looks like this:
+
+1. The scale trigger duration specified via `HRA.spec.scaleUpTriggers[].duration` elapses
+2. The HRA thinks the capacity reservation is expired, removes it from HRA's `capacityReservations` and terminates the expired runner ensuring it isn't busy via the GitHub API beforehand
+
+1. The HRA removes a capacity reservation from its `capacityReservations` and terminates the expired runner ensuring it isn't busy via the GitHub API beforehand
+
+Your `HRA.spec.scaleUpTriggers[].duration` value should be set long enough to account for the following things:
+
+1. the potential amount of time it could take for a pod to become `Running` e.g. you need to scale horizontally because there isn't a node avaliable 
+2. the amount of time it takes for GitHub to allocate a job to that runner
+3. the amount of time it takes for the runner to notice the allocated job and starts running it
 
 ##### Install with Helm
 
@@ -826,14 +847,6 @@ if you followed the example ingress above the URL would be something like this:
 
 Then click on "let me select individual events" and choose `Workflow Jobs`.
 
-You may also want to choose the following event(s) if you use it as a scale trigger in your HRA spec:
-
-- Check runs
-- Pushes
-- Pull Requests
-
-Later you can remove any of these you are not using to reduce the amount of data sent to your server.
-
 Then click on `Add Webhook`.
 
 GitHub will then send a `ping` event to your webhook server to check if it is working, if it is you'll see a green V mark
@@ -877,177 +890,6 @@ spec:
 
 ```
 
-##### Examples
-
-- [Example 1: Scale on each `workflow_job` event](#example-1-scale-on-each-workflow_job-event)
-- [Example 2: Scale up on each `check_run` event](#example-2-scale-up-on-each-check_run-event)
-- [Example 3: Scale on each `pull_request` event against a given set of branches](#example-3-scale-on-each-pull_request-event-against-a-given-set-of-branches)
-- [Example 4: Scale on each `push` event](#example-4-scale-on-each-push-event)
-
-###### Example 1: Scale on each `workflow_job` event
-
-> This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
-
-_Note: GitHub does not include the runner group information of a repository in the payload of `workflow_job` event in the initial `queued` event. The runner group information is only included for `workflow_job` events when the job has already been allocated to a runner (events with a status of `in_progress` or `completed`). Please do raise feature requests against [GitHub](https://support.github.com/tickets/personal/0) for this information to be included in the initial `queued` event if this would improve autoscaling runners for you._
-
-The most flexible webhook GitHub offers is the `workflow_job` webhook, it includes the `runs-on` information in the payload allowing scaling based on runner labels.
-
-This webhook should cover most people's needs, please experiment with this webhook first before considering the others.
-
-```yaml
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerDeployment
-metadata:
-  name: example-runners
-spec:
-  template:
-    spec:
-      repository: example/myrepo
----
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: HorizontalRunnerAutoscaler
-metadata:
-  name: example-runners
-spec:
-  scaleDownDelaySecondsAfterScaleOut: 300
-  minReplicas: 1
-  maxReplicas: 10
-  scaleTargetRef:
-    name: example-runners
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
-    #kind: RunnerSet
-  scaleUpTriggers:
-  - githubEvent:
-      workflowJob: {}
-    duration: "30m"
-```
-
-This webhook requires you to explicitly set the labels in the RunnerDeployment / RunnerSet if you are using them in your workflow to match the agents (field `runs-on`). Only `self-hosted` will be considered as included by default.
-
-You can configure your GitHub webhook settings to only include `Workflows Job` events, so that it sends us three kinds of `workflow_job` events per a job run.
-
-Each kind has a `status` of `queued`, `in_progress` and `completed`. With the above configuration, `actions-runner-controller` adds one runner for a `workflow_job` event whose `status` is `queued`. Similarly, it removes one runner for a `workflow_job` event whose `status` is `completed`. The caveat to this to remember is that this scale-down is within the bounds of your `scaleDownDelaySecondsAfterScaleOut` configuration, if this time hasn't passed the scale down will be deferred.
-
-###### Example 2: Scale up on each `check_run` event
-
-> Note: This should work almost like https://github.com/philips-labs/terraform-aws-github-runner
-
-To scale up replicas of the runners for `example/myrepo` by 1 for 5 minutes on each `check_run`, you write manifests like the below:
-
-```yaml
-kind: RunnerDeployment
-metadata:
-   name: example-runners
-spec:
-  template:
-    spec:
-      repository: example/myrepo
----
-kind: HorizontalRunnerAutoscaler
-spec:
-  minReplicas: 1
-  maxReplicas: 10
-  scaleTargetRef:
-    name: example-runners
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
-    #kind: RunnerSet
-  scaleUpTriggers:
-  - githubEvent:
-      checkRun:
-        types: ["created"]
-        status: "queued"
-    amount: 1
-    duration: "5m"
-```
-
-To scale up replicas of the runners for `myorg` organization by 1 for 5 minutes on each `check_run`, you write manifests like the below:
-
-```yaml
-kind: RunnerDeployment
-metadata:
-   name: example-runners
-spec:
-  template:
-    spec:
-      organization: myorg
----
-kind: HorizontalRunnerAutoscaler
-spec:
-  minReplicas: 1
-  maxReplicas: 10
-  scaleTargetRef:
-    name: example-runners
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
-    #kind: RunnerSet
-  scaleUpTriggers:
-  - githubEvent:
-      checkRun:
-        types: ["created"]
-        status: "queued"
-        # Optionally restrict autoscaling to being triggered by events from specific repositories within your organization still
-        # repositories: ["myrepo", "myanotherrepo"]
-    amount: 1
-    duration: "5m"
-```
-
-###### Example 3: Scale on each `pull_request` event against a given set of branches
-
-To scale up replicas of the runners for `example/myrepo` by 1 for 5 minutes on each `pull_request` against the `main` or `develop` branch you write manifests like the below:
-
-```yaml
-kind: RunnerDeployment
-metadata:
-   name: example-runners
-spec:
-  template:
-    spec:
-      repository: example/myrepo
----
-kind: HorizontalRunnerAutoscaler
-spec:
-  minReplicas: 1
-  maxReplicas: 10
-  scaleTargetRef:
-    name: example-runners
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
-    #kind: RunnerSet
-  scaleUpTriggers:
-  - githubEvent:
-      pullRequest:
-        types: ["synchronize"]
-        branches: ["main", "develop"]
-    amount: 1
-    duration: "5m"
-```
-
-See ["activity types"](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request) for the list of valid values for `scaleUpTriggers[].githubEvent.pullRequest.types`.
-
-###### Example 4: Scale on each push event
-
-To scale up replicas of the runners for `example/myrepo` by 1 for 5 minutes on each `push` write manifests like the below:
-
-```yaml
-kind: RunnerDeployment
-metadata:
-   name: example-runners
-spec:
-  repository: example/myrepo
----
-kind: HorizontalRunnerAutoscaler
-spec:
-  minReplicas: 1
-  maxReplicas: 10
-  scaleTargetRef:
-    name: example-runners
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
-    #kind: RunnerSet
-  scaleUpTriggers:
-  - githubEvent:
-      push:
-    amount: 1
-    duration: "5m"
-```
-
 #### Autoscaling to/from 0
 
 > This feature requires controller version => [v0.19.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.19.0)
@@ -1085,9 +927,10 @@ metadata:
   name: example-runner-deployment-autoscaler
 spec:
   scaleTargetRef:
-    name: example-runner-deployment
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runner-deployment
   scheduledOverrides:
   # Override minReplicas to 100 only between 2021-06-01T00:00:00+09:00 and 2021-06-03T00:00:00+09:00
   - startTime: "2021-06-01T00:00:00+09:00"
@@ -1107,9 +950,10 @@ metadata:
   name: example-runner-deployment-autoscaler
 spec:
   scaleTargetRef:
-    name: example-runner-deployment
-    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    kind: RunnerDeployment
+    # # In case the scale target is RunnerSet:
     # kind: RunnerSet
+    name: example-runner-deployment
   scheduledOverrides:
   # Override minReplicas to 0 only between 0am sat to 0am mon
   - startTime: "2021-05-01T00:00:00+09:00"
@@ -1143,9 +987,13 @@ The earlier entry is prioritized higher than later entries. So you usually defin
 
 A common use case for this may be to have 1 override to scale to 0 during the week outside of core business hours and another override to scale to 0 during all hours of the weekend.
 
-### Runner with DinD
+### Alternative Runners
 
-When using the default runner, the runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
+ARC also offers a few alternative runner options
+
+#### Runner with DinD
+
+When using the default runner, the runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). ARC maintains an alternative all in one runner image with docker running in the same container as the runner. This may be prefered from a resource or complexity perspective or to be compliant with a `LimitRange` namespace configuration.
 
 ```yaml
 # dindrunnerdeployment.yaml
@@ -1154,7 +1002,7 @@ kind: RunnerDeployment
 metadata:
   name: example-dindrunnerdeploy
 spec:
-  replicas: 2
+  replicas: 1
   template:
     spec:
       image: summerwind/actions-runner-dind
@@ -1163,9 +1011,11 @@ spec:
       env: []
 ```
 
-This also helps with resources, as you don't need to give resources separately to docker and runner.
+#### Runner with rootless DinD
 
-### Runner with K8s Jobs
+When using the DinD runner, it assumes that the main runner is rootful, which can be problematic in a regulated or more security-conscious environment, such as co-tenanting across enterprise projects.  The `actions-runner-dind-rootless` image runs rootless Docker inside the container as `runner` user.  Note that this user does not have sudo access, so anything requiring admin privileges must be built into the runner's base image (like running `apt` to install additional software).
+
+#### Runner with K8s Jobs
 
 When using the default runner, jobs that use a container will run in docker. This necessitates privileged mode, either on the runner pod or the sidecar container
 
@@ -1359,6 +1209,7 @@ spec:
 ```
 
 ### Custom Volume mounts
+
 You can configure your own custom volume mounts. For example to have the work/docker data in memory or on NVME SSD, for
 i/o intensive builds. Other custom volume mounts should be possible as well, see [kubernetes documentation](https://kubernetes.io/docs/concepts/storage/volumes/)
 
@@ -1386,7 +1237,7 @@ spec:
         - name: tmp
           emptyDir:
             medium: Memory
-      emphemeral: true # recommended to not leak data between builds.
+      ephemeral: true # recommended to not leak data between builds.
 ```
 
 #### NVME SSD
@@ -1394,7 +1245,7 @@ spec:
 In this example we provide NVME backed storage for the workdir, docker sidecar and /tmp within the runner.
 Here we use a working example on GKE, which will provide the NVME disk at /mnt/disks/ssd0.  We will be placing the respective volumes in subdirs here and in order to be able to run multiple runners we will use the pod name as a prefix for subdirectories. Also the disk will fill up over time and disk space will not be freed until the node is removed.
 
-**Beware** that running these persistent backend volumes **leave data behind** between 2 different jobs on the workdir and `/tmp` with `emphemeral: false`.
+**Beware** that running these persistent backend volumes **leave data behind** between 2 different jobs on the workdir and `/tmp` with `ephemeral: false`.
 
 ```yaml
 kind: RunnerDeployment
@@ -1435,20 +1286,22 @@ spec:
       - hostPath:
           path: /mnt/disks/ssd0
         name: tmp
-    emphemeral: true # VERY important. otherwise data inside the workdir and /tmp is not cleared between builds
+    ephemeral: true # VERY important. otherwise data inside the workdir and /tmp is not cleared between builds
 ```
 
 #### Docker image layers caching
 
 > **Note**: Ensure that the volume mount is added to the container that is running the Docker daemon.
 
-`docker` stores pulled and built image layers in the [daemon's (note not client)](https://docs.docker.com/get-started/overview/#docker-architecture) [local storage area](https://docs.docker.com/storage/storagedriver/#sharing-promotes-smaller-images) which is usually at `/var/lib/docker`.
+`docker` stores pulled and built image layers in the [daemon's (not client)](https://docs.docker.com/get-started/overview/#docker-architecture) [local storage area](https://docs.docker.com/storage/storagedriver/#sharing-promotes-smaller-images) which is usually at `/var/lib/docker`.
 
 By leveraging RunnerSet's dynamic PV provisioning feature and your CSI driver, you can let ARC maintain a pool of PVs that are
 reused across runner pods to retain `/var/lib/docker`.
 
 _Be sure to add the volume mount to the container that is supposed to run the docker daemon._
 
+_Be sure to trigger several workflow runs before checking if the cache is effective. ARC requires an `Available` PV to be reused for the new runner pod, and a PV becomes `Available` only after some time after the previous runner pod that was using the PV terminated. See [the related discussion](https://github.com/actions-runner-controller/actions-runner-controller/discussions/1605)._
+
 By default, ARC creates a sidecar container named `docker` within the runner pod for running the docker daemon. In that case,
 it's where you need the volume mount so that the manifest looks like:
 
@@ -1484,6 +1337,8 @@ With `dockerdWithinRunnerContainer: true`, you need to add the volume mount to t
 The module cache dir can be customized by setting `GOMOD_CACHE` so by setting it to somewhere under `$HOME/.cache`,
 we can have a single PV to host both build and module cache, which might improve Go module downloading and building time.
 
+_Be sure to trigger several workflow runs before checking if the cache is effective. ARC requires an `Available` PV to be reused for the new runner pod, and a PV becomes `Available` only after some time after the previous runner pod that was using the PV terminated. See [the related discussion](https://github.com/actions-runner-controller/actions-runner-controller/discussions/1605)._
+
 ```yaml
 kind: RunnerSet
 metadata:
@@ -1570,7 +1425,6 @@ jobs:
 When you have multiple kinds of self-hosted runners, you can distinguish between them using labels. In order to do so, you can specify one or more labels in your `Runner` or `RunnerDeployment` spec.
 
 ```yaml
-# runnerdeployment.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
@@ -1592,7 +1446,10 @@ jobs:
     runs-on: custom-runner
 ```
 
-Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
+When using labels there are a few things to be aware of:
+
+1. `self-hosted` is implict with every runner as this is an automatic label GitHub apply to any self-hosted runner. As a result ARC can treat all runners as having this label without having it explicitly defined in a runner's manifest. You do not need to explicitly define this label in your runner manifests (you can if you want though).
+2. In addition to the `self-hosted` label, GitHub also applies a few other [default](https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow#using-default-labels-to-route-jobs) labels to any self-hosted runner. The other default labels relate to the architecture of the runner and so can't be implicitly applied by ARC as ARC doesn't know if the runner is `linux` or `windows`, `x64` or `ARM64` etc. If you wish to use these labels in your workflows and have ARC scale runners accurately you must also add them to your runner manifests.
 
 ### Runner Groups
 
@@ -1601,7 +1458,6 @@ Runner groups can be used to limit which repositories are able to use the GitHub
 To add the runner to the group `NewGroup`, specify the group in your `Runner` or `RunnerDeployment` spec.
 
 ```yaml
-# runnerdeployment.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
@@ -1744,6 +1600,178 @@ $ helm --upgrade install actions-runner-controller/actions-runner-controller \
   admissionWebHooks.caBundle=${CA_BUNDLE}
 ```
 
+### Setting up Windows Runners
+
+The main two steps in enabling Windows self-hosted runners are:
+
+- Using `nodeSelector`'s property to filter the `cert-manger` and `actions-runner-controller` pods
+- Deploying a RunnerDeployment using a Windows-based image
+
+For the first step, you need to set the `nodeSelector.kubernetes.io/os` property in both the `cert-manager` and the `actions-runner-controller` deployments to `linux` so that the pods for these two deployments are only scheduled in Linux nodes. You can do this as follows:
+
+```yaml
+nodeSelector:
+  kubernetes.io/os: linux
+```
+
+`cert-manager` has 4 different application within it the main application, the `webhook`, the `cainjector` and the `startupapicheck`. In the parameters or values file you use for the deployment you need to add the `nodeSelector` property four times, one for each application.
+
+For the `actions-runner-controller` you only have to use the `nodeSelector` only for the main deployment, so it only has to be set once.
+
+Once this is set up, you will need to deploy two different `RunnerDeployment`'s, one for Windows and one for Linux.
+The Linux deployment can use either the default image or a custom one, however, there isn't a default Windows image so for Windows deployments you will have to build your own image.
+
+Below we share an example of the YAML used to create the deployment for each Operating System and a Dockerfile for the Windows deployment. 
+
+<details><summary>Windows</summary>
+<p>
+
+#### RunnerDeployment
+
+```yaml
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: k8s-runners-windows
+  namespace: actions-runner-system
+spec:
+  template:
+    spec:
+      image: <repo>/<image>:<windows-tag>
+      dockerdWithinRunnerContainer: true
+      nodeSelector:
+        kubernetes.io/os: windows
+        kubernetes.io/arch: amd64
+      repository: <owner>/<repo>
+      labels:
+        - windows
+        - X64
+        - devops-managed
+```
+
+#### Dockerfile
+
+> Note that you'd need to patch the below Dockerfile if you need a graceful termination.
+> See https://github.com/actions-runner-controller/actions-runner-controller/pull/1608/files#r917319574 for more information.
+
+```Dockerfile
+FROM mcr.microsoft.com/windows/servercore:ltsc2019
+
+WORKDIR /actions-runner
+
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';$ProgressPreference='silentlyContinue';"]
+
+RUN Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v2.292.0/actions-runner-win-x64-2.292.0.zip -OutFile actions-runner-win-x64-2.292.0.zip
+
+RUN if((Get-FileHash -Path actions-runner-win-x64-2.292.0.zip -Algorithm SHA256).Hash.ToUpper() -ne 'f27dae1413263e43f7416d719e0baf338c8d80a366fed849ecf5fffcec1e941f'.ToUpper()){ throw 'Computed checksum did not match' }
+
+RUN Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory('actions-runner-win-x64-2.292.0.zip', $PWD)
+
+RUN Invoke-WebRequest -Uri 'https://aka.ms/install-powershell.ps1' -OutFile install-powershell.ps1; ./install-powershell.ps1 -AddToPath
+
+RUN powershell Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+RUN powershell choco install git.install --params "'/GitAndUnixToolsOnPath'" -y
+
+RUN powershell choco feature enable -n allowGlobalConfirmation
+
+CMD [ "pwsh", "-c", "./config.cmd --name $env:RUNNER_NAME --url https://github.com/$env:RUNNER_REPO --token $env:RUNNER_TOKEN --labels $env:RUNNER_LABELS --unattended --replace --ephemeral; ./run.cmd"]
+```
+</p>
+</details>
+
+
+<details><summary>Linux</summary>
+<p>
+
+#### RunnerDeployment
+
+```yaml
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: k8s-runners-linux
+  namespace: actions-runner-system
+spec:
+  template:
+    spec:
+      image: <repo>/<image>:<linux-tag>
+      nodeSelector:
+        kubernetes.io/os: linux
+        kubernetes.io/arch: amd64
+      repository: <owner>:<repo>
+      labels:
+        - linux
+        - X64
+        - devops-managed
+```
+</p>
+</details>
+
+After both `RunnerDeployment`'s are up and running, you can now proceed to deploy the `HorizontalRunnerAutoscaler` for each deployment.
+
+### Multitenancy
+
+> This feature requires controller version => [v0.26.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.26.0)
+
+In a large enterprise, there might be many GitHub organizations that requires self-hosted runners. Previously, the only way to provide ARC-managed self-hosted runners in such environment was [Deploying Multiple Controllers](#deploying-multiple-controllers), which incurs overhead due to it requires one ARC installation per GitHub organization.
+
+With multitenancy, you can let ARC manage self-hosted runners across organizations. It's enabled by default and the only thing you need to start using it is to set the `spec.githubAPICredentialsFrom.secretRef.name` fields for the following resources:
+
+- `HorizontalRunnerAutoscaler`
+- `RunnerSet`
+
+Or `spec.template.spec.githubAPICredentialsFrom.secretRef.name` field for the following resource:
+
+- `RunnerDeployment`
+
+> Although not explained above, `spec.githubAPICredentialsFrom` fields do exist in `Runner` and `RunnerReplicaSet`. A comparable pod annotation exists for the runner pod, too.
+> However, note that `Runner`, `RunnerReplicaSet` and runner pods are implementation details and are managed by `RunnerDeployment` and ARC.
+> Usually you don't need to manually set the fields for those resources.
+
+`githubAPICredentialsFrom.secretRef.name` should refer to the name of the Kubernetes secret that contains either PAT or GitHub App credentials that is used for GitHub API calls for the said resource.
+
+Usually, you should have a set of GitHub App credentials per a GitHub organization and you would have a RunnerDeployment and a HorizontalRunnerAutoscaler per an organization runner group. So, you might end up having the following resources for each organization:
+
+- 1 Kubernetes secret that contains GitHub App credentials
+- 1 RunnerDeployment/RunnerSet and 1 HorizontalRunnerAutoscaler per Runner Group
+
+And the RunnerDeployment/RunnerSet and HorizontalRunnerAutoscaler should have the same value for `spec.githubAPICredentialsFrom.secretRef.name`, which refers to the name of the Kubernetes secret.
+
+```yaml
+kind: Secret
+data:
+  github_app_id: ...
+  github_app_installation_id: ...
+  github_app_private_key: ...
+---
+kind: RunnerDeployment
+metadata:
+  namespace: org1-runners
+spec:
+  template:
+    spec:
+      githubAPICredentialsFrom:
+        secretRef:
+          name: org1-github-app
+---
+kind: HorizontalRunnerAutoscaler
+metadata:
+  namespace: org1-runners
+spec:
+  githubAPICredentialsFrom:
+    secretRef:
+      name: org1-github-app
+```
+
+> Do note that, as shown in the above example, you usually set the same secret name to `githubAPICredentialsFrom.secretRef.name` fields of both `RunnerDeployment` and `HorizontalRunnerAutoscaler`, so that GitHub API calls for the same set of runners shares the specified credentials, regardless of
+when and which varying ARC component(`horizontalrunnerautoscaler-controller`, `runnerdeployment-controller`, `runnerreplicaset-controller`, `runner-controller` or `runnerpod-controller`) makes specific API calls.
+> Just don't be surprised you have to repeat `githubAPICredentialsFrom.secretRef.name` settings among two resources!
+
+Please refer to [Deploying Using GitHub App Authentication](#deploying-using-github-app-authentication) for how you could create the Kubernetes secret containing GitHub App credentials.
+
 # Troubleshooting
 
 See [troubleshooting guide](TROUBLESHOOTING.md) for solutions to various problems people have run into consistently.

acceptance/deploy.sh

@@ -41,8 +41,23 @@ TEST_ID=${TEST_ID:-default}
 
 if [ "${tool}" == "helm" ]; then
   set -v
+
+  CHART=${CHART:-charts/actions-runner-controller}
+
+  flags=()
+  if [ "${IMAGE_PULL_SECRET}" != "" ]; then
+    flags+=( --set imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set image.actionsRunnerImagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+    flags+=( --set githubWebhookServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET})
+  fi
+  if [ "${CHART_VERSION}" != "" ]; then
+    flags+=( --version ${CHART_VERSION})
+  fi
+
+  set -vx
+
   helm upgrade --install actions-runner-controller \
-    charts/actions-runner-controller \
+    ${CHART} \
     -n actions-runner-system \
     --create-namespace \
     --set syncPeriod=${SYNC_PERIOD} \
@@ -51,9 +66,7 @@ if [ "${tool}" == "helm" ]; then
     --set image.tag=${VERSION} \
     --set podAnnotations.test-id=${TEST_ID} \
     --set githubWebhookServer.podAnnotations.test-id=${TEST_ID} \
-    --set imagePullSecrets[0].name=${IMAGE_PULL_SECRET} \
-    --set image.actionsRunnerImagePullSecrets[0].name=${IMAGE_PULL_SECRET} \
-    --set githubWebhookServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET} \
+    ${flags[@]} --set image.imagePullPolicy=${IMAGE_PULL_POLICY} \
     -f ${VALUES_FILE}
   set +v
   # To prevent `CustomResourceDefinition.apiextensions.k8s.io "runners.actions.summerwind.dev" is invalid: metadata.annotations: Too long: must have at most 262144 bytes`

acceptance/deploy_runners.sh

@@ -6,6 +6,8 @@ OP=${OP:-apply}
 
 RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
 
+cat acceptance/testdata/kubernetes_container_mode.envsubst.yaml  | NAMESPACE=${RUNNER_NAMESPACE} envsubst  | kubectl apply -f -
+
 if [ -n "${TEST_REPO}" ]; then
   if [ "${USE_RUNNERSET}" != "false" ]; then
     cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl ${OP} -f -

acceptance/testdata/kubernetes_container_mode.envsubst.yaml

@@ -0,0 +1,86 @@
+# USAGE:
+#   cat acceptance/testdata/kubernetes_container_mode.envsubst.yaml  | NAMESPACE=default envsubst  | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: k8s-mode-runner
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["get", "create"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get", "list", "watch",]
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "create", "delete"]
+# Needed to report test success by crating a cm from within workflow job step
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: runner-status-updater
+rules:
+- apiGroups: ["actions.summerwind.dev"]
+  resources: ["runners/status"]
+  verbs: ["get", "update", "patch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+---
+# To verify it's working, try:
+#   kubectl auth can-i --as system:serviceaccount:default:runner get pod
+# If incomplete, workflows and jobs would fail with an error message like:
+#   Error: Error: The Service account needs the following permissions [{"group":"","verbs":["get","list","create","delete"],"resource":"pods","subresource":""},{"group":"","verbs":["get","create"],"resource":"pods","subresource":"exec"},{"group":"","verbs":["get","list","watch"],"resource":"pods","subresource":"log"},{"group":"batch","verbs":["get","list","create","delete"],"resource":"jobs","subresource":""},{"group":"","verbs":["create","delete","get","list"],"resource":"secrets","subresource":""}] on the pod resource in the 'default' namespace. Please contact your self hosted runner administrator.
+#   Error: Process completed with exit code 1.
+apiVersion: rbac.authorization.k8s.io/v1
+# This role binding allows "jane" to read pods in the "default" namespace.
+# You need to already have a Role named "pod-reader" in that namespace.
+kind: RoleBinding
+metadata:
+  name: runner-k8s-mode-runner
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: k8s-mode-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: runner-runner-stat-supdater
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: ${RUNNER_SERVICE_ACCOUNT_NAME}
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: runner-status-updater
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: org-runnerdeploy-runner-work-dir
+  labels:
+    content: org-runnerdeploy-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

acceptance/testdata/runnerdeploy.envsubst.yaml

@@ -1,3 +1,13 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ${NAME}-runner-work-dir
+  labels:
+    content: ${NAME}-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
@@ -39,10 +49,26 @@ spec:
       labels:
       - "${RUNNER_LABEL}"
 
+      env:
+      - name: ROLLING_UPDATE_PHASE
+        value: "${ROLLING_UPDATE_PHASE}"
+
       #
       # Non-standard working directory
       #
       # workDir: "/"
+
+      # # Uncomment the below to enable the kubernetes container mode
+      # # See https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs
+      containerMode: ${RUNNER_CONTAINER_MODE}
+      workVolumeClaimTemplate:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: "${NAME}-runner-work-dir"
+        resources:
+          requests:
+            storage: 10Gi
+      serviceAccountName: ${RUNNER_SERVICE_ACCOUNT_NAME}
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: HorizontalRunnerAutoscaler

acceptance/testdata/runnerset.envsubst.yaml

@@ -112,6 +112,7 @@ spec:
       labels:
         app: ${NAME}
     spec:
+      serviceAccountName: ${RUNNER_SERVICE_ACCOUNT_NAME}
       containers:
       - name: runner
         imagePullPolicy: IfNotPresent
@@ -120,6 +121,8 @@ spec:
           value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
         - name: GOMODCACHE
           value: "/home/runner/.cache/go-mod"
+        - name: ROLLING_UPDATE_PHASE
+          value: "${ROLLING_UPDATE_PHASE}"
         # PV-backed runner work dir
         volumeMounts:
         # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
@@ -152,19 +155,19 @@ spec:
           # https://github.com/actions/setup-go/blob/56a61c9834b4a4950dbbf4740af0b8a98c73b768/src/installer.ts#L144
           mountPath: "/opt/hostedtoolcache"
       # Valid only when dockerdWithinRunnerContainer=false
-      - name: docker
-        # PV-backed runner work dir
-        volumeMounts:
-        - name: work
-          mountPath: /runner/_work
-        # Cache docker image layers, in case dockerdWithinRunnerContainer=false
-        - name: var-lib-docker
-          mountPath: /var/lib/docker
-        # image: mumoshu/actions-runner-dind:dev
+      # - name: docker
+      #   # PV-backed runner work dir
+      #   volumeMounts:
+      #   - name: work
+      #     mountPath: /runner/_work
+      #   # Cache docker image layers, in case dockerdWithinRunnerContainer=false
+      #   - name: var-lib-docker
+      #     mountPath: /var/lib/docker
+      #   # image: mumoshu/actions-runner-dind:dev
 
-        # For buildx cache
-        - name: cache
-          mountPath: "/home/runner/.cache"
+      #   # For buildx cache
+      #   - name: cache
+      #     mountPath: "/home/runner/.cache"
         # Comment out the ephemeral work volume if you're going to test the kubernetes container mode
       # volumes:
       # - name: work

acceptance/values.yaml

@@ -1,13 +1,18 @@
 # Set actions-runner-controller settings for testing
 logLevel: "-4"
-imagePullSecrets:
-- name:
+imagePullSecrets: []
 image:
-  actionsRunnerImagePullSecrets:
-  - name:
+  # This needs to be an empty array rather than a single-item array with empty name.
+  # Otherwise you end up with the following error on helm-upgrade:
+  #   Error: UPGRADE FAILED: failed to create patch: map: map[] does not contain declared merge key: name && failed to create patch: map: map[] does not contain declared merge key: name
+  actionsRunnerImagePullSecrets: []
+runner:
+  statusUpdateHook:
+    enabled: true
+rbac:
+  allowGrantingKubernetesContainerModePermissions: true
 githubWebhookServer:
-  imagePullSecrets:
-  - name:
+  imagePullSecrets: []
   logLevel: "-4"
   enabled: true
   labels: {}

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -60,6 +60,9 @@ type HorizontalRunnerAutoscalerSpec struct {
 	// The earlier a scheduled override is, the higher it is prioritized.
 	// +optional
 	ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
+
+	// +optional
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
 }
 
 type ScaleUpTrigger struct {
@@ -130,7 +133,7 @@ type ScaleTargetRef struct {
 
 type MetricSpec struct {
 	// Type is the type of metric to be used for autoscaling.
-	// The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+	// It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
 	Type string `json:"type,omitempty"`
 
 	// RepositoryNames is the list of repository names to be used for calculating the metric.
@@ -170,7 +173,7 @@ type MetricSpec struct {
 }
 
 // ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule.
-// A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+// A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
 type ScheduledOverride struct {
 	// StartTime is the time at which the first override starts.
 	StartTime metav1.Time `json:"startTime"`

api/v1alpha1/runner_types.go

@@ -76,6 +76,16 @@ type RunnerConfig struct {
 
 	// +optional
 	ContainerMode string `json:"containerMode,omitempty"`
+
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
+}
+
+type GitHubAPICredentialsFrom struct {
+	SecretRef SecretReference `json:"secretRef,omitempty"`
+}
+
+type SecretReference struct {
+	Name string `json:"name"`
 }
 
 // RunnerPodSpec defines the desired pod spec fields of the runner pod
@@ -183,11 +193,6 @@ func (rs *RunnerSpec) Validate(rootPath *field.Path) field.ErrorList {
 		errList = append(errList, field.Invalid(rootPath.Child("workVolumeClaimTemplate"), rs.WorkVolumeClaimTemplate, err.Error()))
 	}
 
-	err = rs.validateIsServiceAccountNameSet()
-	if err != nil {
-		errList = append(errList, field.Invalid(rootPath.Child("serviceAccountName"), rs.ServiceAccountName, err.Error()))
-	}
-
 	return errList
 }
 
@@ -226,17 +231,6 @@ func (rs *RunnerSpec) validateWorkVolumeClaimTemplate() error {
 	return rs.WorkVolumeClaimTemplate.validate()
 }
 
-func (rs *RunnerSpec) validateIsServiceAccountNameSet() error {
-	if rs.ContainerMode != "kubernetes" {
-		return nil
-	}
-
-	if rs.ServiceAccountName == "" {
-		return errors.New("service account name is required if container mode is kubernetes")
-	}
-	return nil
-}
-
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
 	// Turns true only if the runner pod is ready.
@@ -315,8 +309,10 @@ func (w *WorkVolumeClaimTemplate) V1VolumeMount(mountPath string) corev1.VolumeM
 // +kubebuilder:printcolumn:JSONPath=".spec.enterprise",name=Enterprise,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.organization",name=Organization,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.repository",name=Repository,type=string
+// +kubebuilder:printcolumn:JSONPath=".spec.group",name=Group,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.labels",name=Labels,type=string
 // +kubebuilder:printcolumn:JSONPath=".status.phase",name=Status,type=string
+// +kubebuilder:printcolumn:JSONPath=".status.message",name=Message,type=string
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
 // Runner is the Schema for the runners API

api/v1alpha1/runnerdeployment_types.go

@@ -33,7 +33,7 @@ type RunnerDeploymentSpec struct {
 
 	// EffectiveTime is the time the upstream controller requested to sync Replicas.
 	// It is usually populated by the webhook-based autoscaler via HRA.
-	// The value is inherited to RunnerRepicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
+	// The value is inherited to RunnerReplicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
 	//
 	// +optional
 	// +nullable

api/v1alpha1/zz_generated.deepcopy.go

@@ -90,6 +90,22 @@ func (in *CheckRunSpec) DeepCopy() *CheckRunSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GitHubAPICredentialsFrom) DeepCopyInto(out *GitHubAPICredentialsFrom) {
+	*out = *in
+	out.SecretRef = in.SecretRef
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubAPICredentialsFrom.
+func (in *GitHubAPICredentialsFrom) DeepCopy() *GitHubAPICredentialsFrom {
+	if in == nil {
+		return nil
+	}
+	out := new(GitHubAPICredentialsFrom)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTriggerSpec) {
 	*out = *in
@@ -231,6 +247,11 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
@@ -425,6 +446,11 @@ func (in *RunnerConfig) DeepCopyInto(out *RunnerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerConfig.
@@ -1136,6 +1162,21 @@ func (in *ScheduledOverride) DeepCopy() *ScheduledOverride {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretReference)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WorkVolumeClaimTemplate) DeepCopyInto(out *WorkVolumeClaimTemplate) {
 	*out = *in

build/version.go

@@ -0,0 +1,4 @@
+package build
+
+// This is overridden at build-time using go-build ldflags. dev is the fallback value
+var Version = "NA"

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.20.0
+version: 0.20.2
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.25.0
+appVersion: 0.25.2
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/README.md

@@ -9,7 +9,7 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 > _Default values are the defaults set in the charts `values.yaml`, some properties have default configurations in the code for when the property is omitted or invalid_
 
 | Key                                                      | Description                                                                                                                | Default                                                                                         |
-|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------|
+|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|
 | `labels`                                                 | Set labels to apply to all resources in the chart                                                                          |                                                                                                 |
 | `replicaCount`                                           | Set the number of controller pods                                                                                          | 1                                                                                               |
 | `webhookPort`                                            | Set the containerPort for the webhook Pod                                                                                  | 9443                                                                                            |
@@ -73,12 +73,13 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `scope.watchNamespace`                                   | Tells the controller and the github webhook server which namespace to watch if `scope.singleNamespace` is true             | `Release.Namespace` (the default namespace of the helm chart).                                  |
 | `scope.singleNamespace`                                  | Limit the controller to watch a single namespace                                                                           | false                                                                                           |
 | `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                   | true                                                                                            |
+| `runner.statusUpdateHook.enabled`                        | Use custom RBAC for runners (role, role binding and service account), this will enable reporting runner statuses           | false                                                                                           |
 | `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                  |                                                                                                 |
 | `githubWebhookServer.logLevel`                           | Set the log level of the githubWebhookServer container                                                                     |                                                                                                 |
 | `githubWebhookServer.replicaCount`                       | Set the number of webhook server pods                                                                                      | 1                                                                                               |
 | `githubWebhookServer.useRunnerGroupsVisibility`          | Enable supporting runner groups with custom visibility. This will incur in extra API calls and may blow up your budget. Currently, you also need to set `githubWebhookServer.secret.enabled` to enable this feature. | false |
-| `githubWebhookServer.syncPeriod`                         | Set the period in which the controller reconciles the resources                                                            | 10m                                                                  |
 | `githubWebhookServer.enabled`                            | Deploy the webhook server pod                                                                                              | false                                                                                           |
+| `githubWebhookServer.queueLimit`                         | Set the queue size limit in the githubWebhookServer                                                                        |                                                                                                 |
 | `githubWebhookServer.secret.enabled`                     | Passes the webhook hook secret to the github-webhook-server                                                                | false                                                                                           |
 | `githubWebhookServer.secret.create`                      | Deploy the webhook hook secret                                                                                             | false                                                                                           |
 | `githubWebhookServer.secret.name`                        | Set the name of the webhook hook secret                                                                                    | github-webhook-server                                                                           |

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer
@@ -92,7 +102,7 @@ spec:
                         description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up.
                         type: string
                       type:
-                        description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                        description: Type is the type of metric to be used for autoscaling. It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
                         type: string
                     type: object
                   type: array
@@ -170,7 +180,7 @@ spec:
                 scheduledOverrides:
                   description: ScheduledOverrides is the list of ScheduledOverride. It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. The earlier a scheduled override is, the higher it is prioritized.
                   items:
-                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
                     properties:
                       endTime:
                         description: EndTime is the time at which the first override ends.

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml

@@ -49,7 +49,7 @@ spec:
               description: RunnerDeploymentSpec defines the desired state of RunnerDeployment
               properties:
                 effectiveTime:
-                  description: EffectiveTime is the time the upstream controller requested to sync Replicas. It is usually populated by the webhook-based autoscaler via HRA. The value is inherited to RunnerRepicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
+                  description: EffectiveTime is the time the upstream controller requested to sync Replicas. It is usually populated by the webhook-based autoscaler via HRA. The value is inherited to RunnerReplicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
                   format: date-time
                   nullable: true
                   type: string
@@ -2415,6 +2415,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml

@@ -2412,6 +2412,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml

@@ -24,12 +24,18 @@ spec:
         - jsonPath: .spec.repository
           name: Repository
           type: string
+        - jsonPath: .spec.group
+          name: Group
+          type: string
         - jsonPath: .spec.labels
           name: Labels
           type: string
         - jsonPath: .status.phase
           name: Status
           type: string
+        - jsonPath: .status.message
+          name: Message
+          type: string
         - jsonPath: .metadata.creationTimestamp
           name: Age
           type: date
@@ -2353,6 +2359,16 @@ spec:
                       - name
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 hostAliases:

charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml

@@ -67,6 +67,16 @@ spec:
                   type: string
                 ephemeral:
                   type: boolean
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 image:

charts/actions-runner-controller/templates/controller.metrics.serviceMonitor.yaml

@@ -8,6 +8,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   name: {{ include "actions-runner-controller.serviceMonitorName" . }}
+  namespace: {{ .Release.Namespace }}
 spec:
   endpoints:
     - path: /metrics

charts/actions-runner-controller/templates/deployment.yaml

@@ -58,15 +58,15 @@ spec:
         {{- if .Values.scope.singleNamespace }}
         - "--watch-namespace={{ default .Release.Namespace .Values.scope.watchNamespace }}"
         {{- end }}
-        {{- if .Values.githubAPICacheDuration }}
-        - "--github-api-cache-duration={{ .Values.githubAPICacheDuration }}"
-        {{- end }}
         {{- if .Values.logLevel }}
         - "--log-level={{ .Values.logLevel }}"
         {{- end }}
         {{- if .Values.runnerGithubURL  }}
         - "--runner-github-url={{ .Values.runnerGithubURL }}"
         {{- end }}
+        {{- if .Values.runner.statusUpdateHook.enabled }}
+        - "--runner-status-update-hook"
+        {{- end }}
         command:
         - "/manager"
         env:
@@ -118,10 +118,14 @@ spec:
               name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         {{- end }}
+        {{- if kindIs "slice" .Values.env }}
+        {{- toYaml .Values.env | nindent 8 }}
+        {{- else }}
         {{- range $key, $val := .Values.env }}
         - name: {{ $key }}
           value: {{ $val | quote }}
         {{- end }}
+        {{- end }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
         name: manager
         imagePullPolicy: {{ .Values.image.pullPolicy }}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -39,7 +39,6 @@ spec:
         {{- $metricsHost := .Values.metrics.proxy.enabled | ternary "127.0.0.1" "0.0.0.0" }}
         {{- $metricsPort := .Values.metrics.proxy.enabled | ternary "8080" .Values.metrics.port }}
         - "--metrics-addr={{ $metricsHost }}:{{ $metricsPort }}"
-        - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}"
         {{- if .Values.githubWebhookServer.logLevel }}
         - "--log-level={{ .Values.githubWebhookServer.logLevel }}"
         {{- end }}
@@ -49,6 +48,9 @@ spec:
         {{- if .Values.runnerGithubURL  }}
         - "--runner-github-url={{ .Values.runnerGithubURL }}"
         {{- end }}
+        {{- if .Values.githubWebhookServer.queueLimit }}
+        - "--queue-limit={{ .Values.githubWebhookServer.queueLimit }}"
+        {{- end }}
         command:
         - "/github-webhook-server"
         env:

charts/actions-runner-controller/templates/githubwebhook.ingress.yaml

@@ -1,13 +1,7 @@
 {{- if .Values.githubWebhookServer.ingress.enabled -}}
 {{- $fullName := include "actions-runner-controller-github-webhook-server.fullname" . -}}
 {{- $svcPort := (index .Values.githubWebhookServer.service.ports 0).port -}}
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }}
 apiVersion: networking.k8s.io/v1
-{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/Ingress" }}
-apiVersion: networking.k8s.io/v1beta1
-{{- else if .Capabilities.APIVersions.Has "extensions/v1beta1/Ingress" }}
-apiVersion: extensions/v1beta1
-{{- end }}
 kind: Ingress
 metadata:
   name: {{ $fullName }}
@@ -42,19 +36,12 @@ spec:
           {{- end }}
           {{- range .paths }}
           - path: {{ .path }}
-            {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }}
             pathType: {{ .pathType }}
-            {{- end }}
             backend:
-              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }}
               service:
                name: {{ $fullName }}
                port:
                  number: {{ $svcPort }}
-              {{- else }}
-              serviceName: {{ $fullName }}
-              servicePort: {{ $svcPort }}
-              {{- end }}
           {{- end }}
     {{- end }}
   {{- end }}

charts/actions-runner-controller/templates/githubwebhook.serviceMonitor.yaml

@@ -8,6 +8,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   name: {{ include "actions-runner-controller-github-webhook-server.serviceMonitorName" . }}
+  namespace: {{ .Release.Namespace }}
 spec:
   endpoints:
     - path: /metrics

charts/actions-runner-controller/templates/manager_role.yaml

@@ -258,3 +258,64 @@ rules:
   - get
   - list
   - watch
+{{- if .Values.runner.statusUpdateHook.enabled }}
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get
+{{- end }}
+{{- if .Values.rbac.allowGrantingKubernetesContainerModePermissions }}
+{{/* These permissions are required by ARC to create RBAC resources for the runner pod to use the kubernetes container mode. */}}
+{{/* See https://github.com/actions-runner-controller/actions-runner-controller/pull/1268/files#r917331632 */}}
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/log
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "batch"
+  resources:
+  - jobs
+  verbs:
+  - get
+  - list
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+{{- end }}

charts/actions-runner-controller/values.yaml

@@ -15,12 +15,6 @@ enableLeaderElection: true
 # Must be unique if more than one controller installed onto the same namespace.
 #leaderElectionId: "actions-runner-controller"
 
-# DEPRECATED: This has been removed as unnecessary in #1192
-# The controller tries its best not to repeat the duplicate GitHub API call
-# within this duration.
-# Defaults to syncPeriod - 10s.
-#githubAPICacheDuration: 30s
-
 # The URL of your GitHub Enterprise server, if you're using one.
 #githubEnterpriseServerURL: https://github.example.com
 
@@ -67,6 +61,18 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+runner:
+  statusUpdateHook:
+    enabled: false
+
+rbac:
+  {}
+  # # This allows ARC to dynamically create a ServiceAccount and a Role for each Runner pod that uses "kubernetes" container mode,
+  # # by extending ARC's manager role to have the same permissions required by the pod runs the runner agent in "kubernetes" container mode.
+  # # Without this, Kubernetes blocks ARC to create the role to prevent a priviledge escalation.
+  # # See https://github.com/actions-runner-controller/actions-runner-controller/pull/1268/files#r917327010
+  # allowGrantingKubernetesContainerModePermissions: true
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true
@@ -143,10 +149,20 @@ priorityClassName: ""
 
 env:
   {}
+  # specify additional environment variables for the controller pod.
+  # It's possible to specify either key vale pairs e.g.:
   # http_proxy: "proxy.com:8080"
   # https_proxy: "proxy.com:8080"
   # no_proxy: ""
 
+  # or a list of complete environment variable definitions e.g.:
+  # - name: GITHUB_APP_INSTALLATION_ID
+  #   valueFrom:
+  #     secretKeyRef:
+  #       key: some_key_in_the_secret
+  #       name: some-secret-name
+  #       optional: true
+
 ## specify additional volumes to mount in the manager container, this can be used
 ## to specify additional storage of material or to inject files from ConfigMaps
 ## into the running container
@@ -175,7 +191,6 @@ admissionWebHooks:
 githubWebhookServer:
   enabled: false
   replicaCount: 1
-  syncPeriod: 10m
   useRunnerGroupsVisibility: false
   secret:
     enabled: false
@@ -255,3 +270,4 @@ githubWebhookServer:
     enabled: false
     # minAvailable: 1
     # maxUnavailable: 3
+  # queueLimit: 100

cmd/githubwebhookserver/main.go

@@ -69,8 +69,6 @@ func main() {
 
 		watchNamespace string
 
-		enableLeaderElection bool
-		syncPeriod           time.Duration
 		logLevel   string
 		queueLimit int
 
@@ -89,9 +87,6 @@ func main() {
 	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&watchNamespace, "watch-namespace", "", "The namespace to watch for HorizontalRunnerAutoscaler's to scale on Webhook. Set to empty for letting it watch for all namespaces.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
-		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
-	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
 	flag.IntVar(&queueLimit, "queue-limit", controllers.DefaultQueueLimit, `The maximum length of the scale operation queue. The scale opration is enqueued per every matching webhook event, and the server returns a 500 HTTP status when the queue was already full on enqueue attempt.`)
 	flag.StringVar(&webhookSecretToken, "github-webhook-secret-token", "", "The personal access token of GitHub.")
@@ -144,10 +139,10 @@ func main() {
 		setupLog.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls")
 	}
 
+	syncPeriod := 10 * time.Minute
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		SyncPeriod:         &syncPeriod,
-		LeaderElection:     enableLeaderElection,
 		Namespace:          watchNamespace,
 		MetricsBindAddress: metricsAddr,
 		Port:               9443,

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer
@@ -92,7 +102,7 @@ spec:
                         description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up.
                         type: string
                       type:
-                        description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                        description: Type is the type of metric to be used for autoscaling. It can be TotalNumberOfQueuedAndInProgressWorkflowRuns or PercentageRunnersBusy.
                         type: string
                     type: object
                   type: array
@@ -170,7 +180,7 @@ spec:
                 scheduledOverrides:
                   description: ScheduledOverrides is the list of ScheduledOverride. It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. The earlier a scheduled override is, the higher it is prioritized.
                   items:
-                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the correspoding override happens every day, week, month, or year.
+                    description: ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally be recurring, so that the corresponding override happens every day, week, month, or year.
                     properties:
                       endTime:
                         description: EndTime is the time at which the first override ends.

config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml

@@ -49,7 +49,7 @@ spec:
               description: RunnerDeploymentSpec defines the desired state of RunnerDeployment
               properties:
                 effectiveTime:
-                  description: EffectiveTime is the time the upstream controller requested to sync Replicas. It is usually populated by the webhook-based autoscaler via HRA. The value is inherited to RunnerRepicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
+                  description: EffectiveTime is the time the upstream controller requested to sync Replicas. It is usually populated by the webhook-based autoscaler via HRA. The value is inherited to RunnerReplicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
                   format: date-time
                   nullable: true
                   type: string
@@ -2415,6 +2415,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml

@@ -2412,6 +2412,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -24,12 +24,18 @@ spec:
         - jsonPath: .spec.repository
           name: Repository
           type: string
+        - jsonPath: .spec.group
+          name: Group
+          type: string
         - jsonPath: .spec.labels
           name: Labels
           type: string
         - jsonPath: .status.phase
           name: Status
           type: string
+        - jsonPath: .status.message
+          name: Message
+          type: string
         - jsonPath: .metadata.creationTimestamp
           name: Age
           type: date
@@ -2353,6 +2359,16 @@ spec:
                       - name
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 hostAliases:

config/crd/bases/actions.summerwind.dev_runnersets.yaml

@@ -67,6 +67,16 @@ spec:
                   type: string
                 ephemeral:
                   type: boolean
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 image:

config/rbac/role.yaml

@@ -258,3 +258,27 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get

controllers/autoscaling.go

@@ -9,7 +9,9 @@ import (
 	"strings"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v45/github"
+	prometheus_metrics "github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
+	arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/google/go-github/v47/github"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -21,7 +23,7 @@ const (
 	defaultScaleDownFactor    = 0.7
 )
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	if hra.Spec.MinReplicas == nil {
 		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
 	} else if hra.Spec.MaxReplicas == nil {
@@ -48,9 +50,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 
 	switch primaryMetricType {
 	case v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns:
-		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &primaryMetric)
+		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &primaryMetric)
 	case v1alpha1.AutoscalingMetricTypePercentageRunnersBusy:
-		suggested, err = r.suggestReplicasByPercentageRunnersBusy(st, hra, primaryMetric)
+		suggested, err = r.suggestReplicasByPercentageRunnersBusy(ghc, st, hra, primaryMetric)
 	default:
 		return nil, fmt.Errorf("validating autoscaling metrics: unsupported metric type %q", primaryMetric)
 	}
@@ -83,11 +85,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 		)
 	}
 
-	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &fallbackMetric)
+	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &fallbackMetric)
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
-
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
 	var repos [][]string
 	repoID := st.repo
 	if repoID == "" {
@@ -126,7 +127,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 		opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}}
 		var allJobs []*github.WorkflowJob
 		for {
-			jobs, resp, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
+			jobs, resp, err := ghc.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
 			if err != nil {
 				r.Log.Error(err, "Error listing workflow jobs")
 				return //err
@@ -184,7 +185,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
-		workflowRuns, err := r.GitHubClient.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
+		workflowRuns, err := ghc.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
 		if err != nil {
 			return nil, err
 		}
@@ -211,6 +212,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	necessaryReplicas := queued + inProgress
 
+	prometheus_metrics.SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		necessaryReplicas,
+		completed,
+		inProgress,
+		queued,
+		unknown,
+	)
+
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas),
 		"workflow_runs_completed", completed,
@@ -226,7 +241,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 	return &necessaryReplicas, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
 	ctx := context.Background()
 	scaleUpThreshold := defaultScaleUpThreshold
 	scaleDownThreshold := defaultScaleDownThreshold
@@ -295,7 +310,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	)
 
 	// ListRunners will return all runners managed by GitHub - not restricted to ns
-	runners, err := r.GitHubClient.ListRunners(
+	runners, err := ghc.ListRunners(
 		ctx,
 		enterprise,
 		organization,
@@ -382,6 +397,19 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	//
 	// - num_runners can be as twice as large as replicas_desired_before while
 	//   the runnerdeployment controller is replacing RunnerReplicaSet for runner update.
+	prometheus_metrics.SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		desiredReplicas,
+		numRunners,
+		numRunnersRegistered,
+		numRunnersBusy,
+		numTerminatingBusy,
+	)
 
 	r.Log.V(1).Info(
 		fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas),

controllers/autoscaling_test.go

@@ -330,7 +330,6 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
-				GitHubClient:          client,
 				Scheme:                scheme,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
@@ -379,7 +378,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)
@@ -720,7 +719,6 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
 				Scheme:                scheme,
-				GitHubClient:          client,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
 
@@ -781,7 +779,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)

controllers/horizontal_runner_autoscaler_webhook.go

@@ -30,7 +30,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/go-logr/logr"
-	gogithub "github.com/google/go-github/v45/github"
+	gogithub "github.com/google/go-github/v47/github"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"

controllers/horizontal_runner_autoscaler_webhook_on_check_run.go

@@ -3,7 +3,7 @@ package controllers
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/pkg/actionsglob"
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchCheckRunEvent(event *github.CheckRunEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPullRequestEvent(event *github.PullRequestEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_push.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event *github.PushEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_test.go

@@ -15,7 +15,7 @@ import (
 
 	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"

controllers/horizontalrunnerautoscaler_controller.go

@@ -24,7 +24,6 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -38,6 +37,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
+	arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 const (
@@ -47,11 +47,10 @@ const (
 // HorizontalRunnerAutoscalerReconciler reconciles a HorizontalRunnerAutoscaler object
 type HorizontalRunnerAutoscalerReconciler struct {
 	client.Client
-	GitHubClient          *github.Client
+	GitHubClient          *MultiGitHubClient
 	Log                   logr.Logger
 	Recorder              record.EventRecorder
 	Scheme                *runtime.Scheme
-	CacheDuration         time.Duration
 	DefaultScaleDownDelay time.Duration
 	Name                  string
 }
@@ -73,6 +72,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(ctx context.Context, re
 	}
 
 	if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForHRA(&hra)
+
 		return ctrl.Result{}, nil
 	}
 
@@ -310,7 +311,12 @@ func (r *HorizontalRunnerAutoscalerReconciler) reconcile(ctx context.Context, re
 		return ctrl.Result{}, err
 	}
 
-	newDesiredReplicas, err := r.computeReplicasWithCache(log, now, st, hra, minReplicas)
+	ghc, err := r.GitHubClient.InitForHRA(context.Background(), &hra)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	newDesiredReplicas, err := r.computeReplicasWithCache(ghc, log, now, st, hra, minReplicas)
 	if err != nil {
 		r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -461,10 +467,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) getMinReplicas(log logr.Logger, n
 	return minReplicas, active, upcoming, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(ghc *arcgithub.Client, log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
 	var suggestedReplicas int
 
-	v, err := r.suggestDesiredReplicas(st, hra)
+	v, err := r.suggestDesiredReplicas(ghc, st, hra)
 	if err != nil {
 		return 0, err
 	}

controllers/integration_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	github2 "github.com/actions-runner-controller/actions-runner-controller/github"
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
 
@@ -99,12 +99,14 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 			return fmt.Sprintf("%s%s", ns.Name, name)
 		}
 
+		multiClient := NewMultiGitHubClient(mgr.GetClient(), env.ghClient)
+
 		runnerController := &RunnerReconciler{
 			Client:                      mgr.GetClient(),
 			Scheme:                      scheme.Scheme,
 			Log:                         logf.Log,
 			Recorder:                    mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient:                env.ghClient,
+			GitHubClient:                multiClient,
 			RunnerImage:                 "example/runner:test",
 			DockerImage:                 "example/docker:test",
 			Name:                        controllerName("runner"),
@@ -120,7 +122,6 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 			Scheme:   scheme.Scheme,
 			Log:      logf.Log,
 			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient: env.ghClient,
 			Name:     controllerName("runnerreplicaset"),
 		}
 		err = replicasetController.SetupWithManager(mgr)
@@ -140,9 +141,8 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 			Client:       mgr.GetClient(),
 			Scheme:       scheme.Scheme,
 			Log:          logf.Log,
-			GitHubClient:  env.ghClient,
+			GitHubClient: multiClient,
 			Recorder:     mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
-			CacheDuration: 1 * time.Second,
 			Name:         controllerName("horizontalrunnerautoscaler"),
 		}
 		err = autoscalerController.SetupWithManager(mgr)

controllers/metrics/horizontalrunnerautoscaler.go

@@ -9,6 +9,11 @@ import (
 const (
 	hraName        = "horizontalrunnerautoscaler"
 	hraNamespace   = "namespace"
+	stEnterprise   = "enterprise"
+	stOrganization = "organization"
+	stRepository   = "repository"
+	stKind         = "kind"
+	stName         = "name"
 )
 
 var (
@@ -16,6 +21,16 @@ var (
 		horizontalRunnerAutoscalerMinReplicas,
 		horizontalRunnerAutoscalerMaxReplicas,
 		horizontalRunnerAutoscalerDesiredReplicas,
+		horizontalRunnerAutoscalerReplicasDesired,
+		horizontalRunnerAutoscalerRunners,
+		horizontalRunnerAutoscalerRunnersRegistered,
+		horizontalRunnerAutoscalerRunnersBusy,
+		horizontalRunnerAutoscalerTerminatingBusy,
+		horizontalRunnerAutoscalerNecessaryReplicas,
+		horizontalRunnerAutoscalerWorkflowRunsCompleted,
+		horizontalRunnerAutoscalerWorkflowRunsInProgress,
+		horizontalRunnerAutoscalerWorkflowRunsQueued,
+		horizontalRunnerAutoscalerWorkflowRunsUnknown,
 	}
 )
 
@@ -41,6 +56,78 @@ var (
 		},
 		[]string{hraName, hraNamespace},
 	)
+	// PercentageRunnersBusy
+	horizontalRunnerAutoscalerReplicasDesired = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_replicas_desired",
+			Help: "replicas_desired of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunners = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners",
+			Help: "num_runners of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersRegistered = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_registered",
+			Help: "num_runners_registered of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_busy",
+			Help: "num_runners_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerTerminatingBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_terminating_busy",
+			Help: "num_terminating_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	// QueuedAndInProgressWorkflowRuns
+	horizontalRunnerAutoscalerNecessaryReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_necessary_replicas",
+			Help: "necessary_replicas of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsCompleted = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_completed",
+			Help: "workflow_runs_completed of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsInProgress = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_in_progress",
+			Help: "workflow_runs_in_progress of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsQueued = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_queued",
+			Help: "workflow_runs_queued of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsUnknown = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_unknown",
+			Help: "workflow_runs_unknown of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
 )
 
 func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) {
@@ -65,3 +152,61 @@ func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.Ho
 		horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas))
 	}
 }
+
+func SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	desiredReplicas int,
+	numRunners int,
+	numRunnersRegistered int,
+	numRunnersBusy int,
+	numTerminatingBusy int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerReplicasDesired.With(labels).Set(float64(desiredReplicas))
+	horizontalRunnerAutoscalerRunners.With(labels).Set(float64(numRunners))
+	horizontalRunnerAutoscalerRunnersRegistered.With(labels).Set(float64(numRunnersRegistered))
+	horizontalRunnerAutoscalerRunnersBusy.With(labels).Set(float64(numRunnersBusy))
+	horizontalRunnerAutoscalerTerminatingBusy.With(labels).Set(float64(numTerminatingBusy))
+}
+
+func SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	necessaryReplicas int,
+	workflowRunsCompleted int,
+	workflowRunsInProgress int,
+	workflowRunsQueued int,
+	workflowRunsUnknown int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerNecessaryReplicas.With(labels).Set(float64(necessaryReplicas))
+	horizontalRunnerAutoscalerWorkflowRunsCompleted.With(labels).Set(float64(workflowRunsCompleted))
+	horizontalRunnerAutoscalerWorkflowRunsInProgress.With(labels).Set(float64(workflowRunsInProgress))
+	horizontalRunnerAutoscalerWorkflowRunsQueued.With(labels).Set(float64(workflowRunsQueued))
+	horizontalRunnerAutoscalerWorkflowRunsUnknown.With(labels).Set(float64(workflowRunsUnknown))
+}

controllers/multi_githubclient.go

@@ -0,0 +1,358 @@
+package controllers
+
+import (
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"strconv"
+	"sync"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// The api creds scret annotation is added by the runner controller or the runnerset controller according to runner.spec.githubAPICredentialsFrom.secretRef.name,
+	// so that the runner pod controller can share the same GitHub API credentials and the instance of the GitHub API client with the upstream controllers.
+	annotationKeyGitHubAPICredsSecret = annotationKeyPrefix + "github-api-creds-secret"
+)
+
+type runnerOwnerRef struct {
+	// kind is either StatefulSet or Runner, and populated via the owner reference in the runner pod controller or via the reconcilation target's kind in
+	// runnerset and runner controllers.
+	kind     string
+	ns, name string
+}
+
+type secretRef struct {
+	ns, name string
+}
+
+// savedClient is the each cache entry that contains the client for the specific set of credentials,
+// like a PAT or a pair of key and cert.
+// the `hash` is a part of the savedClient not the key because we are going to keep only the client for the latest creds
+// in case the operator updated the k8s secret containing the credentials.
+type savedClient struct {
+	hash string
+
+	// refs is the map of all the objects that references this client, used for reference counting to gc
+	// the client if unneeded.
+	refs map[runnerOwnerRef]struct{}
+
+	*github.Client
+}
+
+type resourceReader interface {
+	Get(context.Context, types.NamespacedName, client.Object) error
+}
+
+type MultiGitHubClient struct {
+	mu sync.Mutex
+
+	client resourceReader
+
+	githubClient *github.Client
+
+	// The saved client is freed once all its dependents disappear, or the contents of the secret changed.
+	// We track dependents via a golang map embedded within the savedClient struct. Each dependent is checked on their respective Kubernetes finalizer,
+	// so that we won't miss any dependent's termination.
+	// The change is the secret is determined using the hash of its contents.
+	clients map[secretRef]savedClient
+}
+
+func NewMultiGitHubClient(client resourceReader, githubClient *github.Client) *MultiGitHubClient {
+	return &MultiGitHubClient{
+		client:       client,
+		githubClient: githubClient,
+		clients:      map[secretRef]savedClient{},
+	}
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerPod(ctx context.Context, pod *corev1.Pod) (*github.Client, error) {
+	// These 3 default values are used only when the user created the pod directly, not via Runner, RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunnerPod(pod)
+	secretName := pod.Annotations[annotationKeyGitHubAPICredsSecret]
+
+	// kind can be any of Pod, Runner, RunnerReplicaSet, RunnerDeployment, or RunnerSet depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, pod.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunner(ctx context.Context, r *v1alpha1.Runner) (*github.Client, error) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	// These 3 default values are used only when the user created the runner resource directly, not via RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunner(r)
+	if ref.ns != r.Namespace {
+		return nil, fmt.Errorf("referencing github api creds secret from owner in another namespace is not supported yet")
+	}
+
+	// kind can be any of Runner, RunnerReplicaSet, or RunnerDeployment depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, r.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerSet(ctx context.Context, rs *v1alpha1.RunnerSet) (*github.Client, error) {
+	ref := refFromRunnerSet(rs)
+
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, rs.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForHRA(ctx context.Context, hra *v1alpha1.HorizontalRunnerAutoscaler) (*github.Client, error) {
+	ref := refFromHorizontalRunnerAutoscaler(hra)
+
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, hra.Namespace, secretName, ref)
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerPod(p *corev1.Pod) {
+	secretName := p.Annotations[annotationKeyGitHubAPICredsSecret]
+	c.derefClient(p.Namespace, secretName, refFromRunnerPod(p))
+}
+
+func (c *MultiGitHubClient) DeinitForRunner(r *v1alpha1.Runner) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(r.Namespace, secretName, refFromRunner(r))
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerSet(rs *v1alpha1.RunnerSet) {
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(rs.Namespace, secretName, refFromRunnerSet(rs))
+}
+
+func (c *MultiGitHubClient) deinitClientForRunnerReplicaSet(rs *v1alpha1.RunnerReplicaSet) {
+	c.derefClient(rs.Namespace, rs.Spec.Template.Spec.GitHubAPICredentialsFrom.SecretRef.Name, refFromRunnerReplicaSet(rs))
+}
+
+func (c *MultiGitHubClient) deinitClientForRunnerDeployment(rd *v1alpha1.RunnerDeployment) {
+	c.derefClient(rd.Namespace, rd.Spec.Template.Spec.GitHubAPICredentialsFrom.SecretRef.Name, refFromRunnerDeployment(rd))
+}
+
+func (c *MultiGitHubClient) DeinitForHRA(hra *v1alpha1.HorizontalRunnerAutoscaler) {
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(hra.Namespace, secretName, refFromHorizontalRunnerAutoscaler(hra))
+}
+
+func (c *MultiGitHubClient) initClientForSecret(secret *corev1.Secret, dependent *runnerOwnerRef) (*savedClient, error) {
+	secRef := secretRef{
+		ns:   secret.Namespace,
+		name: secret.Name,
+	}
+
+	cliRef := c.clients[secRef]
+
+	var ks []string
+
+	for k := range secret.Data {
+		ks = append(ks, k)
+	}
+
+	sort.SliceStable(ks, func(i, j int) bool { return ks[i] < ks[j] })
+
+	hash := sha1.New()
+	for _, k := range ks {
+		hash.Write(secret.Data[k])
+	}
+	hashStr := hex.EncodeToString(hash.Sum(nil))
+
+	if cliRef.hash != hashStr {
+		delete(c.clients, secRef)
+
+		conf, err := secretDataToGitHubClientConfig(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		// Fallback to the controller-wide setting if EnterpriseURL is not set and the original client is an enterprise client.
+		if conf.EnterpriseURL == "" && c.githubClient.IsEnterprise {
+			conf.EnterpriseURL = c.githubClient.GithubBaseURL
+		}
+
+		cli, err := conf.NewClient()
+		if err != nil {
+			return nil, err
+		}
+
+		cliRef = savedClient{
+			hash:   hashStr,
+			refs:   map[runnerOwnerRef]struct{}{},
+			Client: cli,
+		}
+
+		c.clients[secRef] = cliRef
+	}
+
+	if dependent != nil {
+		c.clients[secRef].refs[*dependent] = struct{}{}
+	}
+
+	return &cliRef, nil
+}
+
+func (c *MultiGitHubClient) initClientWithSecretName(ctx context.Context, ns, secretName string, runRef *runnerOwnerRef) (*github.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if secretName == "" {
+		return c.githubClient, nil
+	}
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if _, ok := c.clients[secRef]; !ok {
+		c.clients[secRef] = savedClient{}
+	}
+
+	var sec corev1.Secret
+	if err := c.client.Get(ctx, types.NamespacedName{Namespace: ns, Name: secretName}, &sec); err != nil {
+		return nil, err
+	}
+
+	savedClient, err := c.initClientForSecret(&sec, runRef)
+	if err != nil {
+		return nil, err
+	}
+
+	return savedClient.Client, nil
+}
+
+func (c *MultiGitHubClient) derefClient(ns, secretName string, dependent *runnerOwnerRef) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if dependent != nil {
+		delete(c.clients[secRef].refs, *dependent)
+	}
+
+	cliRef := c.clients[secRef]
+
+	if dependent == nil || len(cliRef.refs) == 0 {
+		delete(c.clients, secRef)
+	}
+}
+
+func secretDataToGitHubClientConfig(data map[string][]byte) (*github.Config, error) {
+	var (
+		conf github.Config
+
+		err error
+	)
+
+	conf.URL = string(data["github_url"])
+
+	conf.UploadURL = string(data["github_upload_url"])
+
+	conf.EnterpriseURL = string(data["github_enterprise_url"])
+
+	conf.RunnerGitHubURL = string(data["github_runner_url"])
+
+	conf.Token = string(data["github_token"])
+
+	appID := string(data["github_app_id"])
+
+	conf.AppID, err = strconv.ParseInt(appID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	instID := string(data["github_app_installation_id"])
+
+	conf.AppInstallationID, err = strconv.ParseInt(instID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	conf.AppPrivateKey = string(data["github_app_private_key"])
+
+	return &conf, nil
+}
+
+func refFromRunnerDeployment(rd *v1alpha1.RunnerDeployment) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rd.Kind,
+		ns:   rd.Namespace,
+		name: rd.Name,
+	}
+}
+
+func refFromRunnerReplicaSet(rs *v1alpha1.RunnerReplicaSet) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rs.Kind,
+		ns:   rs.Namespace,
+		name: rs.Name,
+	}
+}
+
+func refFromRunner(r *v1alpha1.Runner) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: r.Kind,
+		ns:   r.Namespace,
+		name: r.Name,
+	}
+}
+
+func refFromRunnerPod(po *corev1.Pod) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: po.Kind,
+		ns:   po.Namespace,
+		name: po.Name,
+	}
+}
+func refFromRunnerSet(rs *v1alpha1.RunnerSet) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rs.Kind,
+		ns:   rs.Namespace,
+		name: rs.Name,
+	}
+}
+
+func refFromHorizontalRunnerAutoscaler(hra *v1alpha1.HorizontalRunnerAutoscaler) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: hra.Kind,
+		ns:   hra.Namespace,
+		name: hra.Name,
+	}
+}

controllers/new_runner_pod_test.go

@@ -10,7 +10,9 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 func newWorkGenericEphemeralVolume(t *testing.T, storageReq string) corev1.Volume {
@@ -125,6 +127,10 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "DOCKER_HOST",
 							Value: "tcp://localhost:2376",
@@ -255,6 +261,10 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -333,6 +343,10 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -515,7 +529,7 @@ func TestNewRunnerPod(t *testing.T) {
 	for i := range testcases {
 		tc := testcases[i]
 		t.Run(tc.description, func(t *testing.T) {
-			got, err := newRunnerPod(tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
+			got, err := newRunnerPod(tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, false)
 			require.NoError(t, err)
 			require.Equal(t, tc.want, got)
 		})
@@ -624,6 +638,10 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "DOCKER_HOST",
 							Value: "tcp://localhost:2376",
@@ -769,6 +787,10 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -866,6 +888,10 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -1105,13 +1131,20 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 
 	for i := range testcases {
 		tc := testcases[i]
+
+		rr := &testResourceReader{
+			objects: map[types.NamespacedName]client.Object{},
+		}
+
+		multiClient := NewMultiGitHubClient(rr, &github.Client{GithubBaseURL: githubBaseURL})
+
 		t.Run(tc.description, func(t *testing.T) {
 			r := &RunnerReconciler{
 				RunnerImage:            defaultRunnerImage,
 				RunnerImagePullSecrets: defaultRunnerImagePullSecrets,
 				DockerImage:            defaultDockerImage,
 				DockerRegistryMirror:   defaultDockerRegistryMirror,
-				GitHubClient:           &github.Client{GithubBaseURL: githubBaseURL},
+				GitHubClient:           multiClient,
 				Scheme:                 scheme,
 			}
 			got, err := r.newPod(tc.runner)

controllers/pod_runner_token_injector.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
 	"gomodules.xyz/jsonpatch/v2"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -29,7 +28,7 @@ type PodRunnerTokenInjector struct {
 	Name         string
 	Log          logr.Logger
 	Recorder     record.EventRecorder
-	GitHubClient *github.Client
+	GitHubClient *MultiGitHubClient
 	decoder      *admission.Decoder
 }
 
@@ -66,7 +65,12 @@ func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Reque
 		return newEmptyResponse()
 	}
 
-	rt, err := t.GitHubClient.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
+	ghc, err := t.GitHubClient.InitForRunnerPod(ctx, &pod)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	rt, err := ghc.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
 	if err != nil {
 		t.Log.Error(err, "Failed to get new registration token")
 		return admission.Errored(http.StatusInternalServerError, err)

controllers/runner_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -35,10 +36,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 const (
@@ -51,6 +52,8 @@ const (
 
 	EnvVarOrg        = "RUNNER_ORG"
 	EnvVarRepo       = "RUNNER_REPO"
+	EnvVarGroup      = "RUNNER_GROUP"
+	EnvVarLabels     = "RUNNER_LABELS"
 	EnvVarEnterprise = "RUNNER_ENTERPRISE"
 	EnvVarEphemeral  = "RUNNER_EPHEMERAL"
 	EnvVarTrue       = "true"
@@ -62,7 +65,7 @@ type RunnerReconciler struct {
 	Log                         logr.Logger
 	Recorder                    record.EventRecorder
 	Scheme                      *runtime.Scheme
-	GitHubClient                *github.Client
+	GitHubClient                *MultiGitHubClient
 	RunnerImage                 string
 	RunnerImagePullSecrets      []string
 	DockerImage                 string
@@ -70,7 +73,7 @@ type RunnerReconciler struct {
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
-
+	UseRunnerStatusUpdateHook   bool
 	UnregistrationRetryDelay    time.Duration
 }
 
@@ -81,6 +84,9 @@ type RunnerReconciler struct {
 // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;delete
 // +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=create;delete;get
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;delete;get
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;delete;get
 
 func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("runner", req.NamespacedName)
@@ -116,6 +122,8 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			return r.processRunnerDeletion(runner, ctx, log, nil)
 		}
 
+		r.GitHubClient.DeinitForRunner(&runner)
+
 		return r.processRunnerDeletion(runner, ctx, log, &pod)
 	}
 
@@ -135,7 +143,7 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 	ready := runnerPodReady(&pod)
 
-	if runner.Status.Phase != phase || runner.Status.Ready != ready {
+	if (runner.Status.Phase != phase || runner.Status.Ready != ready) && !r.UseRunnerStatusUpdateHook || runner.Status.Phase == "" && r.UseRunnerStatusUpdateHook {
 		if pod.Status.Phase == corev1.PodRunning {
 			// Seeing this message, you can expect the runner to become `Running` soon.
 			log.V(1).Info(
@@ -256,6 +264,96 @@ func (r *RunnerReconciler) processRunnerCreation(ctx context.Context, runner v1a
 		return ctrl.Result{}, err
 	}
 
+	needsServiceAccount := runner.Spec.ServiceAccountName == "" && (r.UseRunnerStatusUpdateHook || runner.Spec.ContainerMode == "kubernetes")
+	if needsServiceAccount {
+		serviceAccount := &corev1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+		}
+		if res := r.createObject(ctx, serviceAccount, serviceAccount.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+
+		rules := []rbacv1.PolicyRule{}
+
+		if r.UseRunnerStatusUpdateHook {
+			rules = append(rules, []rbacv1.PolicyRule{
+				{
+					APIGroups:     []string{"actions.summerwind.dev"},
+					Resources:     []string{"runners/status"},
+					Verbs:         []string{"get", "update", "patch"},
+					ResourceNames: []string{runner.ObjectMeta.Name},
+				},
+			}...)
+		}
+
+		if runner.Spec.ContainerMode == "kubernetes" {
+			// Permissions based on https://github.com/actions/runner-container-hooks/blob/main/packages/k8s/README.md
+			rules = append(rules, []rbacv1.PolicyRule{
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods/exec"},
+					Verbs:     []string{"get", "create"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods/log"},
+					Verbs:     []string{"get", "list", "watch"},
+				},
+				{
+					APIGroups: []string{"batch"},
+					Resources: []string{"jobs"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"secrets"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+			}...)
+		}
+
+		role := &rbacv1.Role{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+			Rules: rules,
+		}
+		if res := r.createObject(ctx, role, role.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+
+		roleBinding := &rbacv1.RoleBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+			RoleRef: rbacv1.RoleRef{
+				APIGroup: "rbac.authorization.k8s.io",
+				Kind:     "Role",
+				Name:     runner.ObjectMeta.Name,
+			},
+			Subjects: []rbacv1.Subject{
+				{
+					Kind:      "ServiceAccount",
+					Name:      runner.ObjectMeta.Name,
+					Namespace: runner.ObjectMeta.Namespace,
+				},
+			},
+		}
+		if res := r.createObject(ctx, roleBinding, roleBinding.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+	}
+
 	if err := r.Create(ctx, &newPod); err != nil {
 		if kerrors.IsAlreadyExists(err) {
 			// Gracefully handle pod-already-exists errors due to informer cache delay.
@@ -278,6 +376,27 @@ func (r *RunnerReconciler) processRunnerCreation(ctx context.Context, runner v1a
 	return ctrl.Result{}, nil
 }
 
+func (r *RunnerReconciler) createObject(ctx context.Context, obj client.Object, meta metav1.ObjectMeta, runner *v1alpha1.Runner, log logr.Logger) *ctrl.Result {
+	kind := strings.Split(reflect.TypeOf(obj).String(), ".")[1]
+	if err := ctrl.SetControllerReference(runner, obj, r.Scheme); err != nil {
+		log.Error(err, fmt.Sprintf("Could not add owner reference to %s %s. %s", kind, meta.Name, err.Error()))
+		return &ctrl.Result{Requeue: true}
+	}
+	if err := r.Create(ctx, obj); err != nil {
+		if kerrors.IsAlreadyExists(err) {
+			log.Info(fmt.Sprintf("Failed to create %s %s as it already exists. Reusing existing %s", kind, meta.Name, kind))
+			r.Recorder.Event(runner, corev1.EventTypeNormal, fmt.Sprintf("%sReused", kind), fmt.Sprintf("Reused %s '%s'", kind, meta.Name))
+			return nil
+		}
+
+		log.Error(err, fmt.Sprintf("Retrying as failed to create %s %s resource", kind, meta.Name))
+		return &ctrl.Result{Requeue: true}
+	}
+	r.Recorder.Event(runner, corev1.EventTypeNormal, fmt.Sprintf("%sCreated", kind), fmt.Sprintf("Created %s '%s'", kind, meta.Name))
+	log.Info(fmt.Sprintf("Created %s", kind), "name", meta.Name)
+	return nil
+}
+
 func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v1alpha1.Runner) (bool, error) {
 	if runner.IsRegisterable() {
 		return false, nil
@@ -285,7 +404,12 @@ func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v
 
 	log := r.Log.WithValues("runner", runner.Name)
 
-	rt, err := r.GitHubClient.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+	ghc, err := r.GitHubClient.InitForRunner(ctx, &runner)
+	if err != nil {
+		return false, err
+	}
+
+	rt, err := ghc.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 	if err != nil {
 		// An error can be a permanent, permission issue like the below:
 		//    POST https://api.github.com/enterprises/YOUR_ENTERPRISE/actions/runners/registration-token: 403 Resource not accessible by integration []
@@ -325,6 +449,11 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		labels[k] = v
 	}
 
+	ghc, err := r.GitHubClient.InitForRunner(context.Background(), &runner)
+	if err != nil {
+		return corev1.Pod{}, err
+	}
+
 	// This implies that...
 	//
 	// (1) We recreate the runner pod whenever the runner has changes in:
@@ -348,7 +477,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		filterLabels(runner.ObjectMeta.Labels, LabelKeyRunnerTemplateHash),
 		runner.ObjectMeta.Annotations,
 		runner.Spec,
-		r.GitHubClient.GithubBaseURL,
+		ghc.GithubBaseURL,
 		// Token change should trigger replacement.
 		// We need to include this explicitly here because
 		// runner.Spec does not contain the possibly updated token stored in the
@@ -426,7 +555,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		}
 	}
 
-	pod, err := newRunnerPodWithContainerMode(runner.Spec.ContainerMode, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL)
+	pod, err := newRunnerPodWithContainerMode(runner.Spec.ContainerMode, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, ghc.GithubBaseURL, r.UseRunnerStatusUpdateHook)
 	if err != nil {
 		return pod, err
 	}
@@ -474,9 +603,13 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	if runnerSpec.NodeSelector != nil {
 		pod.Spec.NodeSelector = runnerSpec.NodeSelector
 	}
+
 	if runnerSpec.ServiceAccountName != "" {
 		pod.Spec.ServiceAccountName = runnerSpec.ServiceAccountName
+	} else if r.UseRunnerStatusUpdateHook || runner.Spec.ContainerMode == "kubernetes" {
+		pod.Spec.ServiceAccountName = runner.ObjectMeta.Name
 	}
+
 	if runnerSpec.AutomountServiceAccountToken != nil {
 		pod.Spec.AutomountServiceAccountToken = runnerSpec.AutomountServiceAccountToken
 	}
@@ -589,7 +722,7 @@ func runnerHookEnvs(pod *corev1.Pod) ([]corev1.EnvVar, error) {
 	}, nil
 }
 
-func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
+func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, useRunnerStatusUpdateHook bool) (corev1.Pod, error) {
 	var (
 		privileged                bool = true
 		dockerdInRunner           bool = runnerSpec.DockerdWithinRunnerContainer != nil && *runnerSpec.DockerdWithinRunnerContainer
@@ -609,6 +742,9 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 	// This label selector is used by default when rd.Spec.Selector is empty.
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunner, "")
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyPodMutation, LabelValuePodMutation)
+	if runnerSpec.GitHubAPICredentialsFrom != nil {
+		template.ObjectMeta.Annotations = CloneAndAddLabel(template.ObjectMeta.Annotations, annotationKeyGitHubAPICredsSecret, runnerSpec.GitHubAPICredentialsFrom.SecretRef.Name)
+	}
 
 	workDir := runnerSpec.WorkDir
 	if workDir == "" {
@@ -638,11 +774,11 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 			Value: runnerSpec.Enterprise,
 		},
 		{
-			Name:  "RUNNER_LABELS",
+			Name:  EnvVarLabels,
 			Value: strings.Join(runnerSpec.Labels, ","),
 		},
 		{
-			Name:  "RUNNER_GROUP",
+			Name:  EnvVarGroup,
 			Value: runnerSpec.Group,
 		},
 		{
@@ -665,6 +801,10 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 			Name:  EnvVarEphemeral,
 			Value: fmt.Sprintf("%v", ephemeral),
 		},
+		{
+			Name:  "RUNNER_STATUS_UPDATE_HOOK",
+			Value: fmt.Sprintf("%v", useRunnerStatusUpdateHook),
+		},
 	}
 
 	var seLinuxOptions *corev1.SELinuxOptions
@@ -962,8 +1102,8 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 	return *pod, nil
 }
 
-func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
-	return newRunnerPodWithContainerMode("", template, runnerSpec, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
+func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, useRunnerStatusUpdateHookEphemeralRole bool) (corev1.Pod, error) {
+	return newRunnerPodWithContainerMode("", template, runnerSpec, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, useRunnerStatusUpdateHookEphemeralRole)
 }
 
 func (r *RunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {

controllers/runner_graceful_stop.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
-	gogithub "github.com/google/go-github/v45/github"
+	gogithub "github.com/google/go-github/v47/github"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"

controllers/runner_pod_controller.go

@@ -32,8 +32,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	corev1 "k8s.io/api/core/v1"
-
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 // RunnerPodReconciler reconciles a Runner object
@@ -42,7 +40,7 @@ type RunnerPodReconciler struct {
 	Log                         logr.Logger
 	Recorder                    record.EventRecorder
 	Scheme                      *runtime.Scheme
-	GitHubClient                *github.Client
+	GitHubClient                *MultiGitHubClient
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
@@ -97,6 +95,11 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		}
 	}
 
+	ghc, err := r.GitHubClient.InitForRunnerPod(ctx, &runnerPod)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
 	if runnerPod.ObjectMeta.DeletionTimestamp.IsZero() {
 		finalizers, added := addFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
 
@@ -148,7 +151,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			// In a standard scenario, the upstream controller, like runnerset-controller, ensures this runner to be gracefully stopped before the deletion timestamp is set.
 			// But for the case that the user manually deleted it for whatever reason,
 			// we have to ensure it to gracefully stop now.
-			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 			if res != nil {
 				return *res, err
 			}
@@ -164,6 +167,8 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 			log.V(2).Info("Removed finalizer")
 
+			r.GitHubClient.DeinitForRunnerPod(updatedPod)
+
 			return ctrl.Result{}, nil
 		}
 
@@ -202,7 +207,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
-	po, res, err := ensureRunnerPodRegistered(ctx, log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+	po, res, err := ensureRunnerPodRegistered(ctx, log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 	if res != nil {
 		return *res, err
 	}
@@ -216,7 +221,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		//
 		// In a standard scenario, ARC starts the unregistration process before marking the pod for deletion at all,
 		// so that it isn't subject to terminationGracePeriod and can safely take hours to finish it's work.
-		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 		if res != nil {
 			return *res, err
 		}

controllers/runnerdeployment_controller.go

@@ -165,6 +165,8 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, err
 		}
 
+		log.V(1).Info("Updated runnerreplicaset due to selector change")
+
 		// At this point, we are already sure that there's no need to create a new replicaset
 		// as the runner template hash is not changed.
 		//
@@ -182,7 +184,14 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	//
 	// If we missed taking the EffectiveTime diff into account, you might end up experiencing scale-ups being delayed scale-down.
 	// See https://github.com/actions-runner-controller/actions-runner-controller/pull/1477#issuecomment-1164154496
-	if currentDesiredReplicas != newDesiredReplicas || newestSet.Spec.EffectiveTime != rd.Spec.EffectiveTime {
+	var et1, et2 time.Time
+	if newestSet.Spec.EffectiveTime != nil {
+		et1 = newestSet.Spec.EffectiveTime.Time
+	}
+	if rd.Spec.EffectiveTime != nil {
+		et2 = rd.Spec.EffectiveTime.Time
+	}
+	if currentDesiredReplicas != newDesiredReplicas || et1 != et2 {
 		newestSet.Spec.Replicas = &newDesiredReplicas
 		newestSet.Spec.EffectiveTime = rd.Spec.EffectiveTime
 
@@ -192,6 +201,13 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, err
 		}
 
+		log.V(1).Info("Updated runnerreplicaset due to spec change",
+			"currentDesiredReplicas", currentDesiredReplicas,
+			"newDesiredReplicas", newDesiredReplicas,
+			"currentEffectiveTime", newestSet.Spec.EffectiveTime,
+			"newEffectiveTime", rd.Spec.EffectiveTime,
+		)
+
 		return ctrl.Result{}, err
 	}
 

controllers/runnerreplicaset_controller.go

@@ -32,7 +32,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 // RunnerReplicaSetReconciler reconciles a Runner object
@@ -41,7 +40,6 @@ type RunnerReplicaSetReconciler struct {
 	Log      logr.Logger
 	Recorder record.EventRecorder
 	Scheme   *runtime.Scheme
-	GitHubClient *github.Client
 	Name     string
 }
 

controllers/runnerreplicaset_controller_test.go

@@ -52,14 +52,12 @@ func SetupTest(ctx2 context.Context) *corev1.Namespace {
 
 		runnersList = fake.NewRunnersList()
 		server = runnersList.GetServer()
-		ghClient := newGithubClient(server)
 
 		controller := &RunnerReplicaSetReconciler{
 			Client:   mgr.GetClient(),
 			Scheme:   scheme.Scheme,
 			Log:      logf.Log,
 			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient: ghClient,
 			Name:     "runnerreplicaset-" + ns.Name,
 		}
 		err = controller.SetupWithManager(mgr)

controllers/runnerset_controller.go

@@ -46,11 +46,12 @@ type RunnerSetReconciler struct {
 	Scheme   *runtime.Scheme
 
 	CommonRunnerLabels        []string
-	GitHubBaseURL          string
+	GitHubClient              *MultiGitHubClient
 	RunnerImage               string
 	RunnerImagePullSecrets    []string
 	DockerImage               string
 	DockerRegistryMirror      string
+	UseRunnerStatusUpdateHook bool
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets,verbs=get;list;watch;create;update;patch;delete
@@ -80,6 +81,8 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	if !runnerSet.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForRunnerSet(runnerSet)
+
 		return ctrl.Result{}, nil
 	}
 
@@ -97,7 +100,7 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
-	desiredStatefulSet, err := r.newStatefulSet(runnerSet)
+	desiredStatefulSet, err := r.newStatefulSet(ctx, runnerSet)
 	if err != nil {
 		r.Recorder.Event(runnerSet, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -185,7 +188,7 @@ func getRunnerSetSelector(runnerSet *v1alpha1.RunnerSet) *metav1.LabelSelector {
 var LabelKeyPodMutation = "actions-runner-controller/inject-registration-token"
 var LabelValuePodMutation = "true"
 
-func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
+func (r *RunnerSetReconciler) newStatefulSet(ctx context.Context, runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
 	runnerSetWithOverrides := *runnerSet.Spec.DeepCopy()
 
 	runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, r.CommonRunnerLabels...)
@@ -221,7 +224,14 @@ func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*ap
 
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerSet.Name)
 
-	pod, err := newRunnerPodWithContainerMode(runnerSet.Spec.RunnerConfig.ContainerMode, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL)
+	ghc, err := r.GitHubClient.InitForRunnerSet(ctx, runnerSet)
+	if err != nil {
+		return nil, err
+	}
+
+	githubBaseURL := ghc.GithubBaseURL
+
+	pod, err := newRunnerPodWithContainerMode(runnerSet.Spec.RunnerConfig.ContainerMode, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, githubBaseURL, r.UseRunnerStatusUpdateHook)
 	if err != nil {
 		return nil, err
 	}

controllers/sync_volumes.go

@@ -75,6 +75,10 @@ func syncPVC(ctx context.Context, c client.Client, log logr.Logger, ns string, p
 
 	log.V(2).Info("Reconciling runner PVC")
 
+	// TODO: Probably we'd better remove PVCs related to the RunnetSet that is nowhere now?
+	// Otherwise, a bunch of continuously recreated StatefulSet
+	// can leave dangling PVCs forever, which might stress the cluster.
+
 	var sts appsv1.StatefulSet
 	if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: stsName}, &sts); err != nil {
 		if !kerrors.IsNotFound(err) {

controllers/testresourcereader.go

@@ -0,0 +1,31 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"reflect"
+
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type testResourceReader struct {
+	objects map[types.NamespacedName]client.Object
+}
+
+func (r *testResourceReader) Get(_ context.Context, nsName types.NamespacedName, obj client.Object) error {
+	ret, ok := r.objects[nsName]
+	if !ok {
+		return &kerrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonNotFound}}
+	}
+	v := reflect.ValueOf(obj)
+	if v.Kind() != reflect.Ptr {
+		return errors.New("obj must be a pointer")
+	}
+
+	v.Elem().Set(reflect.ValueOf(ret).Elem())
+
+	return nil
+}

controllers/testresourcereader_test.go

@@ -0,0 +1,35 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestResourceReader(t *testing.T) {
+	rr := &testResourceReader{
+		objects: map[types.NamespacedName]client.Object{
+			{Namespace: "default", Name: "sec1"}: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "sec1",
+				},
+				Data: map[string][]byte{
+					"foo": []byte("bar"),
+				},
+			},
+		},
+	}
+
+	var sec corev1.Secret
+
+	err := rr.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "sec1"}, &sec)
+	require.NoError(t, err)
+
+	require.Equal(t, []byte("bar"), sec.Data["foo"])
+}

docs/releasenotes/0.26.md

@@ -0,0 +1,99 @@
+# actions-runner-controller v0.26.0
+
+All planned changes in this release can be found in the milestone https://github.com/actions-runner-controller/actions-runner-controller/milestone/9.
+
+Also see https://github.com/actions-runner-controller/actions-runner-controller/compare/v0.24.2...v0.26.0 for full changelog.
+
+This log documents breaking changes and major enhancements
+
+## Upgrading
+
+In case you're using our Helm chart to deploy ARC, use the chart 0.21.0 or greater. Don't miss upgrading CRDs as usual! Helm doesn't upgrade CRDs.
+
+## BREAKING CHANGE : Min GHES version is now 3.6
+
+We've bumped the minimum requirement on GHES to [3.6.0](https://docs.github.com/en/enterprise-server@3.6/admin/release-notes#3.6.0) which has been released in August. The motivator for us was to use the new `visible_to_repository` option added to the list runner groups API for the runner group visibility based autoscaling which is crucial when you have a lot of runner groups that have non-distinct set of labels. If you don't use runner groups at all, ARC may just work, but YMMV.
+
+Relevant PR(s): #158
+
+## ENHANCEMENT : Rootless DinD runners
+
+An awesome GitHub staff added the support for rootless DinD powered runners. Compared to the standard DinD, a rootless DinD gives you an additional layer of security without losing the ability to invoke Docker containers and dokcer builds from within your workflow jobs. [If you aren't using the Kubernetes container mode](https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs), you should be using this new rootless DinD.
+
+Rootless DinD is the recent enhancement to Docker that basically allows you to run the Docker daemon and therefore Docker containers without the reliance on the `root` user. In the context of DinD(Docker-in-Docker) and ARC, this rootless DinD runner still requires a privileged container to function at all. But, the Linux user that runs the Docker daemon and the `actions/runner` agent can now be non-root, which is considered more secure than running DinD within a privileged container, as a random worfklow job is no longer able to run privileged operations.
+
+Before using this feature, we highly recommend you to read [the detailed explanation in the original pull request](https://github.com/actions-runner-controller/actions-runner-controller/pull/1644) and [the new section in ARC's documentation](https://github.com/actions-runner-controller/actions-runner-controller#runner-with-rootless-dind).
+
+Big kudos to @some-natalie for implementing and contributing this feature!
+
+Relevant PR(s): #1644
+
+## ENHANCEMENT : More granular and real-time runner statuses
+
+We added another controller flag and a Helm chart value to enable the new runner status update hook. Once enabled, it exposes more granular runner phases via the runner status.
+
+Previously, every `Runner` resource managed by `RunnerDeployment` was only able to expose these three Phases to e.g. `kubectl get runner` output:
+
+- `Pending`- The runner pod is waiting to be scheduled on any Kubernetes node/
+- `Running`-  The runner pod has been scheduled onto a node and its Linux namespace, containers, and the network has been set up. The primary processes of the containers are running.
+- `Succeeded`- The primary processes of the pod containers have stopped with exit status 0.
+
+As you may have realized, it had been quite useless, as it was a direct copy of the pod phase and tells almost nothing about the runner agent running inside the runner pod and the worfklow job that might be running.
+
+Since #1268 though, it can optionally provide two more phases, and the modified version of the `Running` phase. Once enabled via the controller command-line flag or the Helm chart value, you start to see:
+
+- `Registering`- The runner entrypoint started the runner registration process. Once the registration succeeds, it will update the phase to `Idle`.
+- `Idle`- The runner has been registered to GitHub and it's still waiting for GitHub to assign a workflow job to run.
+- `Running`- GitHub assigned a workflow job and the runner agent started running it.
+
+All the three phases should be more useful than before. For example, `Registering` can tell you that it's (still) unable to register itself against the GitHub Actions service. It it's hanging for minutes at the `Registering` phase, it's very likely you misconfigured your GitHub API credentials or you've somehow broken runner pods so that the runner is unable to register itself. If it's stuck in `Idle` like forever even though you queued some workflow runs and jobs, it's very likely you misconfigured runner labels or the `on` field of your workflow definitions.
+
+Big kudos to @fgalind1 for implementing and contributing this feature!
+
+Relevant PR(s): #1268
+
+## ENHANCEMENT : More Autoscaling-related metrics
+
+We added several more metrics related to the pull-based autoscaling so that you can scrape it via the [Prometheus exposition format](https://github.com/Showmax/prometheus-docs/blob/master/content/docs/instrumenting/exposition_formats.md), track and observe the changes on the graphing, dashboarding and alerting solution of your choice.
+
+For `PercentageRunnersBusy` metric, we added:
+
+- horizontalrunnerautoscaler_replicas_desired
+- horizontalrunnerautoscaler_runners
+- horizontalrunnerautoscaler_runners_registered
+- horizontalrunnerautoscaler_runners_busy
+- horizontalrunnerautoscaler_terminating_busy
+
+For `TotalNumberOfQueuedAndInProgressWorkflowRuns` metric, we added:
+
+- horizontalrunnerautoscaler_necessary_replicas
+- horizontalrunnerautoscaler_workflow_runs_completed
+- horizontalrunnerautoscaler_workflow_runs_in_progress
+- horizontalrunnerautoscaler_workflow_runs_queued
+- horizontalrunnerautoscaler_workflow_runs_unknown
+
+Big kudos to @debugger24 for implementing and contributing this feature!
+
+Relevant PR(s): #1720
+
+## ENHANCEMENT : Improved Multi-tenancy
+
+We had a long-living feature request about reducing the number of ARC instances one needs to maintain to provide self-hosted runners across multiple enterprises and organizations, and here it is. You can now manage as many enterprises and organizations with ARC.
+
+Previously you had to set up and manage an ARC instance per enterprise or in many cases per organization, because ARC was able to handle only one set of GitHub API credentials(PAT or GitHub App). The new multitenancy supports breaks this limitation by introducing the new `githubAPICredentialsFrom` field to the runner spec. You create a Kubernetes secret containing a GitHub API credentials and specify the secret name in `githubAPICredentialsFrom`, so that ARC picks it up and use it at the reconcilation time.
+
+We've written a detailed guide about this feature in the ["Multitenancy" section of the README](https://github.com/actions-runner-controller/actions-runner-controller#multitenancy). Please read it and give it a try!
+
+Lastly, this feature was stabilized by many early testers from the community. Big thanks and kudos to everyone who participated in testing, especially @Jalmeida1994 and @bm1216 for not only finding bugs but also contributing fixes ([#1725](https://github.com/actions-runner-controller/actions-runner-controller/pull/1725) and [#1781](https://github.com/actions-runner-controller/actions-runner-controller/pull/1781)!
+
+Relevant PR(s): #1268
+
+## ENHANCEMENT : Print ARC version number on startup
+
+Our build script now injects the version number of ARC into the executable, and prints it on startup so that you can see from logs that which version of ARC you're currently running. Previously when you are to file a bug report, you had to be extra sure to know which version of ARC you're using and encountering an issue. It's now easier than ever because you can grab the version number show in the logs, without consulting the container image tag of chart's appVersion.
+
+In addition to the logs, ARC is enhanced to send a HTTP `User-Agent` header containing the version number for every GitHub Actions API call ARC makes. You don't usually rely on it but GitHub and GitHub Actions's backend service can rely on it to collect the metrics about which versions of ARC folks are using.
+
+Big kudos to @ViktorLindgren95 for implementing and contributing this feature!
+
+Relevant PR(s): #1659

github/fake/runners.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 	"github.com/gorilla/mux"
 )
 

github/github.go

@@ -10,11 +10,12 @@ import (
 	"sync"
 	"time"
 
+	"github.com/actions-runner-controller/actions-runner-controller/build"
 	"github.com/actions-runner-controller/actions-runner-controller/github/metrics"
 	"github.com/actions-runner-controller/actions-runner-controller/logging"
 	"github.com/bradleyfalzon/ghinstallation/v2"
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 	"github.com/gregjones/httpcache"
 	"golang.org/x/oauth2"
 )
@@ -42,6 +43,7 @@ type Client struct {
 	mu        sync.Mutex
 	// GithubBaseURL to Github without API suffix.
 	GithubBaseURL string
+	IsEnterprise  bool
 }
 
 type BasicAuthTransport struct {
@@ -94,8 +96,10 @@ func (c *Config) NewClient() (*Client, error) {
 
 	var client *github.Client
 	var githubBaseURL string
+	var isEnterprise bool
 	if len(c.EnterpriseURL) > 0 {
 		var err error
+		isEnterprise = true
 		client, err = github.NewEnterpriseClient(c.EnterpriseURL, c.EnterpriseURL, httpClient)
 		if err != nil {
 			return nil, fmt.Errorf("enterprise client creation failed: %v", err)
@@ -134,14 +138,13 @@ func (c *Config) NewClient() (*Client, error) {
 			}
 		}
 	}
-
-	client.UserAgent = "actions-runner-controller"
-
+	client.UserAgent = "actions-runner-controller/" + build.Version
 	return &Client{
 		Client:        client,
 		regTokens:     map[string]*github.RegistrationToken{},
 		mu:            sync.Mutex{},
 		GithubBaseURL: githubBaseURL,
+		IsEnterprise:  isEnterprise,
 	}, nil
 }
 
@@ -243,29 +246,6 @@ func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string)
 	return runners, nil
 }
 
-// ListOrganizationRunnerGroups returns all the runner groups defined in the organization and
-// inherited to the organization from an enterprise.
-func (c *Client) ListOrganizationRunnerGroups(ctx context.Context, org string) ([]*github.RunnerGroup, error) {
-	var runnerGroups []*github.RunnerGroup
-
-	opts := github.ListOrgRunnerGroupOptions{}
-	opts.PerPage = 100
-	for {
-		list, res, err := c.Client.Actions.ListOrganizationRunnerGroups(ctx, org, &opts)
-		if err != nil {
-			return runnerGroups, fmt.Errorf("failed to list organization runner groups: %w", err)
-		}
-
-		runnerGroups = append(runnerGroups, list.RunnerGroups...)
-		if res.NextPage == 0 {
-			break
-		}
-		opts.Page = res.NextPage
-	}
-
-	return runnerGroups, nil
-}
-
 // ListOrganizationRunnerGroupsForRepository returns all the runner groups defined in the organization and
 // inherited to the organization from an enterprise.
 // We can remove this when google/go-github library is updated to support this.
@@ -439,7 +419,6 @@ func splitOwnerAndRepo(repo string) (string, string, error) {
 	}
 	return chunk[0], chunk[1], nil
 }
-
 func getEnterpriseApiUrl(baseURL string) (string, error) {
 	baseEndpoint, err := url.Parse(baseURL)
 	if err != nil {

github/github_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 )
 
 var server *httptest.Server
@@ -155,7 +155,7 @@ func TestCleanup(t *testing.T) {
 
 func TestUserAgent(t *testing.T) {
 	client := newTestClient()
-	if client.UserAgent != "actions-runner-controller" {
-		t.Errorf("UserAgent should be set to actions-runner-controller")
+	if client.UserAgent != "actions-runner-controller/NA" {
+		t.Errorf("UserAgent should be set to actions-runner-controller/NA")
 	}
 }

go.mod

@@ -1,28 +1,28 @@
 module github.com/actions-runner-controller/actions-runner-controller
 
-go 1.18
+go 1.19
 
 require (
-	github.com/bradleyfalzon/ghinstallation/v2 v2.0.4
+	github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v1.2.3
 	github.com/google/go-cmp v0.5.8
-	github.com/google/go-github/v45 v45.2.0
+	github.com/google/go-github/v47 v47.0.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.19.0
-	github.com/prometheus/client_golang v1.12.2
+	github.com/onsi/gomega v1.20.0
+	github.com/prometheus/client_golang v1.13.0
 	github.com/stretchr/testify v1.8.0
 	github.com/teambition/rrule-go v1.8.0
-	go.uber.org/zap v1.21.0
-	golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0
+	go.uber.org/zap v1.23.0
+	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094
 	gomodules.xyz/jsonpatch/v2 v2.2.0
-	k8s.io/api v0.24.2
-	k8s.io/apimachinery v0.24.2
-	k8s.io/client-go v0.24.2
-	sigs.k8s.io/controller-runtime v0.12.2
+	k8s.io/api v0.24.3
+	k8s.io/apimachinery v0.24.3
+	k8s.io/client-go v0.24.3
+	sigs.k8s.io/controller-runtime v0.12.3
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -40,15 +40,14 @@ require (
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-github/v41 v41.0.0 // indirect
+	github.com/google/go-github/v45 v45.2.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -61,8 +60,8 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
@@ -73,7 +72,7 @@ require (
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -38,7 +38,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
@@ -79,11 +78,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
-github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 h1:tXKVfhE7FcSkhkv0UwkLvPDeZ4kz6OXd0PKPlFqf81M=
-github.com/bradleyfalzon/ghinstallation/v2 v2.0.4/go.mod h1:B40qPqJxWE0jDZgOR1JmaMy+4AY1eBP+IByOvqyAKp0=
+github.com/bradleyfalzon/ghinstallation/v2 v2.1.0 h1:5+NghM1Zred9Z078QEZtm28G/kfDfZN/92gkDlLwGVA=
+github.com/bradleyfalzon/ghinstallation/v2 v2.1.0/go.mod h1:Xg3xPRN5Mcq6GDqeUVhFbjEWMb4JHCyWEeeBGEYQoTU=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
@@ -152,9 +149,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -179,8 +178,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
-github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
+github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -218,7 +217,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
 github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
 github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA=
 github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
@@ -234,13 +232,12 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg=
-github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
 github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI=
 github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28=
+github.com/google/go-github/v47 v47.0.0 h1:eQap5bIRZibukP0VhngWgpuM0zhY4xntqOzn6DhdkE4=
+github.com/google/go-github/v47 v47.0.0/go.mod h1:DRjdvizXE876j0YOZwInB1ESpOcU/xFBClNiQLSdorE=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -265,9 +262,6 @@ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
@@ -337,7 +331,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
@@ -358,7 +351,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -391,10 +383,11 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q=
+github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -413,6 +406,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
 github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
+github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -423,9 +418,10 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -433,6 +429,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -456,17 +454,13 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -477,10 +471,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q=
-github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@@ -494,7 +485,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
@@ -541,6 +531,10 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
 go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
 go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
+go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -550,7 +544,6 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
@@ -635,13 +628,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -656,16 +646,17 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 h1:uBgVQYJLi/m8M0wzp+aGwBWt90gMRoOVf+aWTW10QHI=
 golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
 golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 h1:VnGaRqoLmqZH/3TMLJwYCEWkR4j1nuIU1U9TvbqsDUw=
 golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -717,7 +708,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -736,22 +726,17 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -768,8 +753,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -830,7 +813,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
 golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -859,7 +841,6 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q
 google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
 google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
 google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
-google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -948,10 +929,11 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -963,7 +945,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -981,7 +962,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -994,57 +974,38 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
-k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
-k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI=
 k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg=
-k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI=
-k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ=
+k8s.io/api v0.24.3 h1:tt55QEmKd6L2k5DP6G/ZzdMQKvG5ro4H4teClqm0sTY=
+k8s.io/api v0.24.3/go.mod h1:elGR/XSZrS7z7cSZPzVWaycpJuGIw57j9b95/1PdJNI=
 k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k=
 k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ=
-k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0=
-k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
-k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM=
 k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
-k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw=
+k8s.io/apimachinery v0.24.3 h1:hrFiNSA2cBZqllakVYyH/VyEh4B581bQRmqATJSeQTg=
+k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
 k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI=
-k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8=
-k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
-k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA=
 k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30=
-k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
+k8s.io/client-go v0.24.3 h1:Nl1840+6p4JqkFWEW2LnMKU667BUxw03REfLAVhuKQY=
+k8s.io/client-go v0.24.3/go.mod h1:AAovolf5Z9bY1wIg2FZ8LPQlEdKHjLI7ZD4rw920BJw=
 k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w=
-k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE=
-k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0=
 k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU=
 k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
-k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
 k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
-k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
 k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU=
 k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
-sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA=
-sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4=
-sigs.k8s.io/controller-runtime v0.12.2 h1:nqV02cvhbAj7tbt21bpPpTByrXGn2INHRsi39lXy9sE=
-sigs.k8s.io/controller-runtime v0.12.2/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
-sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
-sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
+sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio=
+sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=

hack/signrel/go.mod

@@ -1,3 +1,3 @@
 module github.com/actions-runner-controller/actions-runner-controller/hack/sigrel
 
-go 1.17
+go 1.19

main.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/actions-runner-controller/actions-runner-controller/build"
 	"github.com/actions-runner-controller/actions-runner-controller/controllers"
 	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/actions-runner-controller/actions-runner-controller/logging"
@@ -62,7 +63,6 @@ func (i *stringSlice) Set(value string) error {
 	*i = append(*i, value)
 	return nil
 }
-
 func main() {
 	var (
 		err      error
@@ -70,11 +70,11 @@ func main() {
 
 		metricsAddr            string
 		enableLeaderElection   bool
+		runnerStatusUpdateHook bool
 		leaderElectionId       string
 		port                   int
 		syncPeriod             time.Duration
 
-		gitHubAPICacheDuration time.Duration
 		defaultScaleDownDelay time.Duration
 
 		runnerImage            string
@@ -87,7 +87,6 @@ func main() {
 
 		commonRunnerLabels commaSeparatedStringSlice
 	)
-
 	var c github.Config
 	err = envconfig.Process("github", &c)
 	if err != nil {
@@ -104,6 +103,7 @@ func main() {
 	flag.Var(&runnerImagePullSecrets, "runner-image-pull-secret", "The default image-pull secret name for self-hosted runner container.")
 	flag.StringVar(&dockerRegistryMirror, "docker-registry-mirror", "", "The default Docker Registry Mirror used by runners.")
 	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
+	flag.StringVar(&c.EnterpriseURL, "github-enterprise-url", c.EnterpriseURL, "Enterprise URL to be used for your GitHub API calls")
 	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
 	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
 	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
@@ -112,7 +112,7 @@ func main() {
 	flag.StringVar(&c.BasicauthUsername, "github-basicauth-username", c.BasicauthUsername, "Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
 	flag.StringVar(&c.BasicauthPassword, "github-basicauth-password", c.BasicauthPassword, "Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
 	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
-	flag.DurationVar(&gitHubAPICacheDuration, "github-api-cache-duration", 0, "DEPRECATED: The duration until the GitHub API cache expires. Setting this to e.g. 10m results in the controller tries its best not to make the same API call within 10m to reduce the chance of being rate-limited. Defaults to mostly the same value as sync-period. If you're tweaking this in order to make autoscaling more responsive, you'll probably want to tweak sync-period, too")
+	flag.BoolVar(&runnerStatusUpdateHook, "runner-status-update-hook", false, "Use custom RBAC for runners (role, role binding and service account).")
 	flag.DurationVar(&defaultScaleDownDelay, "default-scale-down-delay", controllers.DefaultScaleDownDelay, "The approximate delay for a scale down followed by a scale up, used to prevent flapping (down->up->down->... loop)")
 	flag.IntVar(&port, "port", 9443, "The port to which the admission webhook endpoint should bind")
 	flag.DurationVar(&syncPeriod, "sync-period", 1*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled.")
@@ -122,7 +122,6 @@ func main() {
 	flag.Parse()
 
 	logger := logging.NewLogger(logLevel)
-
 	c.Log = &logger
 
 	ghClient, err = c.NewClient()
@@ -147,13 +146,19 @@ func main() {
 		os.Exit(1)
 	}
 
+	multiClient := controllers.NewMultiGitHubClient(
+		mgr.GetClient(),
+		ghClient,
+	)
+
 	runnerReconciler := &controllers.RunnerReconciler{
 		Client:                    mgr.GetClient(),
 		Log:                       log.WithName("runner"),
 		Scheme:                    mgr.GetScheme(),
-		GitHubClient:         ghClient,
+		GitHubClient:              multiClient,
 		DockerImage:               dockerImage,
 		DockerRegistryMirror:      dockerRegistryMirror,
+		UseRunnerStatusUpdateHook: runnerStatusUpdateHook,
 		// Defaults for self-hosted runner containers
 		RunnerImage:            runnerImage,
 		RunnerImagePullSecrets: runnerImagePullSecrets,
@@ -168,7 +173,6 @@ func main() {
 		Client: mgr.GetClient(),
 		Log:    log.WithName("runnerreplicaset"),
 		Scheme: mgr.GetScheme(),
-		GitHubClient: ghClient,
 	}
 
 	if err = runnerReplicaSetReconciler.SetupWithManager(mgr); err != nil {
@@ -195,27 +199,21 @@ func main() {
 		CommonRunnerLabels:   commonRunnerLabels,
 		DockerImage:          dockerImage,
 		DockerRegistryMirror: dockerRegistryMirror,
-		GitHubBaseURL:        ghClient.GithubBaseURL,
+		GitHubClient:         multiClient,
 		// Defaults for self-hosted runner containers
 		RunnerImage:               runnerImage,
 		RunnerImagePullSecrets:    runnerImagePullSecrets,
+		UseRunnerStatusUpdateHook: runnerStatusUpdateHook,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {
 		log.Error(err, "unable to create controller", "controller", "RunnerSet")
 		os.Exit(1)
 	}
-	if gitHubAPICacheDuration == 0 {
-		gitHubAPICacheDuration = syncPeriod - 10*time.Second
-	}
-
-	if gitHubAPICacheDuration < 0 {
-		gitHubAPICacheDuration = 0
-	}
 
 	log.Info(
 		"Initializing actions-runner-controller",
-		"github-api-cache-duration", gitHubAPICacheDuration,
+		"version", build.Version,
 		"default-scale-down-delay", defaultScaleDownDelay,
 		"sync-period", syncPeriod,
 		"default-runner-image", runnerImage,
@@ -230,8 +228,7 @@ func main() {
 		Client:                mgr.GetClient(),
 		Log:                   log.WithName("horizontalrunnerautoscaler"),
 		Scheme:                mgr.GetScheme(),
-		GitHubClient:          ghClient,
-		CacheDuration:         gitHubAPICacheDuration,
+		GitHubClient:          multiClient,
 		DefaultScaleDownDelay: defaultScaleDownDelay,
 	}
 
@@ -239,7 +236,7 @@ func main() {
 		Client:       mgr.GetClient(),
 		Log:          log.WithName("runnerpod"),
 		Scheme:       mgr.GetScheme(),
-		GitHubClient: ghClient,
+		GitHubClient: multiClient,
 	}
 
 	runnerPersistentVolumeReconciler := &controllers.RunnerPersistentVolumeReconciler{
@@ -290,7 +287,7 @@ func main() {
 
 	injector := &controllers.PodRunnerTokenInjector{
 		Client:       mgr.GetClient(),
-		GitHubClient: ghClient,
+		GitHubClient: multiClient,
 		Log:          ctrl.Log.WithName("webhook").WithName("PodRunnerTokenInjector"),
 	}
 	if err = injector.SetupWithManager(mgr); err != nil {

pkg/githubwebhookdeliveryforwarder/githubwebhookdelivery.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v45/github"
+	gogithub "github.com/google/go-github/v47/github"
 )
 
 type server struct {

pkg/hookdeliveryforwarder/forwarder.go

@@ -12,7 +12,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v45/github"
+	gogithub "github.com/google/go-github/v47/github"
 )
 
 type Forwarder struct {

pkg/hookdeliveryforwarder/hooks.go

@@ -3,7 +3,7 @@ package hookdeliveryforwarder
 import (
 	"context"
 
-	gogithub "github.com/google/go-github/v45/github"
+	gogithub "github.com/google/go-github/v47/github"
 )
 
 type hooksAPI struct {

pkg/hookdeliveryforwarder/hooks_deliveries.go

@@ -3,7 +3,7 @@ package hookdeliveryforwarder
 import (
 	"context"
 
-	gogithub "github.com/google/go-github/v45/github"
+	gogithub "github.com/google/go-github/v47/github"
 )
 
 type hookDeliveriesAPI struct {

pkg/hookdeliveryforwarder/multiforwarder.go

@@ -8,7 +8,7 @@ import (
 	"sync"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v45/github"
+	gogithub "github.com/google/go-github/v47/github"
 )
 
 type MultiForwarder struct {

runner/Makefile

@@ -4,7 +4,7 @@ DIND_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind
 TAG ?= latest
 TARGETPLATFORM ?= $(shell arch)
 
-RUNNER_VERSION ?= 2.294.0
+RUNNER_VERSION ?= 2.296.2
 RUNNER_CONTAINER_HOOKS_VERSION ?= 0.1.2
 DOCKER_VERSION ?= 20.10.12
 

runner/actions-runner-dind-rootless.dockerfile

@@ -0,0 +1,138 @@
+FROM ubuntu:20.04
+
+# Target architecture
+ARG TARGETPLATFORM=linux/amd64
+
+# GitHub runner arguments
+ARG RUNNER_VERSION=2.296.2
+
+# Docker and Docker Compose arguments
+ENV CHANNEL=stable
+ARG COMPOSE_VERSION=v2.6.0
+
+# Dumb-init version
+ARG DUMB_INIT_VERSION=1.2.5
+
+# Other arguments
+ARG DEBUG=false
+
+# Set environment variables needed at build
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update -y \
+    && apt-get install -y software-properties-common \
+    && add-apt-repository -y ppa:git-core/ppa \
+    && apt-get update -y \
+    && apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    ca-certificates \
+    dnsutils \
+    ftp \
+    git \
+    iproute2 \
+    iputils-ping \
+    iptables \
+    jq \
+    libunwind8 \
+    locales \
+    netcat \
+    net-tools \
+    openssh-client \
+    parallel \
+    python3-pip \
+    rsync \
+    shellcheck \
+    supervisor \
+    software-properties-common \
+    sudo \
+    telnet \
+    time \
+    tzdata \
+    uidmap \
+    unzip \
+    upx \
+    wget \
+    zip \
+    zstd \
+    && ln -sf /usr/bin/python3 /usr/bin/python \
+    && ln -sf /usr/bin/pip3 /usr/bin/pip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Runner user
+RUN adduser --disabled-password --gecos "" --uid 1000 runner
+
+RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)
+
+# Setup subuid and subgid so that "--userns-remap=default" works
+RUN set -eux; \
+    addgroup --system dockremap; \
+    adduser --system --ingroup dockremap dockremap; \
+    echo 'dockremap:165536:65536' >> /etc/subuid; \
+    echo 'dockremap:165536:65536' >> /etc/subgid
+
+ENV RUNNER_ASSETS_DIR=/runnertmp
+
+# Runner download supports amd64 as x64
+RUN ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && export ARCH \
+    && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
+    && mkdir -p "$RUNNER_ASSETS_DIR" \
+    && cd "$RUNNER_ASSETS_DIR" \
+    && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
+    && tar xzf ./runner.tar.gz \
+    && rm runner.tar.gz \
+    && ./bin/installdependencies.sh \
+    && apt-get install -y libyaml-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN echo AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache > /runner.env \
+    && mkdir /opt/hostedtoolcache \
+    && chgrp runner /opt/hostedtoolcache \
+    && chmod g+rwx /opt/hostedtoolcache
+
+# Configure hooks folder structure.
+COPY hooks /etc/arc/hooks/
+
+# arch command on OS X reports "i386" for Intel CPUs regardless of bitness
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
+    && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x86_64 ; fi \
+    && curl -f -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_${ARCH} \
+    && chmod +x /usr/local/bin/dumb-init
+
+COPY entrypoint.sh logger.bash rootless-startup.sh update-status /usr/bin/
+
+RUN chmod +x /usr/bin/rootless-startup.sh /usr/bin/entrypoint.sh
+
+# Make the rootless runner directory executable
+RUN mkdir /run/user/1000 \
+    && chown runner:runner /run/user/1000 \
+    && chmod a+x /run/user/1000
+
+# Add the Python "User Script Directory" to the PATH
+ENV PATH="${PATH}:${HOME}/.local/bin:/home/runner/bin"
+ENV ImageOS=ubuntu20
+ENV DOCKER_HOST=unix:///run/user/1000/docker.sock
+ENV XDG_RUNTIME_DIR=/run/user/1000
+
+RUN echo "PATH=${PATH}" > /etc/environment \
+    && echo "ImageOS=${ImageOS}" >> /etc/environment \
+    && echo "DOCKER_HOST=${DOCKER_HOST}" >> /etc/environment \
+    && echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> /etc/environment
+
+ENV HOME=/home/runner
+
+# No group definition, as that makes it harder to run docker.
+USER runner
+
+# Docker installation
+ENV SKIP_IPTABLES=1
+RUN curl -fsSL https://get.docker.com/rootless | sh
+
+# Docker-compose installation
+RUN curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-Linux-x86_64" -o /home/runner/bin/docker-compose ; \
+    chmod +x /home/runner/bin/docker-compose
+
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
+CMD ["rootless-startup.sh"]

runner/actions-runner-dind.dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.294.0
+ARG RUNNER_VERSION=2.296.2
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
@@ -98,10 +98,13 @@ RUN mkdir /opt/hostedtoolcache \
 
 # We place the scripts in `/usr/bin` so that users who extend this image can
 # override them with scripts of the same name placed in `/usr/local/bin`.
-COPY entrypoint.sh logger.bash startup.sh /usr/bin/
+COPY entrypoint.sh logger.bash startup.sh update-status /usr/bin/
 COPY supervisor/ /etc/supervisor/conf.d/
 RUN chmod +x /usr/bin/startup.sh /usr/bin/entrypoint.sh
 
+# Configure hooks folder structure.
+COPY hooks /etc/arc/hooks/
+
 # arch command on OS X reports "i386" for Intel CPUs regardless of bitness
 RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
     && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \

runner/actions-runner.dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.294.0
+ARG RUNNER_VERSION=2.296.2
 ARG RUNNER_CONTAINER_HOOKS_VERSION=0.1.2
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
@@ -116,7 +116,10 @@ RUN mkdir /opt/hostedtoolcache \
 
 # We place the scripts in `/usr/bin` so that users who extend this image can
 # override them with scripts of the same name placed in `/usr/local/bin`.
-COPY entrypoint.sh logger.bash /usr/bin/
+COPY entrypoint.sh logger.bash update-status /usr/bin/
+
+# Configure hooks folder structure.
+COPY hooks /etc/arc/hooks/
 
 ENV HOME=/home/runner
 # Add the Python "User Script Directory" to the PATH

runner/entrypoint.sh

@@ -4,6 +4,13 @@ source logger.bash
 RUNNER_ASSETS_DIR=${RUNNER_ASSETS_DIR:-/runnertmp}
 RUNNER_HOME=${RUNNER_HOME:-/runner}
 
+# Let GitHub runner execute these hooks. These environment variables are used by GitHub's Runner as described here
+# https://github.com/actions/runner/blob/main/docs/adrs/1751-runner-job-hooks.md
+# Scripts referenced in the ACTIONS_RUNNER_HOOK_ environment variables must end in .sh or .ps1
+# for it to become a valid hook script, otherwise GitHub will fail to run the hook
+export ACTIONS_RUNNER_HOOK_JOB_STARTED=/etc/arc/hooks/job-started.sh
+export ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/etc/arc/hooks/job-completed.sh
+
 if [ ! -z "${STARTUP_DELAY_IN_SECONDS}" ]; then
   log.notice "Delaying startup by ${STARTUP_DELAY_IN_SECONDS} seconds"
   sleep ${STARTUP_DELAY_IN_SECONDS}
@@ -77,6 +84,8 @@ if [ "${DISABLE_RUNNER_UPDATE:-}" == "true" ]; then
   log.debug 'Passing --disableupdate to config.sh to disable automatic runner updates.'
 fi
 
+update-status "Registering"
+
 retries_left=10
 while [[ ${retries_left} -gt 0 ]]; do
   log.debug 'Configuring the runner.'
@@ -155,4 +164,5 @@ unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN STARTUP_DELAY_IN_SECONDS DISABLE_WAIT
 if [ -z "${UNITTEST:-}" ]; then
   mapfile -t env </etc/environment
 fi
+update-status "Idle"
 exec env -- "${env[@]}" ./run.sh

runner/hooks/job-completed.d/update-status

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -u
+
+exec update-status Idle

runner/hooks/job-completed.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# shellcheck source=runner/logger.bash
+source logger.bash
+
+log.debug "Running ARC Job Completed Hooks"
+
+for hook in /etc/arc/hooks/job-completed.d/*; do
+  log.debug "Running hook: $hook"
+  "$hook" "$@"
+done

runner/hooks/job-started.d/update-status

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -u
+
+exec update-status Running "Run $GITHUB_RUN_ID from $GITHUB_REPOSITORY"

runner/hooks/job-started.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# shellcheck source=runner/logger.bash
+source logger.bash
+
+log.debug "Running ARC Job Started Hooks"
+
+for hook in /etc/arc/hooks/job-started.d/*; do
+  log.debug "Running hook: $hook"
+  "$hook" "$@"
+done

runner/rootless-startup.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+source logger.bash
+
+log.notice "Writing out Docker config file"
+/bin/bash <<SCRIPT
+mkdir -p /home/runner/.config/docker/
+
+if [ ! -f /home/runner/.config/docker/daemon.json ]; then
+  echo "{}" > /home/runner/.config/docker/daemon.json
+fi
+
+if [ -n "${MTU}" ]; then
+jq ".\"mtu\" = ${MTU}" /home/runner/.config/docker/daemon.json > /tmp/.daemon.json && mv /tmp/.daemon.json /home/runner/.config/docker/daemon.json
+# See https://docs.docker.com/engine/security/rootless/
+echo "environment=DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=${MTU}" >> /etc/supervisor/conf.d/dockerd.conf
+fi
+
+if [ -n "${DOCKER_REGISTRY_MIRROR}" ]; then
+jq ".\"registry-mirrors\"[0] = \"${DOCKER_REGISTRY_MIRROR}\"" /home/runner/.config/docker/daemon.json > /tmp/.daemon.json && mv /tmp/.daemon.json /home/runner/.config/docker/daemon.json
+fi
+SCRIPT
+
+log.notice "Starting Docker (rootless)"
+/home/runner/bin/dockerd-rootless.sh --config-file /home/runner/.config/docker/daemon.json >> /dev/null 2>&1 &
+
+# Wait processes to be running
+entrypoint.sh

runner/update-status

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ ${1:-} == '' ]]; then
+  # shellcheck source=runner/logger.bash
+  source logger.bash
+  log.error "Missing required argument -- '<phase>'"
+  exit 64
+fi
+
+if [[ ${RUNNER_STATUS_UPDATE_HOOK:-false} == true ]]; then
+
+    apiserver=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}
+    serviceaccount=/var/run/secrets/kubernetes.io/serviceaccount
+    namespace=$(cat ${serviceaccount}/namespace)
+    token=$(cat ${serviceaccount}/token)
+    phase=$1
+    shift
+
+    jq -n --arg phase "$phase" --arg message "${*:-}" '.status.phase = $phase | .status.message = $message' | curl \
+        --cacert ${serviceaccount}/ca.crt \
+        --data @- \
+        --noproxy '*' \
+        --header "Content-Type: application/merge-patch+json" \
+        --header "Authorization: Bearer ${token}" \
+        --show-error \
+        --silent \
+        --request PATCH \
+        "${apiserver}/apis/actions.summerwind.dev/v1alpha1/namespaces/${namespace}/runners/${HOSTNAME}/status"
+        1>&-
+fi

simulator/runnergroup_visibility.go

@@ -20,7 +20,6 @@ func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org,
 		panic(fmt.Sprintf("BUG: owner should not be empty in this context. repo=%v", repo))
 	}
 
-	if c.Client.GithubBaseURL == "https://github.com/" {
 	runnerGroups, err := c.Client.ListOrganizationRunnerGroupsForRepository(ctx, org, repo)
 	if err != nil {
 		return visible, err
@@ -39,33 +38,6 @@ func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org,
 
 		visible.Add(ref)
 	}
-	} else {
-		runnerGroups, err := c.Client.ListOrganizationRunnerGroups(ctx, org)
-		if err != nil {
-			return visible, err
-		}
-
-		for _, runnerGroup := range runnerGroups {
-			ref := NewRunnerGroupFromGitHub(runnerGroup)
-
-			if !managed.Includes(ref) {
-				continue
-			}
-
-			if runnerGroup.GetVisibility() != "all" {
-				hasAccess, err := c.hasRepoAccessToOrganizationRunnerGroup(ctx, org, runnerGroup.GetID(), repo)
-				if err != nil {
-					return visible, err
-				}
-
-				if !hasAccess {
-					continue
-				}
-			}
-
-			visible.Add(ref)
-		}
-	}
 
 	return visible, nil
 }

simulator/runnergroups.go

@@ -5,7 +5,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/google/go-github/v45/github"
+	"github.com/google/go-github/v47/github"
 )
 
 type RunnerGroupScope int

test/e2e/e2e_test.go

@@ -3,13 +3,18 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/testing"
+	"github.com/google/go-github/v47/github"
 	"github.com/onsi/gomega"
+	"golang.org/x/oauth2"
 	"sigs.k8s.io/yaml"
 )
 
@@ -21,6 +26,8 @@ const (
 )
 
 var (
+	// See the below link for maintained versions of cert-manager
+	// https://cert-manager.io/docs/installation/supported-releases/
 	certManagerVersion = "v1.8.2"
 
 	images = []testing.ContainerImage{
@@ -32,6 +39,8 @@ var (
 	}
 
 	testResultCMNamePrefix = "test-result-"
+
+	RunnerVersion = "2.296.0"
 )
 
 // If you're willing to run this test via VS Code "run test" or "debug test",
@@ -43,6 +52,7 @@ var (
 // But messages logged via Logf shows up only when the test failed by default.
 // To always enable logging, do not forget to pass `-test.v` to `go test`.
 // If you're using VS Code, open `Workspace Settings` and search for `go test flags`, edit the `.vscode/settings.json` and put the below:
+//
 //	"go.testFlags": ["-v"]
 //
 // This function requires a few environment variables to be set to provide some test data.
@@ -57,11 +67,15 @@ var (
 // https://terratest.gruntwork.io/docs/testing-best-practices/iterating-locally-using-test-stages/
 //
 // This functions leaves PVs undeleted. To delete PVs, run:
+//
 //	kubectl get pv -ojson | jq -rMc '.items[] | select(.status.phase == "Available") | {name:.metadata.name, status:.status.phase} | .name' | xargs kubectl delete pv
 //
 // If you disk full after dozens of test runs, try:
+//
 //	docker system prune
+//
 // and
+//
 //	kind delete cluster --name teste2e
 //
 // The former tend to release 200MB-3GB and the latter can result in releasing like 100GB due to kind node contains loaded container images and
@@ -79,6 +93,29 @@ func TestE2E(t *testing.T) {
 
 	vars := buildVars(os.Getenv("ARC_E2E_IMAGE_REPO"))
 
+	var testedVersions = []struct {
+		label                     string
+		controller, controllerVer string
+		chart, chartVer           string
+	}{
+		{
+			label:         "stable",
+			controller:    "summerwind/actions-runner-controller",
+			controllerVer: "v0.25.2",
+			chart:         "actions-runner-controller/actions-runner-controller",
+			// 0.20.2 accidentally added support for runner-status-update which isn't supported by ARC 0.25.2.
+			// With some chart values, the controller end up with crashlooping with `flag provided but not defined: -runner-status-update-hook`.
+			chartVer: "0.20.1",
+		},
+		{
+			label:         "edge",
+			controller:    vars.controllerImageRepo,
+			controllerVer: vars.controllerImageTag,
+			chart:         "",
+			chartVer:      "",
+		},
+	}
+
 	env := initTestEnv(t, k8sMinorVer, vars)
 	if vt := os.Getenv("ARC_E2E_VERIFY_TIMEOUT"); vt != "" {
 		var err error
@@ -87,6 +124,7 @@ func TestE2E(t *testing.T) {
 			t.Fatalf("Failed to parse duration %q: %v", vt, err)
 		}
 	}
+	env.doDockerBuild = os.Getenv("ARC_E2E_DO_DOCKER_BUILD") != ""
 
 	t.Run("build and load images", func(t *testing.T) {
 		env.buildAndLoadImages(t)
@@ -105,6 +143,10 @@ func TestE2E(t *testing.T) {
 	}
 
 	t.Run("RunnerSets", func(t *testing.T) {
+		if os.Getenv("ARC_E2E_SKIP_RUNNERSETS") != "" {
+			t.Skip("RunnerSets test has been skipped due to ARC_E2E_SKIP_RUNNERSETS")
+		}
+
 		var (
 			testID string
 		)
@@ -119,9 +161,9 @@ func TestE2E(t *testing.T) {
 			})
 		}
 
-		t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
-			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
-		})
+		if t.Failed() {
+			return
+		}
 
 		t.Run("install argo-tunnel", func(t *testing.T) {
 			env.installArgoTunnel(t)
@@ -133,6 +175,24 @@ func TestE2E(t *testing.T) {
 			})
 		}
 
+		if t.Failed() {
+			return
+		}
+
+		for i, v := range testedVersions {
+			t.Run("install actions-runner-controller "+v.label, func(t *testing.T) {
+				t.Logf("Using controller %s:%s and chart %s:%s", v.controller, v.controllerVer, v.chart, v.chartVer)
+				env.installActionsRunnerController(t, v.controller, v.controllerVer, testID, v.chart, v.chartVer)
+			})
+
+			if t.Failed() {
+				return
+			}
+
+			if i > 0 {
+				continue
+			}
+
 			t.Run("deploy runners", func(t *testing.T) {
 				env.deploy(t, RunnerSets, testID)
 			})
@@ -143,13 +203,10 @@ func TestE2E(t *testing.T) {
 				})
 			}
 
-		t.Run("install edge actions-runner-controller", func(t *testing.T) {
-			env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
-		})
-
 			if t.Failed() {
 				return
 			}
+		}
 
 		t.Run("Install workflow", func(t *testing.T) {
 			env.installActionsWorkflow(t, RunnerSets, testID)
@@ -159,12 +216,37 @@ func TestE2E(t *testing.T) {
 			return
 		}
 
+		ctx, cancel := context.WithCancel(context.Background())
+		go func() {
+			for i := 1; ; i++ {
+				select {
+				case _, ok := <-ctx.Done():
+					if !ok {
+						t.Logf("Stopping the continuous rolling-update of runners")
+					}
+				default:
+					time.Sleep(60 * time.Second)
+
+					t.Run(fmt.Sprintf("update runners attempt %d", i), func(t *testing.T) {
+						env.deploy(t, RunnerSets, testID, fmt.Sprintf("ROLLING_UPDATE_PHASE=%d", i))
+					})
+				}
+			}
+		}()
+		t.Cleanup(func() {
+			cancel()
+		})
+
 		t.Run("Verify workflow run result", func(t *testing.T) {
 			env.verifyActionsWorkflowRun(t, testID)
 		})
 	})
 
 	t.Run("RunnerDeployments", func(t *testing.T) {
+		if os.Getenv("ARC_E2E_SKIP_RUNNERDEPLOYMENT") != "" {
+			t.Skip("RunnerSets test has been skipped due to ARC_E2E_SKIP_RUNNERSETS")
+		}
+
 		var (
 			testID string
 		)
@@ -179,9 +261,9 @@ func TestE2E(t *testing.T) {
 			})
 		}
 
-		t.Run("install actions-runner-controller v0.24.1", func(t *testing.T) {
-			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
-		})
+		if t.Failed() {
+			return
+		}
 
 		t.Run("install argo-tunnel", func(t *testing.T) {
 			env.installArgoTunnel(t)
@@ -193,6 +275,24 @@ func TestE2E(t *testing.T) {
 			})
 		}
 
+		if t.Failed() {
+			return
+		}
+
+		for i, v := range testedVersions {
+			t.Run("install actions-runner-controller "+v.label, func(t *testing.T) {
+				t.Logf("Using controller %s:%s and chart %s:%s", v.controller, v.controllerVer, v.chart, v.chartVer)
+				env.installActionsRunnerController(t, v.controller, v.controllerVer, testID, v.chart, v.chartVer)
+			})
+
+			if t.Failed() {
+				return
+			}
+
+			if i > 0 {
+				continue
+			}
+
 			t.Run("deploy runners", func(t *testing.T) {
 				env.deploy(t, RunnerDeployments, testID)
 			})
@@ -203,13 +303,10 @@ func TestE2E(t *testing.T) {
 				})
 			}
 
-		t.Run("install edge actions-runner-controller", func(t *testing.T) {
-			env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
-		})
-
 			if t.Failed() {
 				return
 			}
+		}
 
 		t.Run("Install workflow", func(t *testing.T) {
 			env.installActionsWorkflow(t, RunnerDeployments, testID)
@@ -219,6 +316,27 @@ func TestE2E(t *testing.T) {
 			return
 		}
 
+		ctx, cancel := context.WithCancel(context.Background())
+		go func() {
+			for i := 1; ; i++ {
+				select {
+				case _, ok := <-ctx.Done():
+					if !ok {
+						t.Logf("Stopping the continuous rolling-update of runners")
+					}
+				default:
+					time.Sleep(10 * time.Second)
+
+					t.Run(fmt.Sprintf("update runners - attempt %d", i), func(t *testing.T) {
+						env.deploy(t, RunnerDeployments, testID, fmt.Sprintf("ROLLING_UPDATE_PHASE=%d", i))
+					})
+				}
+			}
+		}()
+		t.Cleanup(func() {
+			cancel()
+		})
+
 		t.Run("Verify workflow run result", func(t *testing.T) {
 			env.verifyActionsWorkflowRun(t, testID)
 		})
@@ -248,8 +366,14 @@ type env struct {
 	scaleDownDelaySecondsAfterScaleOut          int64
 	minReplicas                                 int64
 	dockerdWithinRunnerContainer                bool
+	rootlessDocker                              bool
+	doDockerBuild                               bool
+	containerMode                               string
+	runnerServiceAccuontName                    string
+	runnerNamespace                             string
 	remoteKubeconfig                            string
 	imagePullSecretName                         string
+	imagePullPolicy                             string
 
 	vars          vars
 	VerifyTimeout time.Duration
@@ -260,6 +384,7 @@ type vars struct {
 
 	runnerImageRepo             string
 	runnerDindImageRepo         string
+	runnerRootlessDindImageRepo string
 
 	prebuildImages []testing.ContainerImage
 	builds         []testing.DockerBuild
@@ -278,15 +403,18 @@ func buildVars(repo string) vars {
 		controllerImage             = testing.Img(controllerImageRepo, controllerImageTag)
 		runnerImageRepo             = repo + "/actions-runner"
 		runnerDindImageRepo         = repo + "/actions-runner-dind"
+		runnerRootlessDindImageRepo = repo + "/actions-runner-rootless-dind"
 		runnerImageTag              = "e2e"
 		runnerImage                 = testing.Img(runnerImageRepo, runnerImageTag)
 		runnerDindImage             = testing.Img(runnerDindImageRepo, runnerImageTag)
+		runnerRootlessDindImage     = testing.Img(runnerRootlessDindImageRepo, runnerImageTag)
 	)
 
 	var vs vars
 
 	vs.controllerImageRepo, vs.controllerImageTag = controllerImageRepo, controllerImageTag
 	vs.runnerDindImageRepo = runnerDindImageRepo
+	vs.runnerRootlessDindImageRepo = runnerRootlessDindImageRepo
 	vs.runnerImageRepo = runnerImageRepo
 
 	// vs.controllerImage, vs.controllerImageTag
@@ -295,6 +423,7 @@ func buildVars(repo string) vars {
 		controllerImage,
 		runnerImage,
 		runnerDindImage,
+		runnerRootlessDindImage,
 	}
 
 	vs.builds = []testing.DockerBuild{
@@ -309,7 +438,7 @@ func buildVars(repo string) vars {
 			Args: []testing.BuildArg{
 				{
 					Name:  "RUNNER_VERSION",
-					Value: "2.294.0",
+					Value: RunnerVersion,
 				},
 			},
 			Image:        runnerImage,
@@ -320,12 +449,23 @@ func buildVars(repo string) vars {
 			Args: []testing.BuildArg{
 				{
 					Name:  "RUNNER_VERSION",
-					Value: "2.294.0",
+					Value: RunnerVersion,
 				},
 			},
 			Image:        runnerDindImage,
 			EnableBuildX: true,
 		},
+		{
+			Dockerfile: "../../runner/actions-runner-dind-rootless.dockerfile",
+			Args: []testing.BuildArg{
+				{
+					Name:  "RUNNER_VERSION",
+					Value: RunnerVersion,
+				},
+			},
+			Image:        runnerRootlessDindImage,
+			EnableBuildX: true,
+		},
 	}
 
 	vs.commonScriptEnv = []string{
@@ -359,10 +499,18 @@ func initTestEnv(t *testing.T, k8sMinorVer string, vars vars) *env {
 	e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO", "")
 	e.testEnterprise = testing.Getenv(t, "TEST_ENTERPRISE", "")
 	e.testEphemeral = testing.Getenv(t, "TEST_EPHEMERAL", "")
+	e.runnerServiceAccuontName = testing.Getenv(t, "TEST_RUNNER_SERVICE_ACCOUNT_NAME", "")
+	e.runnerNamespace = testing.Getenv(t, "TEST_RUNNER_NAMESPACE", "default")
 	e.remoteKubeconfig = testing.Getenv(t, "ARC_E2E_REMOTE_KUBECONFIG", "")
 	e.imagePullSecretName = testing.Getenv(t, "ARC_E2E_IMAGE_PULL_SECRET_NAME", "")
 	e.vars = vars
 
+	if e.remoteKubeconfig != "" {
+		e.imagePullPolicy = "Always"
+	} else {
+		e.imagePullPolicy = "IfNotPresent"
+	}
+
 	if e.remoteKubeconfig == "" {
 		e.Kind = testing.StartKind(t, k8sMinorVer, testing.Preload(images...))
 		e.Env.Kubeconfig = e.Kind.Kubeconfig()
@@ -386,9 +534,88 @@ func initTestEnv(t *testing.T, k8sMinorVer string, vars vars) *env {
 		panic(fmt.Sprintf("unable to parse bool from TEST_RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER: %v", err))
 	}
 
+	e.rootlessDocker, err = strconv.ParseBool(testing.Getenv(t, "TEST_RUNNER_ROOTLESS_DOCKER", "false"))
+	if err != nil {
+		panic(fmt.Sprintf("unable to parse bool from TEST_RUNNER_ROOTLESS_DOCKER: %v", err))
+	}
+
+	e.containerMode = testing.Getenv(t, "TEST_CONTAINER_MODE", "")
+	if err != nil {
+		panic(fmt.Sprintf("unable to parse bool from TEST_CONTAINER_MODE: %v", err))
+	}
+
+	if err := e.checkGitHubToken(t, e.githubToken); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := e.checkGitHubToken(t, e.githubTokenWebhook); err != nil {
+		t.Fatal(err)
+	}
+
 	return e
 }
 
+func (e *env) checkGitHubToken(t *testing.T, tok string) error {
+	t.Helper()
+
+	ctx := context.Background()
+
+	transport := oauth2.NewClient(ctx, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: tok})).Transport
+	c := github.NewClient(&http.Client{Transport: transport})
+	aa, res, err := c.Octocat(context.Background(), "hello")
+	if err != nil {
+		b, ioerr := io.ReadAll(res.Body)
+		if ioerr != nil {
+			t.Logf("%v", ioerr)
+			return err
+		}
+		t.Logf(string(b))
+		return err
+	}
+
+	t.Logf("%s", aa)
+
+	if e.testEnterprise != "" {
+		if _, res, err := c.Enterprise.CreateRegistrationToken(ctx, e.testEnterprise); err != nil {
+			b, ioerr := io.ReadAll(res.Body)
+			if ioerr != nil {
+				t.Logf("%v", ioerr)
+				return err
+			}
+			t.Logf(string(b))
+			return err
+		}
+	}
+
+	if e.testOrg != "" {
+		if _, res, err := c.Actions.CreateOrganizationRegistrationToken(ctx, e.testOrg); err != nil {
+			b, ioerr := io.ReadAll(res.Body)
+			if ioerr != nil {
+				t.Logf("%v", ioerr)
+				return err
+			}
+			t.Logf(string(b))
+			return err
+		}
+	}
+
+	if e.testRepo != "" {
+		s := strings.Split(e.testRepo, "/")
+		owner, repo := s[0], s[1]
+		if _, res, err := c.Actions.CreateRegistrationToken(ctx, owner, repo); err != nil {
+			b, ioerr := io.ReadAll(res.Body)
+			if ioerr != nil {
+				t.Logf("%v", ioerr)
+				return err
+			}
+			t.Logf(string(b))
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (e *env) f() {
 }
 
@@ -437,7 +664,7 @@ func (e *env) installCertManager(t *testing.T) {
 	e.KubectlWaitUntilDeployAvailable(t, "cert-manager", waitCfg.WithTimeout(60*time.Second))
 }
 
-func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID string) {
+func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID, chart, chartVer string) {
 	t.Helper()
 
 	e.createControllerNamespaceAndServiceAccount(t)
@@ -445,6 +672,8 @@ func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID str
 	scriptEnv := []string{
 		"KUBECONFIG=" + e.Kubeconfig,
 		"ACCEPTANCE_TEST_DEPLOYMENT_TOOL=" + "helm",
+		"CHART=" + chart,
+		"CHART_VERSION=" + chartVer,
 	}
 
 	varEnv := []string{
@@ -453,6 +682,7 @@ func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID str
 		"NAME=" + repo,
 		"VERSION=" + tag,
 		"IMAGE_PULL_SECRET=" + e.imagePullSecretName,
+		"IMAGE_PULL_POLICY=" + e.imagePullPolicy,
 	}
 
 	if e.useApp {
@@ -475,9 +705,9 @@ func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID str
 	e.RunScript(t, "../../acceptance/deploy.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
 }
 
-func (e *env) deploy(t *testing.T, kind DeployKind, testID string) {
+func (e *env) deploy(t *testing.T, kind DeployKind, testID string, env ...string) {
 	t.Helper()
-	e.do(t, "apply", kind, testID)
+	e.do(t, "apply", kind, testID, env...)
 }
 
 func (e *env) undeploy(t *testing.T, kind DeployKind, testID string) {
@@ -485,7 +715,7 @@ func (e *env) undeploy(t *testing.T, kind DeployKind, testID string) {
 	e.do(t, "delete", kind, testID)
 }
 
-func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
+func (e *env) do(t *testing.T, op string, kind DeployKind, testID string, env ...string) {
 	t.Helper()
 
 	e.createControllerNamespaceAndServiceAccount(t)
@@ -493,7 +723,10 @@ func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
 	scriptEnv := []string{
 		"KUBECONFIG=" + e.Kubeconfig,
 		"OP=" + op,
+		"RUNNER_NAMESPACE=" + e.runnerNamespace,
+		"RUNNER_SERVICE_ACCOUNT_NAME=" + e.runnerServiceAccuontName,
 	}
+	scriptEnv = append(scriptEnv, env...)
 
 	switch kind {
 	case RunnerSets:
@@ -515,13 +748,27 @@ func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
 		fmt.Sprintf("REPO_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
 		fmt.Sprintf("ORG_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
 		fmt.Sprintf("ENTERPRISE_RUNNER_MIN_REPLICAS=%d", e.minReplicas),
+		"RUNNER_CONTAINER_MODE=" + e.containerMode,
+	}
+
+	if e.dockerdWithinRunnerContainer && e.containerMode == "kubernetes" {
+		t.Fatalf("TEST_RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER cannot be set along with TEST_CONTAINER_MODE=kubernetes")
+		t.FailNow()
 	}
 
 	if e.dockerdWithinRunnerContainer {
 		varEnv = append(varEnv,
 			"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=true",
+		)
+		if e.rootlessDocker {
+			varEnv = append(varEnv,
+				"RUNNER_NAME="+e.vars.runnerRootlessDindImageRepo,
+			)
+		} else {
+			varEnv = append(varEnv,
 				"RUNNER_NAME="+e.vars.runnerDindImageRepo,
 			)
+		}
 	} else {
 		varEnv = append(varEnv,
 			"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=false",
@@ -571,7 +818,7 @@ func (e *env) createControllerNamespaceAndServiceAccount(t *testing.T) {
 func (e *env) installActionsWorkflow(t *testing.T, kind DeployKind, testID string) {
 	t.Helper()
 
-	installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID))
+	installActionsWorkflow(t, e.testName+" "+testID, e.runnerLabel(testID), testResultCMNamePrefix, e.repoToCommit, kind, e.testJobs(testID), !e.rootlessDocker, e.doDockerBuild)
 }
 
 func (e *env) testJobs(testID string) []job {
@@ -612,7 +859,8 @@ func createTestJobs(id, testResultCMNamePrefix string, numJobs int) []job {
 
 const Branch = "main"
 
-func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job) {
+// useSudo also implies rootful docker and the use of buildx cache export/import
+func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNamePrefix, testRepo string, kind DeployKind, testJobs []job, useSudo, doDockerBuild bool) {
 	t.Helper()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
@@ -643,26 +891,54 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 			},
 		}
 
+		var sudo string
+		if useSudo {
+			sudo = "sudo "
+		}
+
 		if !kubernetesContainerMode {
 			if kind == RunnerDeployments {
 				steps = append(steps,
 					testing.Step{
-						Run: "sudo mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
+						Run: sudo + "mkdir -p \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
+					},
+				)
+
+				if useSudo {
+					steps = append(steps,
+						testing.Step{
+							// This might be the easiest way to handle permissions without use of securityContext
+							// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
+							Run: sudo + "mkdir -p \"/var/lib/docker\"",
+						},
+					)
+				}
+			}
+
+			if useSudo {
+				steps = append(steps,
+					testing.Step{
+						// This might be the easiest way to handle permissions without use of securityContext
+						// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
+						Run: sudo + "chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
+					},
+					testing.Step{
+						Run: sudo + "chmod 777 -R \"/var/lib/docker\"",
+					},
+					testing.Step{
+						// This might be the easiest way to handle permissions without use of securityContext
+						// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
+						Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\"",
+					},
+					testing.Step{
+						// This might be the easiest way to handle permissions without use of securityContext
+						// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
+						Run: "ls -lah \"/var/lib/docker\" || echo ls failed.",
 					},
 				)
 			}
 
 			steps = append(steps,
-				testing.Step{
-					// This might be the easiest way to handle permissions without use of securityContext
-					// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
-					Run: "sudo chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
-				},
-				testing.Step{
-					// This might be the easiest way to handle permissions without use of securityContext
-					// https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320
-					Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"",
-				},
 				testing.Step{
 					Uses: "actions/setup-go@v3",
 					With: &testing.With{
@@ -681,7 +957,28 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 			},
 		)
 
+		if doDockerBuild {
 			if !kubernetesContainerMode {
+				setupBuildXActionWith := &testing.With{
+					BuildkitdFlags: "--debug",
+					Endpoint:       "mycontext",
+					// As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build`
+					// so we need to use `docker buildx build` in the next step
+					Install: false,
+				}
+				var dockerBuildCache, dockerfile string
+				if useSudo {
+					// This needs to be set only when rootful docker mode.
+					// When rootless, we need to use the `docker` buildx driver, which doesn't support cache export
+					// so we end up with the below error on docker-build:
+					//   error: cache export feature is currently not supported for docker driver. Please switch to a different driver (eg. "docker buildx create --use")
+					dockerBuildCache = "--cache-from=type=local,src=/home/runner/.cache/buildx " +
+						"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max "
+					dockerfile = "Dockerfile"
+				} else {
+					setupBuildXActionWith.Driver = "docker"
+					dockerfile = "Dockerfile.nocache"
+				}
 				steps = append(steps,
 					testing.Step{
 						// https://github.com/docker/buildx/issues/413#issuecomment-710660155
@@ -695,20 +992,19 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 					testing.Step{
 						Name: "Set up Docker Buildx",
 						Uses: "docker/setup-buildx-action@v1",
-					With: &testing.With{
-						BuildkitdFlags: "--debug",
-						Endpoint:       "mycontext",
-						// As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build`
-						// so we need to use `docker buildx build` in the next step
-						Install: false,
-					},
+						With: setupBuildXActionWith,
 					},
 					testing.Step{
 						Run: "docker buildx build --platform=linux/amd64 " +
-						"--cache-from=type=local,src=/home/runner/.cache/buildx " +
-						"--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max " +
-						".",
+							dockerBuildCache +
+							fmt.Sprintf("-f %s .", dockerfile),
 					},
+				)
+			}
+		}
+
+		if useSudo {
+			steps = append(steps,
 				testing.Step{
 					// https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache
 					// See https://github.com/moby/buildkit/issues/1896 for why this is needed
@@ -717,6 +1013,10 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 				testing.Step{
 					Run: "ls -lah /home/runner/.cache/*",
 				},
+			)
+		}
+
+		steps = append(steps,
 			testing.Step{
 				Uses: "azure/setup-kubectl@v1",
 				With: &testing.With{
@@ -727,7 +1027,6 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam
 				Run: fmt.Sprintf("./test.sh %s %s", t.Name(), j.testArg),
 			},
 		)
-		}
 
 		wf.Jobs[j.name] = testing.Job{
 			RunsOn:    runnerLabel,

testing/workflow.go

@@ -52,4 +52,7 @@ type With struct {
 	// This can be either the address or the context name
 	// https://github.com/docker/buildx/blob/master/docs/reference/buildx_create.md#description
 	Endpoint string `json:"endpoint,omitempty"`
+	// Needs to be "docker" in rootless mode
+	// https://stackoverflow.com/questions/66142872/how-to-solve-error-with-rootless-docker-in-github-actions-self-hosted-runner-wr
+	Driver string `json:"driver,omitempty"`
 }