Fix the new ct chart lint error

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.github/workflows/publish-chart.yaml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3.0
+        uses: azure/setup-helm@v3.1
         with:
           version: ${{ env.HELM_VERSION }}
 

.github/workflows/validate-chart.yaml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3.0
+        uses: azure/setup-helm@v3.1
         with:
           version: ${{ env.HELM_VERSION }}
 

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.18.3 as builder
+FROM --platform=$BUILDPLATFORM golang:1.18.4 as builder
 
 WORKDIR /workspace
 

README.md

@@ -29,8 +29,9 @@ ToC:
     - [Webhook Driven Scaling](#webhook-driven-scaling)
     - [Autoscaling to/from 0](#autoscaling-tofrom-0)
     - [Scheduled Overrides](#scheduled-overrides)
-  - [Runner with DinD](#runner-with-dind)
-  - [Runner with k8s jobs](#runner-with-k8s-jobs)
+  - [Alternative Runners](#alternative-runners)
+    - [Runner with DinD](#runner-with-dind)
+    - [Runner with k8s jobs](#runner-with-k8s-jobs)
   - [Additional Tweaks](#additional-tweaks)
   - [Custom Volume mounts](#custom-volume-mounts)
   - [Runner Labels](#runner-labels)
@@ -39,6 +40,7 @@ ToC:
   - [Using IRSA (IAM Roles for Service Accounts) in EKS](#using-irsa-iam-roles-for-service-accounts-in-eks)
   - [Software Installed in the Runner Image](#software-installed-in-the-runner-image)
   - [Using without cert-manager](#using-without-cert-manager)
+  - [Multitenancy](#multitenancy)
 - [Troubleshooting](#troubleshooting)
 - [Contributing](#contributing)
 
@@ -256,7 +258,7 @@ You can deploy multiple controllers either in a single shared namespace, or in a
 
 If you plan on installing all instances of the controller stack into a single namespace there are a few things you need to do for this to work.
 
-1. All resources per stack must have a unique, in the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
+1. All resources per stack must have a unique name, in the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
 2. `authSecret.name` needs to be unique per stack when each stack is tied to runners in different GitHub organizations and repositories AND you want your GitHub credentials to be narrowly scoped.
 3. `leaderElectionId` needs to be unique per stack. If this is not unique to the stack the controller tries to race onto the leader election lock resulting in only one stack working concurrently. Your controller will be stuck with a log message something like this `attempting to acquire leader lease arc-controllers/actions-runner-controller...`
 4. The MutatingWebhookConfiguration in each stack must include a namespace selector for that stack's corresponding runner namespace, this is already configured in the helm chart.
@@ -270,52 +272,50 @@ Alternatively, you can install each controller stack into a unique namespace (re
 - The organization level
 - The enterprise level
 
-There are two ways to use this controller:
+Runners can be deployed as 1 of 2 abstractions: 
 
-- Manage runners one by one with `Runner`.
-- Manage a set of runners with `RunnerDeployment`.
+- A `RunnerDeployment` (similar to k8s's `Deployments`, based on `Pods`)
+- A `RunnerSet` (based on k8s's `StatefulSets`)
+
+We go into details about the differences between the 2 later, initially lets look at how to deploy a basic `RunnerDeployment` at the 3 possible management hierarchies.
 
 ### Repository Runners
 
-To launch a single self-hosted runner, you need to create a manifest file that includes a `Runner` resource as follows. This example launches a self-hosted runner with name *example-runner* for the *actions-runner-controller/actions-runner-controller* repository.
+To launch a single self-hosted runner, you need to create a manifest file that includes a `RunnerDeployment` resource as follows. This example launches a self-hosted runner with name *example-runnerdeploy* for the *actions-runner-controller/actions-runner-controller* repository.
 
 ```yaml
-# runner.yaml
+# runnerdeployment.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
-kind: Runner
+kind: RunnerDeployment
 metadata:
-  name: example-runner
+  name: example-runnerdeploy
 spec:
-  repository: example/myrepo
-  env: []
+  replicas: 1
+  template:
+    spec:
+      repository: mumoshu/actions-runner-controller-ci
 ```
 
 Apply the created manifest file to your Kubernetes.
 
 ```shell
-$ kubectl apply -f runner.yaml
-runner.actions.summerwind.dev/example-runner created
+$ kubectl apply -f runnerdeployment.yaml
+runnerdeployment.actions.summerwind.dev/example-runnerdeploy created
 ```
 
-You can see that the Runner resource has been created.
+You can see that 1 runner and its underlying pod has been created as specified by `replicas: 1` attribute:
 
 ```shell
 $ kubectl get runners
-NAME             REPOSITORY                             STATUS
-example-runner   actions-runner-controller/actions-runner-controller   Running
-```
+NAME                             REPOSITORY                             STATUS
+example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
 
-You can also see that the runner pod has been running.
-
-```shell
 $ kubectl get pods
-NAME           READY   STATUS    RESTARTS   AGE
-example-runner 2/2     Running   0          1m
+NAME                           READY   STATUS    RESTARTS   AGE
+example-runnerdeploy2475ht2qbr 2/2     Running   0          1m
 ```
 
-The runner you created has been registered to your repository.
-
-<img width="756" alt="Actions tab in your repository settings" src="https://user-images.githubusercontent.com/230145/73618667-8cbf9700-466c-11ea-80b6-c67e6d3f70e7.png">
+The runner you created has been registered directly to the defined repository, you should be able to see it in the settings of the repository.
 
 Now you can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
 
@@ -324,13 +324,15 @@ Now you can use your self-hosted runner. See the [official documentation](https:
 To add the runner to an organization, you only need to replace the `repository` field with `organization`, so the runner will register itself to the organization.
 
 ```yaml
-# runner.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
-kind: Runner
+kind: RunnerDeployment
 metadata:
-  name: example-org-runner
+  name: example-runnerdeploy
 spec:
-  organization: your-organization-name
+  replicas: 1
+  template:
+    spec:
+      organization: your-organization-name
 ```
 
 Now you can see the runner on the organization level (if you have organization owner permissions).
@@ -340,24 +342,22 @@ Now you can see the runner on the organization level (if you have organization o
 To add the runner to an enterprise, you only need to replace the `repository` field with `enterprise`, so the runner will register itself to the enterprise.
 
 ```yaml
-# runner.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
-kind: Runner
+kind: RunnerDeployment
 metadata:
-  name: example-enterprise-runner
+  name: example-runnerdeploy
 spec:
-  enterprise: your-enterprise-name
+  replicas: 1
+  template:
+    spec:
+      enterprise: your-enterprise-name
 ```
 
 Now you can see the runner on the enterprise level (if you have enterprise access permissions).
 
 ### RunnerDeployments
 
-You can manage sets of runners instead of individually through the `RunnerDeployment` kind and its `replicas:` attribute. This kind is required for many of the advanced features.
-
-There are `RunnerReplicaSet` and `RunnerDeployment` kinds that corresponds to the `ReplicaSet` and `Deployment` kinds but for the `Runner` kind.
-
-You typically only need `RunnerDeployment` rather than `RunnerReplicaSet` as the former is for managing the latter.
+In our previous examples we were deploying a single runner via the `RunnerDeployment` kind, the amount of runners deployed can be statically set via the `replicas:` field, we can increase this value to deploy additioanl sets of runners instead:
 
 ```yaml
 # runnerdeployment.yaml
@@ -366,11 +366,11 @@ kind: RunnerDeployment
 metadata:
   name: example-runnerdeploy
 spec:
+  # This will deploy 2 runners now
   replicas: 2
   template:
     spec:
       repository: mumoshu/actions-runner-controller-ci
-      env: []
 ```
 
 Apply the manifest file to your cluster:
@@ -389,15 +389,13 @@ example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
 example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 ```
 
-  ### RunnerSets
+### RunnerSets
 
 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
 
 _Ensure you see the limitations before using this kind!!!!!_
 
-For scenarios where you require the advantages of a `StatefulSet`, for example persistent storage, ARC implements a runner based on Kubernetes' `StatefulSets`, the `RunnerSet`.
-
-A basic `RunnerSet` would look like this:
+We can also deploy sets of RunnerSets the same way, a basic `RunnerSet` would look like this:
 
 ```yaml
 apiVersion: actions.summerwind.dev/v1alpha1
@@ -405,8 +403,7 @@ kind: RunnerSet
 metadata:
   name: example
 spec:
-  ephemeral: false
-  replicas: 2
+  replicas: 1
   repository: mumoshu/actions-runner-controller-ci
   # Other mandatory fields from StatefulSet
   selector:
@@ -437,8 +434,7 @@ kind: RunnerSet
 metadata:
   name: example
 spec:
-  ephemeral: false
-  replicas: 2
+  replicas: 1
   repository: mumoshu/actions-runner-controller-ci
   dockerdWithinRunnerContainer: true
   template:
@@ -1143,9 +1139,13 @@ The earlier entry is prioritized higher than later entries. So you usually defin
 
 A common use case for this may be to have 1 override to scale to 0 during the week outside of core business hours and another override to scale to 0 during all hours of the weekend.
 
-### Runner with DinD
+### Alternative Runners
 
-When using the default runner, the runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
+ARC also offers a few altenrative runner options
+
+#### Runner with DinD
+
+When using the default runner, the runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). ARC maintains an alternative all in one runner image with docker running in the same container as the runner. This may be prefered from a resource or complexity perspective or to be compliant with a `LimitRange` namespace configuration.
 
 ```yaml
 # dindrunnerdeployment.yaml
@@ -1163,9 +1163,7 @@ spec:
       env: []
 ```
 
-This also helps with resources, as you don't need to give resources separately to docker and runner.
-
-### Runner with K8s Jobs
+#### Runner with K8s Jobs
 
 When using the default runner, jobs that use a container will run in docker. This necessitates privileged mode, either on the runner pod or the sidecar container
 
@@ -1386,7 +1384,7 @@ spec:
         - name: tmp
           emptyDir:
             medium: Memory
-      emphemeral: true # recommended to not leak data between builds.
+      ephemeral: true # recommended to not leak data between builds.
 ```
 
 #### NVME SSD
@@ -1394,7 +1392,7 @@ spec:
 In this example we provide NVME backed storage for the workdir, docker sidecar and /tmp within the runner.
 Here we use a working example on GKE, which will provide the NVME disk at /mnt/disks/ssd0.  We will be placing the respective volumes in subdirs here and in order to be able to run multiple runners we will use the pod name as a prefix for subdirectories. Also the disk will fill up over time and disk space will not be freed until the node is removed.
 
-**Beware** that running these persistent backend volumes **leave data behind** between 2 different jobs on the workdir and `/tmp` with `emphemeral: false`.
+**Beware** that running these persistent backend volumes **leave data behind** between 2 different jobs on the workdir and `/tmp` with `ephemeral: false`.
 
 ```yaml
 kind: RunnerDeployment
@@ -1435,7 +1433,7 @@ spec:
       - hostPath:
           path: /mnt/disks/ssd0
         name: tmp
-    emphemeral: true # VERY important. otherwise data inside the workdir and /tmp is not cleared between builds
+    ephemeral: true # VERY important. otherwise data inside the workdir and /tmp is not cleared between builds
 ```
 
 #### Docker image layers caching
@@ -1464,7 +1462,7 @@ spec:
         volumeMounts:
         - name: var-lib-docker
           mountPath: /var/lib/docker
-  volumeClaimtemplates:
+  volumeClaimTemplates:
   - metadata:
       name: var-lib-docker
     spec:
@@ -1570,7 +1568,6 @@ jobs:
 When you have multiple kinds of self-hosted runners, you can distinguish between them using labels. In order to do so, you can specify one or more labels in your `Runner` or `RunnerDeployment` spec.
 
 ```yaml
-# runnerdeployment.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
@@ -1592,7 +1589,10 @@ jobs:
     runs-on: custom-runner
 ```
 
-Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
+When using labels there are a few things to be aware of:
+
+1. `self-hosted` is implict with every runner as this is an automatic label GitHub apply to any self-hosted runner. As a result ARC can treat all runners as having this label without having it explicitly defined in a runner's manifest. You do not need to explicitly define this label in your runner manifests (you can if you want though).
+2. In addition to the `self-hosted` label, GitHub also applies a few other [default](https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow#using-default-labels-to-route-jobs) labels to any self-hosted runner. The other default labels relate to the architecture of the runner and so can't be implicitly applied by ARC as ARC doesn't know if the runner is `linux` or `windows`, `x64` or `ARM64` etc. If you wish to use these labels in your workflows and have ARC scale runners accurately you must also add them to your runner manifests.
 
 ### Runner Groups
 
@@ -1601,7 +1601,6 @@ Runner groups can be used to limit which repositories are able to use the GitHub
 To add the runner to the group `NewGroup`, specify the group in your `Runner` or `RunnerDeployment` spec.
 
 ```yaml
-# runnerdeployment.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
 metadata:
@@ -1744,6 +1743,64 @@ $ helm --upgrade install actions-runner-controller/actions-runner-controller \
   admissionWebHooks.caBundle=${CA_BUNDLE}
 ```
 
+### Multitenancy
+
+> This feature requires controller version => [v0.26.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.26.0)
+
+In a large enterprise, there might be many GitHub organizations that requires self-hosted runners. Previously, the only way to provide ARC-managed self-hosted runners in such environment was [Deploying Multiple Controllers](#deploying-multiple-controllers), which incurs overhead due to it requires one ARC installation per GitHub organization.
+
+With multitenancy, you can let ARC manage self-hosted runners across organizations. It's enabled by default and the only thing you need to start using it is to set the `spec.githubAPICredentialsFrom.secretRef.name` fields for the following resources:
+
+- `HorizontalRunnerAutoscaler`
+- `RunnerSet`
+
+Or `spec.template.spec.githubAPICredentialsFrom.secretRef.name` field for the following resource:
+
+- `RunnerDeployment`
+
+> Although not explained above, `spec.githubAPICredentialsFrom` fields do exist in `Runner` and `RunnerReplicaSet`. A comparable pod annotation exists for the runner pod, too.
+> However, note that `Runner`, `RunnerReplicaSet` and runner pods are implementation details and are managed by `RunnerDeployment` and ARC.
+> Usually you don't need to manually set the fields for those resources.
+
+`githubAPICredentialsFrom.secretRef.name` should refer to the name of the Kubernetes secret that contains either PAT or GitHub App credentials that is used for GitHub API calls for the said resource.
+
+Usually, you should have a set of GitHub App credentials per a GitHub organization and you would have a RunnerDeployment and a HorizontalRunnerAutoscaler per an organization runner group. So, you might end up having the following resources for each organization:
+
+- 1 Kuernetes secret that contains GitHub App credentials
+- 1 RunnerDeployment/RunnerSet and 1 HorizontalRunnerAutoscaler per Runner Group
+
+And the RunnerDeployment/RunnerSet and HorizontalRunnerAutoscaler should have the same value for `spec.githubAPICredentialsFrom.secretRef.name`, which refers to the name of the Kubernetes secret.
+
+```yaml
+kind: Secret
+data:
+  github_app_id: ...
+  github_app_installation_id: ...
+  github_app_private_key: ...
+---
+kind: RunnerDeployment
+metadata:
+  namespace: org1-runners
+spec:
+  githubAPICredentialsFrom:
+    secretRef:
+      name: org1-github-app
+---
+kind: HorizontalRunnerAutoscaler
+metadata:
+  namespace: org1-runners
+spec:
+  githubAPICredentialsFrom:
+    secretRef:
+      name: org1-github-app
+```
+
+> Do note that, as shown in the above example, you usually set the same secret name to `githubAPICredentialsFrom.secretRef.name` fields of both `RunnerDeployment` and `HorizontalRunnerAutoscaler`, so that GitHub API calls for the same set of runners shares the specified credentials, regardless of
+when and which varying ARC component(`horizontalrunnerautoscaler-controller`, `runnerdeployment-controller`, `runnerreplicaset-controller`, `runner-controller` or `runnerpod-controller`) makes specific API calls.
+> Just don't be surprised you have to repeat `githubAPICredentialsFrom.secretRef.name` settings among two resources!
+
+Please refer to [Deploying Using GitHub App Authentication](#deploying-using-github-app-authentication) for how you could create the Kubernetes secret containing GitHub App credentials.
+
 # Troubleshooting
 
 See [troubleshooting guide](TROUBLESHOOTING.md) for solutions to various problems people have run into consistently.

TROUBLESHOOTING.md

@@ -4,6 +4,7 @@
 * [Installation](#installation)
   * [InternalError when calling webhook: context deadline exceeded](#internalerror-when-calling-webhook-context-deadline-exceeded)
   * [Invalid header field value](#invalid-header-field-value)
+  * [Helm chart install failure: certificate signed by unknown authority](#helm-chart-install-failure-certificate-signed-by-unknown-authority)
 * [Operations](#operations)
   * [Stuck runner kind or backing pod](#stuck-runner-kind-or-backing-pod)
   * [Delay in jobs being allocated to runners](#delay-in-jobs-being-allocated-to-runners)
@@ -105,6 +106,37 @@ Your base64'ed PAT token has a new line at the end, it needs to be created witho
 * `echo -n $TOKEN | base64`
 * Create the secret as described in the docs using the shell and documented flags
 
+### Helm chart install failure: certificate signed by unknown authority
+
+**Problem**
+
+```
+Error: UPGRADE FAILED: failed to create resource: Internal error occurred: failed calling webhook "webhook.cert-manager.io": failed to call webhook: Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority
+```
+
+Apparently, it's failing while `helm` is creating one of resources defined in the ARC chart and the cause was that cert-manager's webhook is not working correctly, due to the missing or the invalid CA certficate.
+
+You'd try to tail logs from the `cert-manager-cainjector` and see it's failing with an error like:
+
+```
+$ kubectl -n cert-manager logs cert-manager-cainjector-7cdbb9c945-g6bt4
+I0703 03:31:55.159339       1 start.go:91] "starting" version="v1.1.1" revision="3ac7418070e22c87fae4b22603a6b952f797ae96"
+I0703 03:31:55.615061       1 leaderelection.go:243] attempting to acquire leader lease  kube-system/cert-manager-cainjector-leader-election...
+I0703 03:32:10.738039       1 leaderelection.go:253] successfully acquired lease kube-system/cert-manager-cainjector-leader-election
+I0703 03:32:10.739941       1 recorder.go:52] cert-manager/controller-runtime/manager/events "msg"="Normal"  "message"="cert-manager-cainjector-7cdbb9c945-g6bt4_88e4bc70-eded-4343-a6fb-0ddd6434eb55 became leader" "object"={"kind":"ConfigMap","namespace":"kube-system","name":"cert-manager-cainjector-leader-election","uid":"942a021e-364c-461a-978c-f54a95723cdc","apiVersion":"v1","resourceVersion":"1576"} "reason"="LeaderElection"
+E0703 03:32:11.192128       1 start.go:119] cert-manager/ca-injector "msg"="manager goroutine exited" "error"=null
+I0703 03:32:12.339197       1 request.go:645] Throttling request took 1.047437675s, request: GET:https://10.96.0.1:443/apis/storage.k8s.io/v1beta1?timeout=32s
+E0703 03:32:13.143790       1 start.go:151] cert-manager/ca-injector "msg"="Error registering certificate based controllers. Retrying after 5 seconds." "error"="no matches for kind \"MutatingWebhookConfiguration\" in version \"admissionregistration.k8s.io/v1beta1\""
+Error: error registering secret controller: no matches for kind "MutatingWebhookConfiguration" in version "admissionregistration.k8s.io/v1beta1"
+```
+
+**Solution**
+
+Your cluster is based on a new enough Kubernetes of version 1.22 or greater which does not support the legacy `admissionregistration.k8s.io/v1beta1` API anymore, and your `cert-manager` is not up-to-date hence it's still trying to use the leagcy Kubernetes API.
+
+In many cases, it's not an option to downgrade Kubernetes. So, just upgrade `cert-manager` to a more recent version that does have have the support for the specific Kubernetes version you're using.
+
+See https://cert-manager.io/docs/installation/supported-releases/ for the list of available cert-manager versions.
 
 ## Operations
 

acceptance/argotunnel.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# See https://developers.cloudflare.com/cloudflare-one/tutorials/many-cfd-one-tunnel/
+
+kubectl create ns tunnel || :
+
+kubectl -n tunnel delete secret tunnel-credentials || :
+
+kubectl -n tunnel create secret generic tunnel-credentials \
+  --from-file=credentials.json=$HOME/.cloudflared/${TUNNEL_ID}.json || :
+
+cat <<MANIFEST | kubectl -n tunnel ${OP} -f -
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared
+spec:
+  selector:
+    matchLabels:
+      app: cloudflared
+  replicas: 2 # You could also consider elastic scaling for this deployment
+  template:
+    metadata:
+      labels:
+        app: cloudflared
+    spec:
+      containers:
+      - name: cloudflared
+        image: cloudflare/cloudflared:latest
+        args:
+        - tunnel
+        # Points cloudflared to the config file, which configures what
+        # cloudflared will actually do. This file is created by a ConfigMap
+        # below.
+        - --config
+        - /etc/cloudflared/config/config.yaml
+        - run
+        livenessProbe:
+          httpGet:
+            # Cloudflared has a /ready endpoint which returns 200 if and only if
+            # it has an active connection to the edge.
+            path: /ready
+            port: 2000
+          failureThreshold: 1
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        volumeMounts:
+        - name: config
+          mountPath: /etc/cloudflared/config
+          readOnly: true
+        # Each tunnel has an associated "credentials file" which authorizes machines
+        # to run the tunnel. cloudflared will read this file from its local filesystem,
+        # and it'll be stored in a k8s secret.
+        - name: creds
+          mountPath: /etc/cloudflared/creds
+          readOnly: true
+      volumes:
+      - name: creds
+        secret:
+          secretName: tunnel-credentials
+      # Create a config.yaml file from the ConfigMap below.
+      - name: config
+        configMap:
+          name: cloudflared
+          items:
+          - key: config.yaml
+            path: config.yaml
+---
+# This ConfigMap is just a way to define the cloudflared config.yaml file in k8s.
+# It's useful to define it in k8s, rather than as a stand-alone .yaml file, because
+# this lets you use various k8s templating solutions (e.g. Helm charts) to
+# parameterize your config, instead of just using string literals.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared
+data:
+  config.yaml: |
+    # Name of the tunnel you want to run
+    tunnel: ${TUNNEL_NAME}
+    credentials-file: /etc/cloudflared/creds/credentials.json
+    # Serves the metrics server under /metrics and the readiness server under /ready
+    metrics: 0.0.0.0:2000
+    # Autoupdates applied in a k8s pod will be lost when the pod is removed or restarted, so
+    # autoupdate doesn't make sense in Kubernetes. However, outside of Kubernetes, we strongly
+    # recommend using autoupdate.
+    no-autoupdate: true
+    ingress:
+    # The first rule proxies traffic to the httpbin sample Service defined in app.yaml
+    - hostname: ${TUNNEL_HOSTNAME}
+      service: http://actions-runner-controller-github-webhook-server.actions-runner-system:80
+    # This rule matches any traffic which didn't match a previous rule, and responds with HTTP 404.
+    - service: http_status:404
+MANIFEST
+
+kubectl -n tunnel delete po -l app=cloudflared || :

acceptance/deploy.sh

@@ -51,6 +51,10 @@ if [ "${tool}" == "helm" ]; then
     --set image.tag=${VERSION} \
     --set podAnnotations.test-id=${TEST_ID} \
     --set githubWebhookServer.podAnnotations.test-id=${TEST_ID} \
+    --set imagePullSecrets[0].name=${IMAGE_PULL_SECRET} \
+    --set image.actionsRunnerImagePullSecrets[0].name=${IMAGE_PULL_SECRET} \
+    --set githubWebhookServer.imagePullSecrets[0].name=${IMAGE_PULL_SECRET} \
+    --set image.imagePullPolicy=${IMAGE_PULL_POLICY} \
     -f ${VALUES_FILE}
   set +v
   # To prevent `CustomResourceDefinition.apiextensions.k8s.io "runners.actions.summerwind.dev" is invalid: metadata.annotations: Too long: must have at most 262144 bytes`

acceptance/testdata/kubernetes_container_mode.envsubst.yaml

@@ -0,0 +1,82 @@
+# USAGE:
+#   cat acceptance/testdata/kubernetes_container_mode.envsubst.yaml  | NAMESPACE=default envsubst  | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: k8s-mode-runner
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["get", "create"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get", "list", "watch",]
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["get", "list", "create", "delete"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: runner-status-updater
+rules:
+- apiGroups: ["actions.summerwind.dev"]
+  resources: ["runners/status"]
+  verbs: ["get", "update", "patch"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: runner
+  namespace: ${NAMESPACE}
+---
+# To verify it's working, try:
+#   kubectl auth can-i --as system:serviceaccount:default:runner get pod
+# If incomplete, workflows and jobs would fail with an error message like:
+#   Error: Error: The Service account needs the following permissions [{"group":"","verbs":["get","list","create","delete"],"resource":"pods","subresource":""},{"group":"","verbs":["get","create"],"resource":"pods","subresource":"exec"},{"group":"","verbs":["get","list","watch"],"resource":"pods","subresource":"log"},{"group":"batch","verbs":["get","list","create","delete"],"resource":"jobs","subresource":""},{"group":"","verbs":["create","delete","get","list"],"resource":"secrets","subresource":""}] on the pod resource in the 'default' namespace. Please contact your self hosted runner administrator.
+#   Error: Process completed with exit code 1.
+apiVersion: rbac.authorization.k8s.io/v1
+# This role binding allows "jane" to read pods in the "default" namespace.
+# You need to already have a Role named "pod-reader" in that namespace.
+kind: RoleBinding
+metadata:
+  name: runner-k8s-mode-runner
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: runner
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: k8s-mode-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: runner-runner-stat-supdater
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: runner
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: ClusterRole
+  name: runner-status-updater
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: org-runnerdeploy-runner-work-dir
+  labels:
+    content: org-runnerdeploy-runner-work-dir
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer

acceptance/testdata/runnerdeploy.envsubst.yaml

@@ -43,6 +43,17 @@ spec:
       # Non-standard working directory
       #
       # workDir: "/"
+
+      # # Uncomment the below to enable the kubernetes container mode
+      # # See https://github.com/actions-runner-controller/actions-runner-controller#runner-with-k8s-jobs
+      containerMode: kubernetes
+      workVolumeClaimTemplate:
+        accessModes:
+        - ReadWriteOnce
+        storageClassName: "${NAME}-runner-work-dir"
+        resources:
+          requests:
+            storage: 10Gi
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: HorizontalRunnerAutoscaler

acceptance/values.yaml

@@ -1,6 +1,18 @@
 # Set actions-runner-controller settings for testing
 logLevel: "-4"
+imagePullSecrets:
+- name:
+image:
+  actionsRunnerImagePullSecrets:
+  - name:
+runner:
+  statusUpdateHook:
+    enabled: true
+rbac:
+  allowGrantingKubernetesContainerModePermissions: true
 githubWebhookServer:
+  imagePullSecrets:
+  - name:
   logLevel: "-4"
   enabled: true
   labels: {}

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -60,6 +60,9 @@ type HorizontalRunnerAutoscalerSpec struct {
 	// The earlier a scheduled override is, the higher it is prioritized.
 	// +optional
 	ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"`
+
+	// +optional
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
 }
 
 type ScaleUpTrigger struct {

api/v1alpha1/runner_types.go

@@ -76,6 +76,16 @@ type RunnerConfig struct {
 
 	// +optional
 	ContainerMode string `json:"containerMode,omitempty"`
+
+	GitHubAPICredentialsFrom *GitHubAPICredentialsFrom `json:"githubAPICredentialsFrom,omitempty"`
+}
+
+type GitHubAPICredentialsFrom struct {
+	SecretRef SecretReference `json:"secretRef,omitempty"`
+}
+
+type SecretReference struct {
+	Name string `json:"name"`
 }
 
 // RunnerPodSpec defines the desired pod spec fields of the runner pod
@@ -183,11 +193,6 @@ func (rs *RunnerSpec) Validate(rootPath *field.Path) field.ErrorList {
 		errList = append(errList, field.Invalid(rootPath.Child("workVolumeClaimTemplate"), rs.WorkVolumeClaimTemplate, err.Error()))
 	}
 
-	err = rs.validateIsServiceAccountNameSet()
-	if err != nil {
-		errList = append(errList, field.Invalid(rootPath.Child("serviceAccountName"), rs.ServiceAccountName, err.Error()))
-	}
-
 	return errList
 }
 
@@ -226,17 +231,6 @@ func (rs *RunnerSpec) validateWorkVolumeClaimTemplate() error {
 	return rs.WorkVolumeClaimTemplate.validate()
 }
 
-func (rs *RunnerSpec) validateIsServiceAccountNameSet() error {
-	if rs.ContainerMode != "kubernetes" {
-		return nil
-	}
-
-	if rs.ServiceAccountName == "" {
-		return errors.New("service account name is required if container mode is kubernetes")
-	}
-	return nil
-}
-
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
 	// Turns true only if the runner pod is ready.
@@ -315,8 +309,10 @@ func (w *WorkVolumeClaimTemplate) V1VolumeMount(mountPath string) corev1.VolumeM
 // +kubebuilder:printcolumn:JSONPath=".spec.enterprise",name=Enterprise,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.organization",name=Organization,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.repository",name=Repository,type=string
+// +kubebuilder:printcolumn:JSONPath=".spec.group",name=Group,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.labels",name=Labels,type=string
 // +kubebuilder:printcolumn:JSONPath=".status.phase",name=Status,type=string
+// +kubebuilder:printcolumn:JSONPath=".status.message",name=Message,type=string
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
 // Runner is the Schema for the runners API

api/v1alpha1/zz_generated.deepcopy.go

@@ -90,6 +90,22 @@ func (in *CheckRunSpec) DeepCopy() *CheckRunSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GitHubAPICredentialsFrom) DeepCopyInto(out *GitHubAPICredentialsFrom) {
+	*out = *in
+	out.SecretRef = in.SecretRef
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubAPICredentialsFrom.
+func (in *GitHubAPICredentialsFrom) DeepCopy() *GitHubAPICredentialsFrom {
+	if in == nil {
+		return nil
+	}
+	out := new(GitHubAPICredentialsFrom)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTriggerSpec) {
 	*out = *in
@@ -231,6 +247,11 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
@@ -425,6 +446,11 @@ func (in *RunnerConfig) DeepCopyInto(out *RunnerConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.GitHubAPICredentialsFrom != nil {
+		in, out := &in.GitHubAPICredentialsFrom, &out.GitHubAPICredentialsFrom
+		*out = new(GitHubAPICredentialsFrom)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerConfig.
@@ -1136,6 +1162,21 @@ func (in *ScheduledOverride) DeepCopy() *ScheduledOverride {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretReference)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *WorkVolumeClaimTemplate) DeepCopyInto(out *WorkVolumeClaimTemplate) {
 	*out = *in

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.20.0
+version: 0.20.2
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.25.0
+appVersion: 0.25.2
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/README.md

@@ -73,11 +73,11 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `scope.watchNamespace`                                   | Tells the controller and the github webhook server which namespace to watch if `scope.singleNamespace` is true             | `Release.Namespace` (the default namespace of the helm chart).       |
 | `scope.singleNamespace`                                  | Limit the controller to watch a single namespace                                                                           | false                                                                |
 | `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                   | true                                                                 |
+| `runner.statusUpdateHook.enabled`                        | Use custom RBAC for runners (role, role binding and service account), this will enable reporting runner statuses           | false                                                                |
 | `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                  |                                                                      |
 | `githubWebhookServer.logLevel`                           | Set the log level of the githubWebhookServer container                                                                     |                                                                      |
 | `githubWebhookServer.replicaCount`                       | Set the number of webhook server pods                                                                                      | 1                                                                    |
 | `githubWebhookServer.useRunnerGroupsVisibility`          | Enable supporting runner groups with custom visibility. This will incur in extra API calls and may blow up your budget. Currently, you also need to set `githubWebhookServer.secret.enabled` to enable this feature. | false                                                                |
-| `githubWebhookServer.syncPeriod`                         | Set the period in which the controller reconciles the resources                                                            | 10m                                                                  |
 | `githubWebhookServer.enabled`                            | Deploy the webhook server pod                                                                                              | false                                                                |
 | `githubWebhookServer.secret.enabled`                      | Passes the webhook hook secret to the github-webhook-server                                                                             | false                                                                |
 | `githubWebhookServer.secret.create`                      | Deploy the webhook hook secret                                                                                             | false                                                                |

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml

@@ -2415,6 +2415,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml

@@ -2412,6 +2412,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml

@@ -24,12 +24,18 @@ spec:
         - jsonPath: .spec.repository
           name: Repository
           type: string
+        - jsonPath: .spec.group
+          name: Group
+          type: string
         - jsonPath: .spec.labels
           name: Labels
           type: string
         - jsonPath: .status.phase
           name: Status
           type: string
+        - jsonPath: .status.message
+          name: Message
+          type: string
         - jsonPath: .metadata.creationTimestamp
           name: Age
           type: date
@@ -2353,6 +2359,16 @@ spec:
                       - name
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 hostAliases:

charts/actions-runner-controller/crds/actions.summerwind.dev_runnersets.yaml

@@ -67,6 +67,16 @@ spec:
                   type: string
                 ephemeral:
                   type: boolean
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 image:

charts/actions-runner-controller/templates/deployment.yaml

@@ -58,15 +58,15 @@ spec:
         {{- if .Values.scope.singleNamespace }}
         - "--watch-namespace={{ default .Release.Namespace .Values.scope.watchNamespace }}"
         {{- end }}
-        {{- if .Values.githubAPICacheDuration }}
-        - "--github-api-cache-duration={{ .Values.githubAPICacheDuration }}"
-        {{- end }}
         {{- if .Values.logLevel }}
         - "--log-level={{ .Values.logLevel }}"
         {{- end }}
         {{- if .Values.runnerGithubURL  }}
         - "--runner-github-url={{ .Values.runnerGithubURL }}"
         {{- end }}
+        {{- if .Values.runner.statusUpdateHook.enabled }}
+        - "--runner-status-update-hook"
+        {{- end }}
         command:
         - "/manager"
         env:
@@ -118,10 +118,14 @@ spec:
               name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         {{- end }}
+        {{- if kindIs "slice" .Values.env }}
+        {{- toYaml .Values.env | nindent 8 }}
+        {{- else }}
         {{- range $key, $val := .Values.env }}
         - name: {{ $key }}
           value: {{ $val | quote }}
         {{- end }}
+        {{- end }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
         name: manager
         imagePullPolicy: {{ .Values.image.pullPolicy }}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -39,7 +39,6 @@ spec:
         {{- $metricsHost := .Values.metrics.proxy.enabled | ternary "127.0.0.1" "0.0.0.0" }}
         {{- $metricsPort := .Values.metrics.proxy.enabled | ternary "8080" .Values.metrics.port }}
         - "--metrics-addr={{ $metricsHost }}:{{ $metricsPort }}"
-        - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}"
         {{- if .Values.githubWebhookServer.logLevel }}
         - "--log-level={{ .Values.githubWebhookServer.logLevel }}"
         {{- end }}

charts/actions-runner-controller/templates/manager_role.yaml

@@ -258,3 +258,64 @@ rules:
   - get
   - list
   - watch
+{{- if .Values.runner.statusUpdateHook.enabled }}
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get
+{{- end }}
+{{- if .Values.rbac.allowGrantingKubernetesContainerModePermissions }}
+{{/* These permissions are required by ARC to create RBAC resources for the runner pod to use the kubernetes container mode. */}}
+{{/* See https://github.com/actions-runner-controller/actions-runner-controller/pull/1268/files#r917331632 */}}
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/log
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "batch"
+  resources:
+  - jobs
+  verbs:
+  - get
+  - list
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+{{- end }}

charts/actions-runner-controller/values.yaml

@@ -15,12 +15,6 @@ enableLeaderElection: true
 # Must be unique if more than one controller installed onto the same namespace.
 #leaderElectionId: "actions-runner-controller"
 
-# DEPRECATED: This has been removed as unnecessary in #1192
-# The controller tries its best not to repeat the duplicate GitHub API call
-# within this duration.
-# Defaults to syncPeriod - 10s.
-#githubAPICacheDuration: 30s
-
 # The URL of your GitHub Enterprise server, if you're using one.
 #githubEnterpriseServerURL: https://github.example.com
 
@@ -67,6 +61,18 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+runner:
+  statusUpdateHook:
+    enabled: false
+
+rbac:
+  {}
+  # # This allows ARC to dynamically create a ServiceAccount and a Role for each Runner pod that uses "kubernetes" container mode,
+  # # by extending ARC's manager role to have the same permissions required by the pod runs the runner agent in "kubernetes" container mode.
+  # # Without this, Kubernetes blocks ARC to create the role to prevent a priviledge escalation.
+  # # See https://github.com/actions-runner-controller/actions-runner-controller/pull/1268/files#r917327010
+  # allowGrantingKubernetesContainerModePermissions: true
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true
@@ -143,10 +149,20 @@ priorityClassName: ""
 
 env:
   {}
+  # specify additional environment variables for the controller pod.
+  # It's possible to specify either key vale pairs e.g.:
   # http_proxy: "proxy.com:8080"
   # https_proxy: "proxy.com:8080"
   # no_proxy: ""
 
+  # or a list of complete environment variable definitions e.g.:
+  # - name: GITHUB_APP_INSTALLATION_ID
+  #   valueFrom:
+  #     secretKeyRef:
+  #       key: some_key_in_the_secret
+  #       name: some-secret-name
+  #       optional: true
+
 ## specify additional volumes to mount in the manager container, this can be used
 ## to specify additional storage of material or to inject files from ConfigMaps
 ## into the running container
@@ -175,7 +191,6 @@ admissionWebHooks:
 githubWebhookServer:
   enabled: false
   replicaCount: 1
-  syncPeriod: 10m
   useRunnerGroupsVisibility: false
   secret:
     enabled: false

cmd/githubwebhookserver/main.go

@@ -69,10 +69,8 @@ func main() {
 
 		watchNamespace string
 
-		enableLeaderElection bool
-		syncPeriod           time.Duration
-		logLevel             string
-		queueLimit           int
+		logLevel   string
+		queueLimit int
 
 		ghClient *github.Client
 	)
@@ -89,9 +87,6 @@ func main() {
 	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&watchNamespace, "watch-namespace", "", "The namespace to watch for HorizontalRunnerAutoscaler's to scale on Webhook. Set to empty for letting it watch for all namespaces.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
-		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
-	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
 	flag.IntVar(&queueLimit, "queue-limit", controllers.DefaultQueueLimit, `The maximum length of the scale operation queue. The scale opration is enqueued per every matching webhook event, and the server returns a 500 HTTP status when the queue was already full on enqueue attempt.`)
 	flag.StringVar(&webhookSecretToken, "github-webhook-secret-token", "", "The personal access token of GitHub.")
@@ -144,10 +139,10 @@ func main() {
 		setupLog.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls")
 	}
 
+	syncPeriod := 10 * time.Minute
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		SyncPeriod:         &syncPeriod,
-		LeaderElection:     enableLeaderElection,
 		Namespace:          watchNamespace,
 		MetricsBindAddress: metricsAddr,
 		Port:               9443,

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -61,6 +61,16 @@ spec:
                         type: integer
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 maxReplicas:
                   description: MaxReplicas is the maximum number of replicas the deployment is allowed to scale
                   type: integer

config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml

@@ -2415,6 +2415,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml

@@ -2412,6 +2412,16 @@ spec:
                               - name
                             type: object
                           type: array
+                        githubAPICredentialsFrom:
+                          properties:
+                            secretRef:
+                              properties:
+                                name:
+                                  type: string
+                              required:
+                                - name
+                              type: object
+                          type: object
                         group:
                           type: string
                         hostAliases:

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -24,12 +24,18 @@ spec:
         - jsonPath: .spec.repository
           name: Repository
           type: string
+        - jsonPath: .spec.group
+          name: Group
+          type: string
         - jsonPath: .spec.labels
           name: Labels
           type: string
         - jsonPath: .status.phase
           name: Status
           type: string
+        - jsonPath: .status.message
+          name: Message
+          type: string
         - jsonPath: .metadata.creationTimestamp
           name: Age
           type: date
@@ -2353,6 +2359,16 @@ spec:
                       - name
                     type: object
                   type: array
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 hostAliases:

config/crd/bases/actions.summerwind.dev_runnersets.yaml

@@ -67,6 +67,16 @@ spec:
                   type: string
                 ephemeral:
                   type: boolean
+                githubAPICredentialsFrom:
+                  properties:
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                        - name
+                      type: object
+                  type: object
                 group:
                   type: string
                 image:

config/rbac/role.yaml

@@ -258,3 +258,27 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - roles
+  verbs:
+  - create
+  - delete
+  - get

controllers/autoscaling.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/google/go-github/v45/github"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -21,7 +22,7 @@ const (
 	defaultScaleDownFactor    = 0.7
 )
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	if hra.Spec.MinReplicas == nil {
 		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
 	} else if hra.Spec.MaxReplicas == nil {
@@ -48,9 +49,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 
 	switch primaryMetricType {
 	case v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns:
-		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &primaryMetric)
+		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &primaryMetric)
 	case v1alpha1.AutoscalingMetricTypePercentageRunnersBusy:
-		suggested, err = r.suggestReplicasByPercentageRunnersBusy(st, hra, primaryMetric)
+		suggested, err = r.suggestReplicasByPercentageRunnersBusy(ghc, st, hra, primaryMetric)
 	default:
 		return nil, fmt.Errorf("validating autoscaling metrics: unsupported metric type %q", primaryMetric)
 	}
@@ -83,11 +84,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 		)
 	}
 
-	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, &fallbackMetric)
+	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &fallbackMetric)
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
-
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
 	var repos [][]string
 	repoID := st.repo
 	if repoID == "" {
@@ -126,7 +126,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 		opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}}
 		var allJobs []*github.WorkflowJob
 		for {
-			jobs, resp, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
+			jobs, resp, err := ghc.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
 			if err != nil {
 				r.Log.Error(err, "Error listing workflow jobs")
 				return //err
@@ -184,7 +184,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
-		workflowRuns, err := r.GitHubClient.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
+		workflowRuns, err := ghc.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
 		if err != nil {
 			return nil, err
 		}
@@ -226,7 +226,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 	return &necessaryReplicas, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
 	ctx := context.Background()
 	scaleUpThreshold := defaultScaleUpThreshold
 	scaleDownThreshold := defaultScaleDownThreshold
@@ -295,7 +295,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunner
 	)
 
 	// ListRunners will return all runners managed by GitHub - not restricted to ns
-	runners, err := r.GitHubClient.ListRunners(
+	runners, err := ghc.ListRunners(
 		ctx,
 		enterprise,
 		organization,

controllers/autoscaling_test.go

@@ -330,7 +330,6 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
-				GitHubClient:          client,
 				Scheme:                scheme,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
@@ -379,7 +378,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)
@@ -720,7 +719,6 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			h := &HorizontalRunnerAutoscalerReconciler{
 				Log:                   log,
 				Scheme:                scheme,
-				GitHubClient:          client,
 				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
 
@@ -781,7 +779,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)

controllers/horizontalrunnerautoscaler_controller.go

@@ -24,7 +24,6 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -38,6 +37,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
+	arcgithub "github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 const (
@@ -47,11 +47,10 @@ const (
 // HorizontalRunnerAutoscalerReconciler reconciles a HorizontalRunnerAutoscaler object
 type HorizontalRunnerAutoscalerReconciler struct {
 	client.Client
-	GitHubClient          *github.Client
+	GitHubClient          *MultiGitHubClient
 	Log                   logr.Logger
 	Recorder              record.EventRecorder
 	Scheme                *runtime.Scheme
-	CacheDuration         time.Duration
 	DefaultScaleDownDelay time.Duration
 	Name                  string
 }
@@ -73,6 +72,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(ctx context.Context, re
 	}
 
 	if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForHRA(&hra)
+
 		return ctrl.Result{}, nil
 	}
 
@@ -310,7 +311,12 @@ func (r *HorizontalRunnerAutoscalerReconciler) reconcile(ctx context.Context, re
 		return ctrl.Result{}, err
 	}
 
-	newDesiredReplicas, err := r.computeReplicasWithCache(log, now, st, hra, minReplicas)
+	ghc, err := r.GitHubClient.InitForHRA(context.Background(), &hra)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	newDesiredReplicas, err := r.computeReplicasWithCache(ghc, log, now, st, hra, minReplicas)
 	if err != nil {
 		r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -461,10 +467,10 @@ func (r *HorizontalRunnerAutoscalerReconciler) getMinReplicas(log logr.Logger, n
 	return minReplicas, active, upcoming, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(ghc *arcgithub.Client, log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
 	var suggestedReplicas int
 
-	v, err := r.suggestDesiredReplicas(st, hra)
+	v, err := r.suggestDesiredReplicas(ghc, st, hra)
 	if err != nil {
 		return 0, err
 	}

controllers/integration_test.go

@@ -99,12 +99,14 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 			return fmt.Sprintf("%s%s", ns.Name, name)
 		}
 
+		multiClient := NewMultiGitHubClient(mgr.GetClient(), env.ghClient)
+
 		runnerController := &RunnerReconciler{
 			Client:                      mgr.GetClient(),
 			Scheme:                      scheme.Scheme,
 			Log:                         logf.Log,
 			Recorder:                    mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient:                env.ghClient,
+			GitHubClient:                multiClient,
 			RunnerImage:                 "example/runner:test",
 			DockerImage:                 "example/docker:test",
 			Name:                        controllerName("runner"),
@@ -116,12 +118,11 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 		Expect(err).NotTo(HaveOccurred(), "failed to setup runner controller")
 
 		replicasetController := &RunnerReplicaSetReconciler{
-			Client:       mgr.GetClient(),
-			Scheme:       scheme.Scheme,
-			Log:          logf.Log,
-			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient: env.ghClient,
-			Name:         controllerName("runnerreplicaset"),
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     controllerName("runnerreplicaset"),
 		}
 		err = replicasetController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup runnerreplicaset controller")
@@ -137,13 +138,12 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 		Expect(err).NotTo(HaveOccurred(), "failed to setup runnerdeployment controller")
 
 		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
-			Client:        mgr.GetClient(),
-			Scheme:        scheme.Scheme,
-			Log:           logf.Log,
-			GitHubClient:  env.ghClient,
-			Recorder:      mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
-			CacheDuration: 1 * time.Second,
-			Name:          controllerName("horizontalrunnerautoscaler"),
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			GitHubClient: multiClient,
+			Recorder:     mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+			Name:         controllerName("horizontalrunnerautoscaler"),
 		}
 		err = autoscalerController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler controller")

controllers/multi_githubclient.go

@@ -0,0 +1,389 @@
+package controllers
+
+import (
+	"context"
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"strconv"
+	"sync"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// The api creds scret annotation is added by the runner controller or the runnerset controller according to runner.spec.githubAPICredentialsFrom.secretRef.name,
+	// so that the runner pod controller can share the same GitHub API credentials and the instance of the GitHub API client with the upstream controllers.
+	annotationKeyGitHubAPICredsSecret = annotationKeyPrefix + "github-api-creds-secret"
+)
+
+type runnerOwnerRef struct {
+	// kind is either StatefulSet or Runner, and populated via the owner reference in the runner pod controller or via the reconcilation target's kind in
+	// runnerset and runner controllers.
+	kind     string
+	ns, name string
+}
+
+type secretRef struct {
+	ns, name string
+}
+
+// savedClient is the each cache entry that contains the client for the specific set of credentials,
+// like a PAT or a pair of key and cert.
+// the `hash` is a part of the savedClient not the key because we are going to keep only the client for the latest creds
+// in case the operator updated the k8s secret containing the credentials.
+type savedClient struct {
+	hash string
+
+	// refs is the map of all the objects that references this client, used for reference counting to gc
+	// the client if unneeded.
+	refs map[runnerOwnerRef]struct{}
+
+	*github.Client
+}
+
+type resourceReader interface {
+	Get(context.Context, types.NamespacedName, client.Object) error
+}
+
+type MultiGitHubClient struct {
+	mu sync.Mutex
+
+	client resourceReader
+
+	githubClient *github.Client
+
+	// The saved client is freed once all its dependents disappear, or the contents of the secret changed.
+	// We track dependents via a golang map embedded within the savedClient struct. Each dependent is checked on their respective Kubernetes finalizer,
+	// so that we won't miss any dependent's termination.
+	// The change is the secret is determined using the hash of its contents.
+	clients map[secretRef]savedClient
+}
+
+func NewMultiGitHubClient(client resourceReader, githubClient *github.Client) *MultiGitHubClient {
+	return &MultiGitHubClient{
+		client:       client,
+		githubClient: githubClient,
+		clients:      map[secretRef]savedClient{},
+	}
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerPod(ctx context.Context, pod *corev1.Pod) (*github.Client, error) {
+	// These 3 default values are used only when the user created the pod directly, not via Runner, RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunnerPod(pod)
+	secretName := pod.Annotations[annotationKeyGitHubAPICredsSecret]
+
+	// kind can be any of Pod, Runner, RunnerReplicaSet, RunnerDeployment, or RunnerSet depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, pod.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunner(ctx context.Context, r *v1alpha1.Runner) (*github.Client, error) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	// These 3 default values are used only when the user created the runner resource directly, not via RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunner(r)
+	if ref.ns != r.Namespace {
+		return nil, fmt.Errorf("referencing github api creds secret from owner in another namespace is not supported yet")
+	}
+
+	// kind can be any of Runner, RunnerReplicaSet, or RunnerDeployment depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, r.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerSet(ctx context.Context, rs *v1alpha1.RunnerSet) (*github.Client, error) {
+	ref := refFromRunnerSet(rs)
+
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, rs.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForHRA(ctx context.Context, hra *v1alpha1.HorizontalRunnerAutoscaler) (*github.Client, error) {
+	ref := refFromHorizontalRunnerAutoscaler(hra)
+
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, hra.Namespace, secretName, ref)
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerPod(p *corev1.Pod) {
+	secretName := p.Annotations[annotationKeyGitHubAPICredsSecret]
+	c.derefClient(p.Namespace, secretName, refFromRunnerPod(p))
+}
+
+func (c *MultiGitHubClient) DeinitForRunner(r *v1alpha1.Runner) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(r.Namespace, secretName, refFromRunner(r))
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerSet(rs *v1alpha1.RunnerSet) {
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(rs.Namespace, secretName, refFromRunnerSet(rs))
+}
+
+func (c *MultiGitHubClient) deinitClientForRunnerReplicaSet(rs *v1alpha1.RunnerReplicaSet) {
+	c.derefClient(rs.Namespace, rs.Spec.Template.Spec.GitHubAPICredentialsFrom.SecretRef.Name, refFromRunnerReplicaSet(rs))
+}
+
+func (c *MultiGitHubClient) deinitClientForRunnerDeployment(rd *v1alpha1.RunnerDeployment) {
+	c.derefClient(rd.Namespace, rd.Spec.Template.Spec.GitHubAPICredentialsFrom.SecretRef.Name, refFromRunnerDeployment(rd))
+}
+
+func (c *MultiGitHubClient) DeinitForHRA(hra *v1alpha1.HorizontalRunnerAutoscaler) {
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(hra.Namespace, secretName, refFromHorizontalRunnerAutoscaler(hra))
+}
+
+func (c *MultiGitHubClient) initClientForSecret(secret *corev1.Secret, dependent *runnerOwnerRef) (*savedClient, error) {
+	secRef := secretRef{
+		ns:   secret.Namespace,
+		name: secret.Name,
+	}
+
+	cliRef := c.clients[secRef]
+
+	var ks []string
+
+	for k := range secret.Data {
+		ks = append(ks, k)
+	}
+
+	sort.SliceStable(ks, func(i, j int) bool { return ks[i] < ks[j] })
+
+	hash := sha1.New()
+	for _, k := range ks {
+		hash.Write(secret.Data[k])
+	}
+	hashStr := hex.EncodeToString(hash.Sum(nil))
+
+	if cliRef.hash != hashStr {
+		delete(c.clients, secRef)
+
+		conf, err := secretDataToGitHubClientConfig(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		cli, err := conf.NewClient()
+		if err != nil {
+			return nil, err
+		}
+
+		cliRef = savedClient{
+			hash:   hashStr,
+			refs:   map[runnerOwnerRef]struct{}{},
+			Client: cli,
+		}
+
+		c.clients[secRef] = cliRef
+	}
+
+	if dependent != nil {
+		c.clients[secRef].refs[*dependent] = struct{}{}
+	}
+
+	return &cliRef, nil
+}
+
+func (c *MultiGitHubClient) initClientWithSecretName(ctx context.Context, ns, secretName string, runRef *runnerOwnerRef) (*github.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if secretName == "" {
+		return c.githubClient, nil
+	}
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if _, ok := c.clients[secRef]; !ok {
+		c.clients[secRef] = savedClient{}
+	}
+
+	var sec corev1.Secret
+	if err := c.client.Get(ctx, types.NamespacedName{Namespace: ns, Name: secretName}, &sec); err != nil {
+		return nil, err
+	}
+
+	savedClient, err := c.initClientForSecret(&sec, runRef)
+	if err != nil {
+		return nil, err
+	}
+
+	return savedClient.Client, nil
+}
+
+func (c *MultiGitHubClient) derefClient(ns, secretName string, dependent *runnerOwnerRef) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if dependent != nil {
+		delete(c.clients[secRef].refs, *dependent)
+	}
+
+	cliRef := c.clients[secRef]
+
+	if dependent == nil || len(cliRef.refs) == 0 {
+		delete(c.clients, secRef)
+	}
+}
+
+func decodeBase64(s []byte) (string, error) {
+	enc := base64.RawStdEncoding
+	dbuf := make([]byte, enc.DecodedLen(len(s)))
+	n, err := enc.Decode(dbuf, []byte(s))
+	if err != nil {
+		return "", err
+	}
+
+	return string(dbuf[:n]), nil
+}
+
+func secretDataToGitHubClientConfig(data map[string][]byte) (*github.Config, error) {
+	var (
+		conf github.Config
+
+		err error
+	)
+
+	conf.URL, err = decodeBase64(data["github_url"])
+	if err != nil {
+		return nil, err
+	}
+
+	conf.UploadURL, err = decodeBase64(data["github_upload_url"])
+	if err != nil {
+		return nil, err
+	}
+
+	conf.EnterpriseURL, err = decodeBase64(data["github_enterprise_url"])
+	if err != nil {
+		return nil, err
+	}
+
+	conf.RunnerGitHubURL, err = decodeBase64(data["github_runner_url"])
+	if err != nil {
+		return nil, err
+	}
+
+	conf.Token, err = decodeBase64(data["github_token"])
+	if err != nil {
+		return nil, err
+	}
+
+	appID, err := decodeBase64(data["github_app_id"])
+	if err != nil {
+		return nil, err
+	}
+
+	conf.AppID, err = strconv.ParseInt(appID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	instID, err := decodeBase64(data["github_app_installation_id"])
+	if err != nil {
+		return nil, err
+	}
+
+	conf.AppInstallationID, err = strconv.ParseInt(instID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	conf.AppPrivateKey, err = decodeBase64(data["github_app_private_key"])
+	if err != nil {
+		return nil, err
+	}
+
+	return &conf, nil
+}
+
+func refFromRunnerDeployment(rd *v1alpha1.RunnerDeployment) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rd.Kind,
+		ns:   rd.Namespace,
+		name: rd.Name,
+	}
+}
+
+func refFromRunnerReplicaSet(rs *v1alpha1.RunnerReplicaSet) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rs.Kind,
+		ns:   rs.Namespace,
+		name: rs.Name,
+	}
+}
+
+func refFromRunner(r *v1alpha1.Runner) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: r.Kind,
+		ns:   r.Namespace,
+		name: r.Name,
+	}
+}
+
+func refFromRunnerPod(po *corev1.Pod) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: po.Kind,
+		ns:   po.Namespace,
+		name: po.Name,
+	}
+}
+func refFromRunnerSet(rs *v1alpha1.RunnerSet) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rs.Kind,
+		ns:   rs.Namespace,
+		name: rs.Name,
+	}
+}
+
+func refFromHorizontalRunnerAutoscaler(hra *v1alpha1.HorizontalRunnerAutoscaler) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: hra.Kind,
+		ns:   hra.Namespace,
+		name: hra.Name,
+	}
+}

controllers/new_runner_pod_test.go

@@ -10,7 +10,9 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 func newWorkGenericEphemeralVolume(t *testing.T, storageReq string) corev1.Volume {
@@ -125,6 +127,10 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "DOCKER_HOST",
 							Value: "tcp://localhost:2376",
@@ -255,6 +261,10 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -333,6 +343,10 @@ func TestNewRunnerPod(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{
@@ -515,7 +529,7 @@ func TestNewRunnerPod(t *testing.T) {
 	for i := range testcases {
 		tc := testcases[i]
 		t.Run(tc.description, func(t *testing.T) {
-			got, err := newRunnerPod(tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
+			got, err := newRunnerPod(tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, false)
 			require.NoError(t, err)
 			require.Equal(t, tc.want, got)
 		})
@@ -624,6 +638,10 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "DOCKER_HOST",
 							Value: "tcp://localhost:2376",
@@ -769,6 +787,10 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -866,6 +888,10 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 							Name:  "RUNNER_EPHEMERAL",
 							Value: "true",
 						},
+						{
+							Name:  "RUNNER_STATUS_UPDATE_HOOK",
+							Value: "false",
+						},
 						{
 							Name:  "RUNNER_NAME",
 							Value: "runner",
@@ -1105,13 +1131,20 @@ func TestNewRunnerPodFromRunnerController(t *testing.T) {
 
 	for i := range testcases {
 		tc := testcases[i]
+
+		rr := &testResourceReader{
+			objects: map[types.NamespacedName]client.Object{},
+		}
+
+		multiClient := NewMultiGitHubClient(rr, &github.Client{GithubBaseURL: githubBaseURL})
+
 		t.Run(tc.description, func(t *testing.T) {
 			r := &RunnerReconciler{
 				RunnerImage:            defaultRunnerImage,
 				RunnerImagePullSecrets: defaultRunnerImagePullSecrets,
 				DockerImage:            defaultDockerImage,
 				DockerRegistryMirror:   defaultDockerRegistryMirror,
-				GitHubClient:           &github.Client{GithubBaseURL: githubBaseURL},
+				GitHubClient:           multiClient,
 				Scheme:                 scheme,
 			}
 			got, err := r.newPod(tc.runner)

controllers/pod_runner_token_injector.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 	"github.com/go-logr/logr"
 	"gomodules.xyz/jsonpatch/v2"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -29,7 +28,7 @@ type PodRunnerTokenInjector struct {
 	Name         string
 	Log          logr.Logger
 	Recorder     record.EventRecorder
-	GitHubClient *github.Client
+	GitHubClient *MultiGitHubClient
 	decoder      *admission.Decoder
 }
 
@@ -66,7 +65,12 @@ func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Reque
 		return newEmptyResponse()
 	}
 
-	rt, err := t.GitHubClient.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
+	ghc, err := t.GitHubClient.InitForRunnerPod(ctx, &pod)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	rt, err := ghc.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
 	if err != nil {
 		t.Log.Error(err, "Failed to get new registration token")
 		return admission.Errored(http.StatusInternalServerError, err)

controllers/runner_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -35,10 +36,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 const (
@@ -51,6 +52,8 @@ const (
 
 	EnvVarOrg        = "RUNNER_ORG"
 	EnvVarRepo       = "RUNNER_REPO"
+	EnvVarGroup      = "RUNNER_GROUP"
+	EnvVarLabels     = "RUNNER_LABELS"
 	EnvVarEnterprise = "RUNNER_ENTERPRISE"
 	EnvVarEphemeral  = "RUNNER_EPHEMERAL"
 	EnvVarTrue       = "true"
@@ -62,7 +65,7 @@ type RunnerReconciler struct {
 	Log                         logr.Logger
 	Recorder                    record.EventRecorder
 	Scheme                      *runtime.Scheme
-	GitHubClient                *github.Client
+	GitHubClient                *MultiGitHubClient
 	RunnerImage                 string
 	RunnerImagePullSecrets      []string
 	DockerImage                 string
@@ -70,8 +73,8 @@ type RunnerReconciler struct {
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
-
-	UnregistrationRetryDelay time.Duration
+	UseRunnerStatusUpdateHook   bool
+	UnregistrationRetryDelay    time.Duration
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
@@ -81,6 +84,9 @@ type RunnerReconciler struct {
 // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;delete
 // +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=create;delete;get
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=create;delete;get
+// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=create;delete;get
 
 func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := r.Log.WithValues("runner", req.NamespacedName)
@@ -116,6 +122,8 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			return r.processRunnerDeletion(runner, ctx, log, nil)
 		}
 
+		r.GitHubClient.DeinitForRunner(&runner)
+
 		return r.processRunnerDeletion(runner, ctx, log, &pod)
 	}
 
@@ -135,7 +143,7 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 	ready := runnerPodReady(&pod)
 
-	if runner.Status.Phase != phase || runner.Status.Ready != ready {
+	if (runner.Status.Phase != phase || runner.Status.Ready != ready) && !r.UseRunnerStatusUpdateHook || runner.Status.Phase == "" && r.UseRunnerStatusUpdateHook {
 		if pod.Status.Phase == corev1.PodRunning {
 			// Seeing this message, you can expect the runner to become `Running` soon.
 			log.V(1).Info(
@@ -256,6 +264,96 @@ func (r *RunnerReconciler) processRunnerCreation(ctx context.Context, runner v1a
 		return ctrl.Result{}, err
 	}
 
+	needsServiceAccount := runner.Spec.ServiceAccountName == "" && (r.UseRunnerStatusUpdateHook || runner.Spec.ContainerMode == "kubernetes")
+	if needsServiceAccount {
+		serviceAccount := &corev1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+		}
+		if res := r.createObject(ctx, serviceAccount, serviceAccount.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+
+		rules := []rbacv1.PolicyRule{}
+
+		if r.UseRunnerStatusUpdateHook {
+			rules = append(rules, []rbacv1.PolicyRule{
+				{
+					APIGroups:     []string{"actions.summerwind.dev"},
+					Resources:     []string{"runners/status"},
+					Verbs:         []string{"get", "update", "patch"},
+					ResourceNames: []string{runner.ObjectMeta.Name},
+				},
+			}...)
+		}
+
+		if runner.Spec.ContainerMode == "kubernetes" {
+			// Permissions based on https://github.com/actions/runner-container-hooks/blob/main/packages/k8s/README.md
+			rules = append(rules, []rbacv1.PolicyRule{
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods/exec"},
+					Verbs:     []string{"get", "create"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"pods/log"},
+					Verbs:     []string{"get", "list", "watch"},
+				},
+				{
+					APIGroups: []string{"batch"},
+					Resources: []string{"jobs"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+				{
+					APIGroups: []string{""},
+					Resources: []string{"secrets"},
+					Verbs:     []string{"get", "list", "create", "delete"},
+				},
+			}...)
+		}
+
+		role := &rbacv1.Role{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+			Rules: rules,
+		}
+		if res := r.createObject(ctx, role, role.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+
+		roleBinding := &rbacv1.RoleBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      runner.ObjectMeta.Name,
+				Namespace: runner.ObjectMeta.Namespace,
+			},
+			RoleRef: rbacv1.RoleRef{
+				APIGroup: "rbac.authorization.k8s.io",
+				Kind:     "Role",
+				Name:     runner.ObjectMeta.Name,
+			},
+			Subjects: []rbacv1.Subject{
+				{
+					Kind:      "ServiceAccount",
+					Name:      runner.ObjectMeta.Name,
+					Namespace: runner.ObjectMeta.Namespace,
+				},
+			},
+		}
+		if res := r.createObject(ctx, roleBinding, roleBinding.ObjectMeta, &runner, log); res != nil {
+			return *res, nil
+		}
+	}
+
 	if err := r.Create(ctx, &newPod); err != nil {
 		if kerrors.IsAlreadyExists(err) {
 			// Gracefully handle pod-already-exists errors due to informer cache delay.
@@ -278,6 +376,27 @@ func (r *RunnerReconciler) processRunnerCreation(ctx context.Context, runner v1a
 	return ctrl.Result{}, nil
 }
 
+func (r *RunnerReconciler) createObject(ctx context.Context, obj client.Object, meta metav1.ObjectMeta, runner *v1alpha1.Runner, log logr.Logger) *ctrl.Result {
+	kind := strings.Split(reflect.TypeOf(obj).String(), ".")[1]
+	if err := ctrl.SetControllerReference(runner, obj, r.Scheme); err != nil {
+		log.Error(err, fmt.Sprintf("Could not add owner reference to %s %s. %s", kind, meta.Name, err.Error()))
+		return &ctrl.Result{Requeue: true}
+	}
+	if err := r.Create(ctx, obj); err != nil {
+		if kerrors.IsAlreadyExists(err) {
+			log.Info(fmt.Sprintf("Failed to create %s %s as it already exists. Reusing existing %s", kind, meta.Name, kind))
+			r.Recorder.Event(runner, corev1.EventTypeNormal, fmt.Sprintf("%sReused", kind), fmt.Sprintf("Reused %s '%s'", kind, meta.Name))
+			return nil
+		}
+
+		log.Error(err, fmt.Sprintf("Retrying as failed to create %s %s resource", kind, meta.Name))
+		return &ctrl.Result{Requeue: true}
+	}
+	r.Recorder.Event(runner, corev1.EventTypeNormal, fmt.Sprintf("%sCreated", kind), fmt.Sprintf("Created %s '%s'", kind, meta.Name))
+	log.Info(fmt.Sprintf("Created %s", kind), "name", meta.Name)
+	return nil
+}
+
 func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v1alpha1.Runner) (bool, error) {
 	if runner.IsRegisterable() {
 		return false, nil
@@ -285,7 +404,12 @@ func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v
 
 	log := r.Log.WithValues("runner", runner.Name)
 
-	rt, err := r.GitHubClient.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+	ghc, err := r.GitHubClient.InitForRunner(ctx, &runner)
+	if err != nil {
+		return false, err
+	}
+
+	rt, err := ghc.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 	if err != nil {
 		// An error can be a permanent, permission issue like the below:
 		//    POST https://api.github.com/enterprises/YOUR_ENTERPRISE/actions/runners/registration-token: 403 Resource not accessible by integration []
@@ -325,6 +449,11 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		labels[k] = v
 	}
 
+	ghc, err := r.GitHubClient.InitForRunner(context.Background(), &runner)
+	if err != nil {
+		return corev1.Pod{}, err
+	}
+
 	// This implies that...
 	//
 	// (1) We recreate the runner pod whenever the runner has changes in:
@@ -348,7 +477,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		filterLabels(runner.ObjectMeta.Labels, LabelKeyRunnerTemplateHash),
 		runner.ObjectMeta.Annotations,
 		runner.Spec,
-		r.GitHubClient.GithubBaseURL,
+		ghc.GithubBaseURL,
 		// Token change should trigger replacement.
 		// We need to include this explicitly here because
 		// runner.Spec does not contain the possibly updated token stored in the
@@ -426,7 +555,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		}
 	}
 
-	pod, err := newRunnerPodWithContainerMode(runner.Spec.ContainerMode, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL)
+	pod, err := newRunnerPodWithContainerMode(runner.Spec.ContainerMode, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, ghc.GithubBaseURL, r.UseRunnerStatusUpdateHook)
 	if err != nil {
 		return pod, err
 	}
@@ -474,9 +603,13 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	if runnerSpec.NodeSelector != nil {
 		pod.Spec.NodeSelector = runnerSpec.NodeSelector
 	}
+
 	if runnerSpec.ServiceAccountName != "" {
 		pod.Spec.ServiceAccountName = runnerSpec.ServiceAccountName
+	} else if r.UseRunnerStatusUpdateHook || runner.Spec.ContainerMode == "kubernetes" {
+		pod.Spec.ServiceAccountName = runner.ObjectMeta.Name
 	}
+
 	if runnerSpec.AutomountServiceAccountToken != nil {
 		pod.Spec.AutomountServiceAccountToken = runnerSpec.AutomountServiceAccountToken
 	}
@@ -589,7 +722,7 @@ func runnerHookEnvs(pod *corev1.Pod) ([]corev1.EnvVar, error) {
 	}, nil
 }
 
-func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
+func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, useRunnerStatusUpdateHook bool) (corev1.Pod, error) {
 	var (
 		privileged                bool = true
 		dockerdInRunner           bool = runnerSpec.DockerdWithinRunnerContainer != nil && *runnerSpec.DockerdWithinRunnerContainer
@@ -609,6 +742,9 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 	// This label selector is used by default when rd.Spec.Selector is empty.
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunner, "")
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyPodMutation, LabelValuePodMutation)
+	if runnerSpec.GitHubAPICredentialsFrom != nil {
+		template.ObjectMeta.Annotations = CloneAndAddLabel(template.ObjectMeta.Annotations, annotationKeyGitHubAPICredsSecret, runnerSpec.GitHubAPICredentialsFrom.SecretRef.Name)
+	}
 
 	workDir := runnerSpec.WorkDir
 	if workDir == "" {
@@ -638,11 +774,11 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 			Value: runnerSpec.Enterprise,
 		},
 		{
-			Name:  "RUNNER_LABELS",
+			Name:  EnvVarLabels,
 			Value: strings.Join(runnerSpec.Labels, ","),
 		},
 		{
-			Name:  "RUNNER_GROUP",
+			Name:  EnvVarGroup,
 			Value: runnerSpec.Group,
 		},
 		{
@@ -665,6 +801,10 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 			Name:  EnvVarEphemeral,
 			Value: fmt.Sprintf("%v", ephemeral),
 		},
+		{
+			Name:  "RUNNER_STATUS_UPDATE_HOOK",
+			Value: fmt.Sprintf("%v", useRunnerStatusUpdateHook),
+		},
 	}
 
 	var seLinuxOptions *corev1.SELinuxOptions
@@ -962,8 +1102,8 @@ func newRunnerPodWithContainerMode(containerMode string, template corev1.Pod, ru
 	return *pod, nil
 }
 
-func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string) (corev1.Pod, error) {
-	return newRunnerPodWithContainerMode("", template, runnerSpec, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL)
+func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, useRunnerStatusUpdateHookEphemeralRole bool) (corev1.Pod, error) {
+	return newRunnerPodWithContainerMode("", template, runnerSpec, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, useRunnerStatusUpdateHookEphemeralRole)
 }
 
 func (r *RunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {

controllers/runner_graceful_stop.go

@@ -67,25 +67,6 @@ func annotatePodOnce(ctx context.Context, c client.Client, log logr.Logger, pod
 
 	return updated, nil
 }
-func labelPod(ctx context.Context, c client.Client, log logr.Logger, pod *corev1.Pod, k, v string) (*corev1.Pod, error) {
-	if pod == nil {
-		return nil, nil
-	}
-
-	updated := pod.DeepCopy()
-	if updated.Labels == nil {
-		updated.Labels = map[string]string{}
-	}
-	updated.Labels[k] = v
-	if err := c.Patch(ctx, updated, client.MergeFrom(pod)); err != nil {
-		log.Error(err, fmt.Sprintf("Failed to patch pod to have %s annotation", k))
-		return nil, err
-	}
-
-	log.V(2).Info("Labeled pod", "key", k, "value", v)
-
-	return updated, nil
-}
 
 // If the first return value is nil, it's safe to delete the runner pod.
 func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*ctrl.Result, error) {
@@ -210,7 +191,7 @@ func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, l
 		}
 
 		if runnerBusy {
-			_, err := labelPod(ctx, c, log, pod, AnnotationKeyUnregistrationFailureMessage, runnerUnregistrationFailureMessage)
+			_, err := annotatePodOnce(ctx, c, log, pod, AnnotationKeyUnregistrationFailureMessage, runnerUnregistrationFailureMessage)
 			if err != nil {
 				return &ctrl.Result{}, err
 			}

controllers/runner_pod_controller.go

@@ -32,8 +32,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	corev1 "k8s.io/api/core/v1"
-
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 // RunnerPodReconciler reconciles a Runner object
@@ -42,7 +40,7 @@ type RunnerPodReconciler struct {
 	Log                         logr.Logger
 	Recorder                    record.EventRecorder
 	Scheme                      *runtime.Scheme
-	GitHubClient                *github.Client
+	GitHubClient                *MultiGitHubClient
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
@@ -97,6 +95,11 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		}
 	}
 
+	ghc, err := r.GitHubClient.InitForRunnerPod(ctx, &runnerPod)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
 	if runnerPod.ObjectMeta.DeletionTimestamp.IsZero() {
 		finalizers, added := addFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
 
@@ -148,7 +151,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			// In a standard scenario, the upstream controller, like runnerset-controller, ensures this runner to be gracefully stopped before the deletion timestamp is set.
 			// But for the case that the user manually deleted it for whatever reason,
 			// we have to ensure it to gracefully stop now.
-			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 			if res != nil {
 				return *res, err
 			}
@@ -164,6 +167,8 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 			log.V(2).Info("Removed finalizer")
 
+			r.GitHubClient.DeinitForRunnerPod(updatedPod)
+
 			return ctrl.Result{}, nil
 		}
 
@@ -202,7 +207,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
-	po, res, err := ensureRunnerPodRegistered(ctx, log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+	po, res, err := ensureRunnerPodRegistered(ctx, log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 	if res != nil {
 		return *res, err
 	}
@@ -216,7 +221,7 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		//
 		// In a standard scenario, ARC starts the unregistration process before marking the pod for deletion at all,
 		// so that it isn't subject to terminationGracePeriod and can safely take hours to finish it's work.
-		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
 		if res != nil {
 			return *res, err
 		}

controllers/runnerreplicaset_controller.go

@@ -32,17 +32,15 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/actions-runner-controller/actions-runner-controller/github"
 )
 
 // RunnerReplicaSetReconciler reconciles a Runner object
 type RunnerReplicaSetReconciler struct {
 	client.Client
-	Log          logr.Logger
-	Recorder     record.EventRecorder
-	Scheme       *runtime.Scheme
-	GitHubClient *github.Client
-	Name         string
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
 }
 
 const (

controllers/runnerreplicaset_controller_test.go

@@ -52,15 +52,13 @@ func SetupTest(ctx2 context.Context) *corev1.Namespace {
 
 		runnersList = fake.NewRunnersList()
 		server = runnersList.GetServer()
-		ghClient := newGithubClient(server)
 
 		controller := &RunnerReplicaSetReconciler{
-			Client:       mgr.GetClient(),
-			Scheme:       scheme.Scheme,
-			Log:          logf.Log,
-			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
-			GitHubClient: ghClient,
-			Name:         "runnerreplicaset-" + ns.Name,
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     "runnerreplicaset-" + ns.Name,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")

controllers/runnerset_controller.go

@@ -45,12 +45,13 @@ type RunnerSetReconciler struct {
 	Recorder record.EventRecorder
 	Scheme   *runtime.Scheme
 
-	CommonRunnerLabels     []string
-	GitHubBaseURL          string
-	RunnerImage            string
-	RunnerImagePullSecrets []string
-	DockerImage            string
-	DockerRegistryMirror   string
+	CommonRunnerLabels        []string
+	GitHubClient              *MultiGitHubClient
+	RunnerImage               string
+	RunnerImagePullSecrets    []string
+	DockerImage               string
+	DockerRegistryMirror      string
+	UseRunnerStatusUpdateHook bool
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets,verbs=get;list;watch;create;update;patch;delete
@@ -80,6 +81,8 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	}
 
 	if !runnerSet.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForRunnerSet(runnerSet)
+
 		return ctrl.Result{}, nil
 	}
 
@@ -97,7 +100,7 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
-	desiredStatefulSet, err := r.newStatefulSet(runnerSet)
+	desiredStatefulSet, err := r.newStatefulSet(ctx, runnerSet)
 	if err != nil {
 		r.Recorder.Event(runnerSet, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -185,7 +188,7 @@ func getRunnerSetSelector(runnerSet *v1alpha1.RunnerSet) *metav1.LabelSelector {
 var LabelKeyPodMutation = "actions-runner-controller/inject-registration-token"
 var LabelValuePodMutation = "true"
 
-func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
+func (r *RunnerSetReconciler) newStatefulSet(ctx context.Context, runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
 	runnerSetWithOverrides := *runnerSet.Spec.DeepCopy()
 
 	runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, r.CommonRunnerLabels...)
@@ -221,7 +224,14 @@ func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*ap
 
 	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerSet.Name)
 
-	pod, err := newRunnerPodWithContainerMode(runnerSet.Spec.RunnerConfig.ContainerMode, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL)
+	ghc, err := r.GitHubClient.InitForRunnerSet(ctx, runnerSet)
+	if err != nil {
+		return nil, err
+	}
+
+	githubBaseURL := ghc.GithubBaseURL
+
+	pod, err := newRunnerPodWithContainerMode(runnerSet.Spec.RunnerConfig.ContainerMode, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, githubBaseURL, r.UseRunnerStatusUpdateHook)
 	if err != nil {
 		return nil, err
 	}

controllers/testresourcereader.go

@@ -0,0 +1,31 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"reflect"
+
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type testResourceReader struct {
+	objects map[types.NamespacedName]client.Object
+}
+
+func (r *testResourceReader) Get(_ context.Context, nsName types.NamespacedName, obj client.Object) error {
+	ret, ok := r.objects[nsName]
+	if !ok {
+		return &kerrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonNotFound}}
+	}
+	v := reflect.ValueOf(obj)
+	if v.Kind() != reflect.Ptr {
+		return errors.New("obj must be a pointer")
+	}
+
+	v.Elem().Set(reflect.ValueOf(ret).Elem())
+
+	return nil
+}

controllers/testresourcereader_test.go

@@ -0,0 +1,35 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestResourceReader(t *testing.T) {
+	rr := &testResourceReader{
+		objects: map[types.NamespacedName]client.Object{
+			{Namespace: "default", Name: "sec1"}: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "sec1",
+				},
+				Data: map[string][]byte{
+					"foo": []byte("bar"),
+				},
+			},
+		},
+	}
+
+	var sec corev1.Secret
+
+	err := rr.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "sec1"}, &sec)
+	require.NoError(t, err)
+
+	require.Equal(t, []byte("bar"), sec.Data["foo"])
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/actions-runner-controller/actions-runner-controller
 go 1.18
 
 require (
-	github.com/bradleyfalzon/ghinstallation/v2 v2.0.4
+	github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v1.2.3
 	github.com/google/go-cmp v0.5.8
@@ -22,7 +22,7 @@ require (
 	k8s.io/api v0.24.2
 	k8s.io/apimachinery v0.24.2
 	k8s.io/client-go v0.24.2
-	sigs.k8s.io/controller-runtime v0.12.2
+	sigs.k8s.io/controller-runtime v0.12.3
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -40,7 +40,7 @@ require (
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect

go.sum

@@ -84,6 +84,8 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 h1:tXKVfhE7FcSkhkv0UwkLvPDeZ4kz6OXd0PKPlFqf81M=
 github.com/bradleyfalzon/ghinstallation/v2 v2.0.4/go.mod h1:B40qPqJxWE0jDZgOR1JmaMy+4AY1eBP+IByOvqyAKp0=
+github.com/bradleyfalzon/ghinstallation/v2 v2.1.0 h1:5+NghM1Zred9Z078QEZtm28G/kfDfZN/92gkDlLwGVA=
+github.com/bradleyfalzon/ghinstallation/v2 v2.1.0/go.mod h1:Xg3xPRN5Mcq6GDqeUVhFbjEWMb4JHCyWEeeBGEYQoTU=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
@@ -181,6 +183,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
+github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -1043,6 +1047,8 @@ sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCw
 sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4=
 sigs.k8s.io/controller-runtime v0.12.2 h1:nqV02cvhbAj7tbt21bpPpTByrXGn2INHRsi39lXy9sE=
 sigs.k8s.io/controller-runtime v0.12.2/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
+sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio=
+sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=

main.go

@@ -68,14 +68,14 @@ func main() {
 		err      error
 		ghClient *github.Client
 
-		metricsAddr          string
-		enableLeaderElection bool
-		leaderElectionId     string
-		port                 int
-		syncPeriod           time.Duration
+		metricsAddr            string
+		enableLeaderElection   bool
+		runnerStatusUpdateHook bool
+		leaderElectionId       string
+		port                   int
+		syncPeriod             time.Duration
 
-		gitHubAPICacheDuration time.Duration
-		defaultScaleDownDelay  time.Duration
+		defaultScaleDownDelay time.Duration
 
 		runnerImage            string
 		runnerImagePullSecrets stringSlice
@@ -112,7 +112,7 @@ func main() {
 	flag.StringVar(&c.BasicauthUsername, "github-basicauth-username", c.BasicauthUsername, "Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
 	flag.StringVar(&c.BasicauthPassword, "github-basicauth-password", c.BasicauthPassword, "Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
 	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
-	flag.DurationVar(&gitHubAPICacheDuration, "github-api-cache-duration", 0, "DEPRECATED: The duration until the GitHub API cache expires. Setting this to e.g. 10m results in the controller tries its best not to make the same API call within 10m to reduce the chance of being rate-limited. Defaults to mostly the same value as sync-period. If you're tweaking this in order to make autoscaling more responsive, you'll probably want to tweak sync-period, too")
+	flag.BoolVar(&runnerStatusUpdateHook, "runner-status-update-hook", false, "Use custom RBAC for runners (role, role binding and service account).")
 	flag.DurationVar(&defaultScaleDownDelay, "default-scale-down-delay", controllers.DefaultScaleDownDelay, "The approximate delay for a scale down followed by a scale up, used to prevent flapping (down->up->down->... loop)")
 	flag.IntVar(&port, "port", 9443, "The port to which the admission webhook endpoint should bind")
 	flag.DurationVar(&syncPeriod, "sync-period", 1*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled.")
@@ -147,13 +147,19 @@ func main() {
 		os.Exit(1)
 	}
 
+	multiClient := controllers.NewMultiGitHubClient(
+		mgr.GetClient(),
+		ghClient,
+	)
+
 	runnerReconciler := &controllers.RunnerReconciler{
-		Client:               mgr.GetClient(),
-		Log:                  log.WithName("runner"),
-		Scheme:               mgr.GetScheme(),
-		GitHubClient:         ghClient,
-		DockerImage:          dockerImage,
-		DockerRegistryMirror: dockerRegistryMirror,
+		Client:                    mgr.GetClient(),
+		Log:                       log.WithName("runner"),
+		Scheme:                    mgr.GetScheme(),
+		GitHubClient:              multiClient,
+		DockerImage:               dockerImage,
+		DockerRegistryMirror:      dockerRegistryMirror,
+		UseRunnerStatusUpdateHook: runnerStatusUpdateHook,
 		// Defaults for self-hosted runner containers
 		RunnerImage:            runnerImage,
 		RunnerImagePullSecrets: runnerImagePullSecrets,
@@ -165,10 +171,9 @@ func main() {
 	}
 
 	runnerReplicaSetReconciler := &controllers.RunnerReplicaSetReconciler{
-		Client:       mgr.GetClient(),
-		Log:          log.WithName("runnerreplicaset"),
-		Scheme:       mgr.GetScheme(),
-		GitHubClient: ghClient,
+		Client: mgr.GetClient(),
+		Log:    log.WithName("runnerreplicaset"),
+		Scheme: mgr.GetScheme(),
 	}
 
 	if err = runnerReplicaSetReconciler.SetupWithManager(mgr); err != nil {
@@ -195,27 +200,20 @@ func main() {
 		CommonRunnerLabels:   commonRunnerLabels,
 		DockerImage:          dockerImage,
 		DockerRegistryMirror: dockerRegistryMirror,
-		GitHubBaseURL:        ghClient.GithubBaseURL,
+		GitHubClient:         multiClient,
 		// Defaults for self-hosted runner containers
-		RunnerImage:            runnerImage,
-		RunnerImagePullSecrets: runnerImagePullSecrets,
+		RunnerImage:               runnerImage,
+		RunnerImagePullSecrets:    runnerImagePullSecrets,
+		UseRunnerStatusUpdateHook: runnerStatusUpdateHook,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {
 		log.Error(err, "unable to create controller", "controller", "RunnerSet")
 		os.Exit(1)
 	}
-	if gitHubAPICacheDuration == 0 {
-		gitHubAPICacheDuration = syncPeriod - 10*time.Second
-	}
-
-	if gitHubAPICacheDuration < 0 {
-		gitHubAPICacheDuration = 0
-	}
 
 	log.Info(
 		"Initializing actions-runner-controller",
-		"github-api-cache-duration", gitHubAPICacheDuration,
 		"default-scale-down-delay", defaultScaleDownDelay,
 		"sync-period", syncPeriod,
 		"default-runner-image", runnerImage,
@@ -230,8 +228,7 @@ func main() {
 		Client:                mgr.GetClient(),
 		Log:                   log.WithName("horizontalrunnerautoscaler"),
 		Scheme:                mgr.GetScheme(),
-		GitHubClient:          ghClient,
-		CacheDuration:         gitHubAPICacheDuration,
+		GitHubClient:          multiClient,
 		DefaultScaleDownDelay: defaultScaleDownDelay,
 	}
 
@@ -239,7 +236,7 @@ func main() {
 		Client:       mgr.GetClient(),
 		Log:          log.WithName("runnerpod"),
 		Scheme:       mgr.GetScheme(),
-		GitHubClient: ghClient,
+		GitHubClient: multiClient,
 	}
 
 	runnerPersistentVolumeReconciler := &controllers.RunnerPersistentVolumeReconciler{
@@ -290,7 +287,7 @@ func main() {
 
 	injector := &controllers.PodRunnerTokenInjector{
 		Client:       mgr.GetClient(),
-		GitHubClient: ghClient,
+		GitHubClient: multiClient,
 		Log:          ctrl.Log.WithName("webhook").WithName("PodRunnerTokenInjector"),
 	}
 	if err = injector.SetupWithManager(mgr); err != nil {

runner/actions-runner-dind.dockerfile

@@ -98,10 +98,13 @@ RUN mkdir /opt/hostedtoolcache \
 
 # We place the scripts in `/usr/bin` so that users who extend this image can
 # override them with scripts of the same name placed in `/usr/local/bin`.
-COPY entrypoint.sh logger.bash startup.sh /usr/bin/
+COPY entrypoint.sh logger.bash startup.sh update-status /usr/bin/
 COPY supervisor/ /etc/supervisor/conf.d/
 RUN chmod +x /usr/bin/startup.sh /usr/bin/entrypoint.sh
 
+# Configure hooks folder structure.
+COPY hooks /etc/arc/hooks/
+
 # arch command on OS X reports "i386" for Intel CPUs regardless of bitness
 RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
     && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \

runner/actions-runner.dockerfile

@@ -116,7 +116,10 @@ RUN mkdir /opt/hostedtoolcache \
 
 # We place the scripts in `/usr/bin` so that users who extend this image can
 # override them with scripts of the same name placed in `/usr/local/bin`.
-COPY entrypoint.sh logger.bash /usr/bin/
+COPY entrypoint.sh logger.bash update-status /usr/bin/
+
+# Configure hooks folder structure.
+COPY hooks /etc/arc/hooks/
 
 ENV HOME=/home/runner
 # Add the Python "User Script Directory" to the PATH

runner/entrypoint.sh

@@ -4,6 +4,13 @@ source logger.bash
 RUNNER_ASSETS_DIR=${RUNNER_ASSETS_DIR:-/runnertmp}
 RUNNER_HOME=${RUNNER_HOME:-/runner}
 
+# Let GitHub runner execute these hooks. These environment variables are used by GitHub's Runner as described here
+# https://github.com/actions/runner/blob/main/docs/adrs/1751-runner-job-hooks.md
+# Scripts referenced in the ACTIONS_RUNNER_HOOK_ environment variables must end in .sh or .ps1
+# for it to become a valid hook script, otherwise GitHub will fail to run the hook
+export ACTIONS_RUNNER_HOOK_JOB_STARTED=/etc/arc/hooks/job-started.sh
+export ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/etc/arc/hooks/job-completed.sh
+
 if [ ! -z "${STARTUP_DELAY_IN_SECONDS}" ]; then
   log.notice "Delaying startup by ${STARTUP_DELAY_IN_SECONDS} seconds"
   sleep ${STARTUP_DELAY_IN_SECONDS}
@@ -77,6 +84,8 @@ if [ "${DISABLE_RUNNER_UPDATE:-}" == "true" ]; then
   log.debug 'Passing --disableupdate to config.sh to disable automatic runner updates.'
 fi
 
+update-status "Registering"
+
 retries_left=10
 while [[ ${retries_left} -gt 0 ]]; do
   log.debug 'Configuring the runner.'
@@ -155,4 +164,5 @@ unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN STARTUP_DELAY_IN_SECONDS DISABLE_WAIT
 if [ -z "${UNITTEST:-}" ]; then
   mapfile -t env </etc/environment
 fi
+update-status "Idle"
 exec env -- "${env[@]}" ./run.sh

runner/hooks/job-completed.d/update-status

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -u
+
+exec update-status Idle

runner/hooks/job-completed.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# shellcheck source=runner/logger.bash
+source logger.bash
+
+log.debug "Running ARC Job Completed Hooks"
+
+for hook in /etc/arc/hooks/job-completed.d/*; do
+  log.debug "Running hook: $hook"
+  "$hook" "$@"
+done

runner/hooks/job-started.d/update-status

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -u
+
+exec update-status Running "Run $GITHUB_RUN_ID from $GITHUB_REPOSITORY"

runner/hooks/job-started.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# shellcheck source=runner/logger.bash
+source logger.bash
+
+log.debug "Running ARC Job Started Hooks"
+
+for hook in /etc/arc/hooks/job-started.d/*; do
+  log.debug "Running hook: $hook"
+  "$hook" "$@"
+done

runner/update-status

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ ${1:-} == '' ]]; then
+  # shellcheck source=runner/logger.bash
+  source logger.bash
+  log.error "Missing required argument -- '<phase>'"
+  exit 64
+fi
+
+if [[ ${RUNNER_STATUS_UPDATE_HOOK:-false} == true ]]; then
+
+    apiserver=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}
+    serviceaccount=/var/run/secrets/kubernetes.io/serviceaccount
+    namespace=$(cat ${serviceaccount}/namespace)
+    token=$(cat ${serviceaccount}/token)
+    phase=$1
+    shift
+
+    jq -n --arg phase "$phase" --arg message "${*:-}" '.status.phase = $phase | .status.message = $message' | curl \
+        --cacert ${serviceaccount}/ca.crt \
+        --data @- \
+        --noproxy '*' \
+        --header "Content-Type: application/merge-patch+json" \
+        --header "Authorization: Bearer ${token}" \
+        --show-error \
+        --silent \
+        --request PATCH \
+        "${apiserver}/apis/actions.summerwind.dev/v1alpha1/namespaces/${namespace}/runners/${HOSTNAME}/status"
+        1>&-
+fi

test/e2e/e2e_test.go

@@ -21,53 +21,7 @@ const (
 )
 
 var (
-	controllerImageRepo = "actionsrunnercontrollere2e/actions-runner-controller"
-	controllerImageTag  = "e2e"
-	controllerImage     = testing.Img(controllerImageRepo, controllerImageTag)
-	runnerImageRepo     = "actionsrunnercontrollere2e/actions-runner"
-	runnerDindImageRepo = "actionsrunnercontrollere2e/actions-runner-dind"
-	runnerImageTag      = "e2e"
-	runnerImage         = testing.Img(runnerImageRepo, runnerImageTag)
-	runnerDindImage     = testing.Img(runnerDindImageRepo, runnerImageTag)
-
-	prebuildImages = []testing.ContainerImage{
-		controllerImage,
-		runnerImage,
-		runnerDindImage,
-	}
-
-	builds = []testing.DockerBuild{
-		{
-			Dockerfile:   "../../Dockerfile",
-			Args:         []testing.BuildArg{},
-			Image:        controllerImage,
-			EnableBuildX: true,
-		},
-		{
-			Dockerfile: "../../runner/actions-runner.dockerfile",
-			Args: []testing.BuildArg{
-				{
-					Name:  "RUNNER_VERSION",
-					Value: "2.294.0",
-				},
-			},
-			Image:        runnerImage,
-			EnableBuildX: true,
-		},
-		{
-			Dockerfile: "../../runner/actions-runner-dind.dockerfile",
-			Args: []testing.BuildArg{
-				{
-					Name:  "RUNNER_VERSION",
-					Value: "2.294.0",
-				},
-			},
-			Image:        runnerDindImage,
-			EnableBuildX: true,
-		},
-	}
-
-	certManagerVersion = "v1.1.1"
+	certManagerVersion = "v1.8.2"
 
 	images = []testing.ContainerImage{
 		testing.Img("docker", "dind"),
@@ -77,11 +31,6 @@ var (
 		testing.Img("quay.io/jetstack/cert-manager-webhook", certManagerVersion),
 	}
 
-	commonScriptEnv = []string{
-		"SYNC_PERIOD=" + "30s",
-		"RUNNER_TAG=" + runnerImageTag,
-	}
-
 	testResultCMNamePrefix = "test-result-"
 )
 
@@ -122,16 +71,31 @@ func TestE2E(t *testing.T) {
 		t.Skip("Skipped as -short is set")
 	}
 
+	k8sMinorVer := os.Getenv("ARC_E2E_KUBE_VERSION")
 	skipRunnerCleanUp := os.Getenv("ARC_E2E_SKIP_RUNNER_CLEANUP") != ""
 	retainCluster := os.Getenv("ARC_E2E_RETAIN_CLUSTER") != ""
 	skipTestIDCleanUp := os.Getenv("ARC_E2E_SKIP_TEST_ID_CLEANUP") != ""
+	skipArgoTunnelCleanUp := os.Getenv("ARC_E2E_SKIP_ARGO_TUNNEL_CLEAN_UP") != ""
 
-	env := initTestEnv(t)
+	vars := buildVars(os.Getenv("ARC_E2E_IMAGE_REPO"))
+
+	env := initTestEnv(t, k8sMinorVer, vars)
+	if vt := os.Getenv("ARC_E2E_VERIFY_TIMEOUT"); vt != "" {
+		var err error
+		env.VerifyTimeout, err = time.ParseDuration(vt)
+		if err != nil {
+			t.Fatalf("Failed to parse duration %q: %v", vt, err)
+		}
+	}
 
 	t.Run("build and load images", func(t *testing.T) {
 		env.buildAndLoadImages(t)
 	})
 
+	if t.Failed() {
+		return
+	}
+
 	t.Run("install cert-manager", func(t *testing.T) {
 		env.installCertManager(t)
 	})
@@ -141,6 +105,10 @@ func TestE2E(t *testing.T) {
 	}
 
 	t.Run("RunnerSets", func(t *testing.T) {
+		if os.Getenv("ARC_E2E_SKIP_RUNNERSETS") != "" {
+			t.Skip("RunnerSets test has been skipped due to ARC_E2E_SKIP_RUNNERSETS")
+		}
+
 		var (
 			testID string
 		)
@@ -159,6 +127,16 @@ func TestE2E(t *testing.T) {
 			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
 		})
 
+		t.Run("install argo-tunnel", func(t *testing.T) {
+			env.installArgoTunnel(t)
+		})
+
+		if !skipArgoTunnelCleanUp {
+			t.Cleanup(func() {
+				env.uninstallArgoTunnel(t)
+			})
+		}
+
 		t.Run("deploy runners", func(t *testing.T) {
 			env.deploy(t, RunnerSets, testID)
 		})
@@ -170,7 +148,7 @@ func TestE2E(t *testing.T) {
 		}
 
 		t.Run("install edge actions-runner-controller", func(t *testing.T) {
-			env.installActionsRunnerController(t, controllerImageRepo, controllerImageTag, testID)
+			env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
 		})
 
 		if t.Failed() {
@@ -209,6 +187,16 @@ func TestE2E(t *testing.T) {
 			env.installActionsRunnerController(t, "summerwind/actions-runner-controller", "v0.24.1", testID)
 		})
 
+		t.Run("install argo-tunnel", func(t *testing.T) {
+			env.installArgoTunnel(t)
+		})
+
+		if !skipArgoTunnelCleanUp {
+			t.Cleanup(func() {
+				env.uninstallArgoTunnel(t)
+			})
+		}
+
 		t.Run("deploy runners", func(t *testing.T) {
 			env.deploy(t, RunnerDeployments, testID)
 		})
@@ -220,7 +208,7 @@ func TestE2E(t *testing.T) {
 		}
 
 		t.Run("install edge actions-runner-controller", func(t *testing.T) {
-			env.installActionsRunnerController(t, controllerImageRepo, controllerImageTag, testID)
+			env.installActionsRunnerController(t, vars.controllerImageRepo, vars.controllerImageTag, testID)
 		})
 
 		if t.Failed() {
@@ -248,6 +236,8 @@ func TestE2E(t *testing.T) {
 type env struct {
 	*testing.Env
 
+	Kind *testing.Kind
+
 	// Uses GITHUB_APP_ID, GITHUB_APP_INSTALLATION_ID, and GITHUB_APP_PRIVATE_KEY
 	// to let ARC authenticate as a GitHub App
 	useApp bool
@@ -262,12 +252,99 @@ type env struct {
 	scaleDownDelaySecondsAfterScaleOut          int64
 	minReplicas                                 int64
 	dockerdWithinRunnerContainer                bool
+	remoteKubeconfig                            string
+	imagePullSecretName                         string
+	imagePullPolicy                             string
+
+	vars          vars
+	VerifyTimeout time.Duration
 }
 
-func initTestEnv(t *testing.T) *env {
+type vars struct {
+	controllerImageRepo, controllerImageTag string
+
+	runnerImageRepo     string
+	runnerDindImageRepo string
+
+	prebuildImages []testing.ContainerImage
+	builds         []testing.DockerBuild
+
+	commonScriptEnv []string
+}
+
+func buildVars(repo string) vars {
+	if repo == "" {
+		repo = "actionsrunnercontrollere2e"
+	}
+
+	var (
+		controllerImageRepo = repo + "/actions-runner-controller"
+		controllerImageTag  = "e2e"
+		controllerImage     = testing.Img(controllerImageRepo, controllerImageTag)
+		runnerImageRepo     = repo + "/actions-runner"
+		runnerDindImageRepo = repo + "/actions-runner-dind"
+		runnerImageTag      = "e2e"
+		runnerImage         = testing.Img(runnerImageRepo, runnerImageTag)
+		runnerDindImage     = testing.Img(runnerDindImageRepo, runnerImageTag)
+	)
+
+	var vs vars
+
+	vs.controllerImageRepo, vs.controllerImageTag = controllerImageRepo, controllerImageTag
+	vs.runnerDindImageRepo = runnerDindImageRepo
+	vs.runnerImageRepo = runnerImageRepo
+
+	// vs.controllerImage, vs.controllerImageTag
+
+	vs.prebuildImages = []testing.ContainerImage{
+		controllerImage,
+		runnerImage,
+		runnerDindImage,
+	}
+
+	vs.builds = []testing.DockerBuild{
+		{
+			Dockerfile:   "../../Dockerfile",
+			Args:         []testing.BuildArg{},
+			Image:        controllerImage,
+			EnableBuildX: true,
+		},
+		{
+			Dockerfile: "../../runner/actions-runner.dockerfile",
+			Args: []testing.BuildArg{
+				{
+					Name:  "RUNNER_VERSION",
+					Value: "2.294.0",
+				},
+			},
+			Image:        runnerImage,
+			EnableBuildX: true,
+		},
+		{
+			Dockerfile: "../../runner/actions-runner-dind.dockerfile",
+			Args: []testing.BuildArg{
+				{
+					Name:  "RUNNER_VERSION",
+					Value: "2.294.0",
+				},
+			},
+			Image:        runnerDindImage,
+			EnableBuildX: true,
+		},
+	}
+
+	vs.commonScriptEnv = []string{
+		"SYNC_PERIOD=" + "30s",
+		"RUNNER_TAG=" + runnerImageTag,
+	}
+
+	return vs
+}
+
+func initTestEnv(t *testing.T, k8sMinorVer string, vars vars) *env {
 	t.Helper()
 
-	testingEnv := testing.Start(t, testing.Preload(images...))
+	testingEnv := testing.Start(t, k8sMinorVer)
 
 	e := &env{Env: testingEnv}
 
@@ -287,6 +364,29 @@ func initTestEnv(t *testing.T) *env {
 	e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO", "")
 	e.testEnterprise = testing.Getenv(t, "TEST_ENTERPRISE", "")
 	e.testEphemeral = testing.Getenv(t, "TEST_EPHEMERAL", "")
+	e.remoteKubeconfig = testing.Getenv(t, "ARC_E2E_REMOTE_KUBECONFIG", "")
+	e.imagePullSecretName = testing.Getenv(t, "ARC_E2E_IMAGE_PULL_SECRET_NAME", "")
+	e.vars = vars
+
+	if e.remoteKubeconfig != "" {
+		e.imagePullPolicy = "Always"
+	} else {
+		e.imagePullPolicy = "IfNotPresent"
+	}
+
+	if e.remoteKubeconfig == "" {
+		e.Kind = testing.StartKind(t, k8sMinorVer, testing.Preload(images...))
+		e.Env.Kubeconfig = e.Kind.Kubeconfig()
+	} else {
+		e.Env.Kubeconfig = e.remoteKubeconfig
+
+		// Kind automatically installs https://github.com/rancher/local-path-provisioner for PVs.
+		// But assuming the remote cluster isn't a kind Kubernetes cluster,
+		// we need to install any provisioner manually.
+		// Here, we install the local-path-provisioner on the remote cluster too,
+		// so that we won't suffer from E2E failures due to the provisioner difference.
+		e.KubectlApply(t, "https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.22/deploy/local-path-storage.yaml", testing.KubectlConfig{})
+	}
 
 	e.scaleDownDelaySecondsAfterScaleOut, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT", "10"), 10, 32)
 	e.minReplicas, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_MIN_REPLICAS", "1"), 10, 32)
@@ -306,8 +406,29 @@ func (e *env) f() {
 func (e *env) buildAndLoadImages(t *testing.T) {
 	t.Helper()
 
-	e.DockerBuild(t, builds)
-	e.KindLoadImages(t, prebuildImages)
+	e.DockerBuild(t, e.vars.builds)
+
+	if e.remoteKubeconfig == "" {
+		e.KindLoadImages(t, e.vars.prebuildImages)
+	} else {
+		// If it fails with `no basic auth credentials` here, you might have missed logging into the container registry beforehand.
+		// For ECR, run something like:
+		//   aws ecr get-login-password | docker login --username AWS --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com
+		// Also note that the authenticated session can be expired in a day or so(probably depends on your AWS config),
+		// so you might better write a script to do docker login before running the E2E test.
+		e.DockerPush(t, e.vars.prebuildImages)
+	}
+}
+
+func (e *env) KindLoadImages(t *testing.T, prebuildImages []testing.ContainerImage) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
+	defer cancel()
+
+	if err := e.Kind.LoadImages(ctx, prebuildImages); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func (e *env) installCertManager(t *testing.T) {
@@ -333,7 +454,7 @@ func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID str
 	e.createControllerNamespaceAndServiceAccount(t)
 
 	scriptEnv := []string{
-		"KUBECONFIG=" + e.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 		"ACCEPTANCE_TEST_DEPLOYMENT_TOOL=" + "helm",
 	}
 
@@ -342,6 +463,8 @@ func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID str
 		"TEST_ID=" + testID,
 		"NAME=" + repo,
 		"VERSION=" + tag,
+		"IMAGE_PULL_SECRET=" + e.imagePullSecretName,
+		"IMAGE_PULL_POLICY=" + e.imagePullPolicy,
 	}
 
 	if e.useApp {
@@ -359,7 +482,7 @@ func (e *env) installActionsRunnerController(t *testing.T, repo, tag, testID str
 	}
 
 	scriptEnv = append(scriptEnv, varEnv...)
-	scriptEnv = append(scriptEnv, commonScriptEnv...)
+	scriptEnv = append(scriptEnv, e.vars.commonScriptEnv...)
 
 	e.RunScript(t, "../../acceptance/deploy.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
 }
@@ -380,7 +503,7 @@ func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
 	e.createControllerNamespaceAndServiceAccount(t)
 
 	scriptEnv := []string{
-		"KUBECONFIG=" + e.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 		"OP=" + op,
 	}
 
@@ -409,21 +532,43 @@ func (e *env) do(t *testing.T, op string, kind DeployKind, testID string) {
 	if e.dockerdWithinRunnerContainer {
 		varEnv = append(varEnv,
 			"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=true",
-			"RUNNER_NAME="+runnerDindImageRepo,
+			"RUNNER_NAME="+e.vars.runnerDindImageRepo,
 		)
 	} else {
 		varEnv = append(varEnv,
 			"RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER=false",
-			"RUNNER_NAME="+runnerImageRepo,
+			"RUNNER_NAME="+e.vars.runnerImageRepo,
 		)
 	}
 
 	scriptEnv = append(scriptEnv, varEnv...)
-	scriptEnv = append(scriptEnv, commonScriptEnv...)
+	scriptEnv = append(scriptEnv, e.vars.commonScriptEnv...)
 
 	e.RunScript(t, "../../acceptance/deploy_runners.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
 }
 
+func (e *env) installArgoTunnel(t *testing.T) {
+	e.doArgoTunnel(t, "apply")
+}
+
+func (e *env) uninstallArgoTunnel(t *testing.T) {
+	e.doArgoTunnel(t, "delete")
+}
+
+func (e *env) doArgoTunnel(t *testing.T, op string) {
+	t.Helper()
+
+	scriptEnv := []string{
+		"KUBECONFIG=" + e.Kubeconfig,
+		"OP=" + op,
+		"TUNNEL_ID=" + os.Getenv("TUNNEL_ID"),
+		"TUNNE_NAME=" + os.Getenv("TUNNEL_NAME"),
+		"TUNNEL_HOSTNAME=" + os.Getenv("TUNNEL_HOSTNAME"),
+	}
+
+	e.RunScript(t, "../../acceptance/argotunnel.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv})
+}
+
 func (e *env) runnerLabel(testID string) string {
 	return "test-" + testID
 }
@@ -448,7 +593,15 @@ func (e *env) testJobs(testID string) []job {
 func (e *env) verifyActionsWorkflowRun(t *testing.T, testID string) {
 	t.Helper()
 
-	verifyActionsWorkflowRun(t, e.Env, e.testJobs(testID))
+	verifyActionsWorkflowRun(t, e.Env, e.testJobs(testID), e.verifyTimeout())
+}
+
+func (e *env) verifyTimeout() time.Duration {
+	if e.VerifyTimeout > 0 {
+		return e.VerifyTimeout
+	}
+
+	return 8 * 60 * time.Second
 }
 
 type job struct {
@@ -625,7 +778,7 @@ kubectl create cm %s$id --from-literal=status=ok
 	}
 }
 
-func verifyActionsWorkflowRun(t *testing.T, env *testing.Env, testJobs []job) {
+func verifyActionsWorkflowRun(t *testing.T, env *testing.Env, testJobs []job, timeout time.Duration) {
 	t.Helper()
 
 	var expected []string
@@ -643,7 +796,7 @@ func verifyActionsWorkflowRun(t *testing.T, env *testing.Env, testJobs []job) {
 			testResultCMName := testJobs[i].configMapName
 
 			kubectlEnv := []string{
-				"KUBECONFIG=" + env.Kubeconfig(),
+				"KUBECONFIG=" + env.Kubeconfig,
 			}
 
 			cmCfg := testing.KubectlConfig{
@@ -675,5 +828,5 @@ func verifyActionsWorkflowRun(t *testing.T, env *testing.Env, testJobs []job) {
 		}
 
 		return results, err
-	}, 8*60*time.Second, 30*time.Second).Should(gomega.Equal(expected))
+	}, timeout, 30*time.Second).Should(gomega.Equal(expected))
 }

testing/docker.go

@@ -71,3 +71,18 @@ func (k *Docker) dockerBuildCombinedOutput(ctx context.Context, build DockerBuil
 
 	return k.CombinedOutput(cmd)
 }
+
+func (k *Docker) Push(ctx context.Context, images []ContainerImage) error {
+	for _, img := range images {
+		_, err := k.CombinedOutput(dockerPushCmd(ctx, img.Repo, img.Tag))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func dockerPushCmd(ctx context.Context, repo, tag string) *exec.Cmd {
+	return exec.CommandContext(ctx, "docker", "push", repo+":"+tag)
+}

testing/testing.go

@@ -17,6 +17,12 @@ type T = testing.T
 
 var Short = testing.Short
 
+var images = map[string]string{
+	"1.22": "kindest/node:v1.22.9@sha256:8135260b959dfe320206eb36b3aeda9cffcb262f4b44cda6b33f7bb73f453105",
+	"1.23": "kindest/node:v1.23.6@sha256:b1fa224cc6c7ff32455e0b1fd9cbfd3d3bc87ecaa8fcb06961ed1afb3db0f9ae",
+	"1.24": "kindest/node:v1.24.0@sha256:0866296e693efe1fed79d5e6c7af8df71fc73ae45e3679af05342239cdc5bc8e",
+}
+
 func Img(repo, tag string) ContainerImage {
 	return ContainerImage{
 		Repo: repo,
@@ -28,21 +34,17 @@ func Img(repo, tag string) ContainerImage {
 // All of its methods are idempotent so that you can safely call it from within each subtest
 // and you can rerun the individual subtest until it works as you expect.
 type Env struct {
-	kind    *Kind
-	docker  *Docker
-	Kubectl *Kubectl
-	bash    *Bash
+	Kubeconfig string
+	docker     *Docker
+	Kubectl    *Kubectl
+	bash       *Bash
 }
 
-func Start(t *testing.T, opts ...Option) *Env {
+func Start(t *testing.T, k8sMinorVer string) *Env {
 	t.Helper()
 
-	k := StartKind(t, opts...)
-
 	var env Env
 
-	env.kind = k
-
 	d := &Docker{}
 
 	env.docker = d
@@ -59,12 +61,12 @@ func Start(t *testing.T, opts ...Option) *Env {
 }
 
 func (e *Env) GetOrGenerateTestID(t *testing.T) string {
-	k, kctl := e.kind, e.Kubectl
+	kctl := e.Kubectl
 
 	cmKey := "id"
 
 	kubectlEnv := []string{
-		"KUBECONFIG=" + k.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 	}
 
 	cmCfg := KubectlConfig{
@@ -89,10 +91,10 @@ func (e *Env) GetOrGenerateTestID(t *testing.T) string {
 }
 
 func (e *Env) DeleteTestID(t *testing.T) {
-	k, kctl := e.kind, e.Kubectl
+	kctl := e.Kubectl
 
 	kubectlEnv := []string{
-		"KUBECONFIG=" + k.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 	}
 
 	cmCfg := KubectlConfig{
@@ -119,13 +121,13 @@ func (e *Env) DockerBuild(t *testing.T, builds []DockerBuild) {
 	}
 }
 
-func (e *Env) KindLoadImages(t *testing.T, prebuildImages []ContainerImage) {
+func (e *Env) DockerPush(t *testing.T, images []ContainerImage) {
 	t.Helper()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
 	defer cancel()
 
-	if err := e.kind.LoadImages(ctx, prebuildImages); err != nil {
+	if err := e.docker.Push(ctx, images); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -137,7 +139,7 @@ func (e *Env) KubectlApply(t *testing.T, path string, cfg KubectlConfig) {
 	defer cancel()
 
 	kubectlEnv := []string{
-		"KUBECONFIG=" + e.kind.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 	}
 
 	cfg.Env = append(kubectlEnv, cfg.Env...)
@@ -154,7 +156,7 @@ func (e *Env) KubectlWaitUntilDeployAvailable(t *testing.T, name string, cfg Kub
 	defer cancel()
 
 	kubectlEnv := []string{
-		"KUBECONFIG=" + e.kind.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 	}
 
 	cfg.Env = append(kubectlEnv, cfg.Env...)
@@ -171,7 +173,7 @@ func (e *Env) KubectlEnsureNS(t *testing.T, name string, cfg KubectlConfig) {
 	defer cancel()
 
 	kubectlEnv := []string{
-		"KUBECONFIG=" + e.kind.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 	}
 
 	cfg.Env = append(kubectlEnv, cfg.Env...)
@@ -188,7 +190,7 @@ func (e *Env) KubectlEnsureClusterRoleBindingServiceAccount(t *testing.T, bindin
 	defer cancel()
 
 	kubectlEnv := []string{
-		"KUBECONFIG=" + e.kind.Kubeconfig(),
+		"KUBECONFIG=" + e.Kubeconfig,
 	}
 
 	cfg.Env = append(kubectlEnv, cfg.Env...)
@@ -200,10 +202,6 @@ func (e *Env) KubectlEnsureClusterRoleBindingServiceAccount(t *testing.T, bindin
 	}
 }
 
-func (e *Env) Kubeconfig() string {
-	return e.kind.Kubeconfig()
-}
-
 func (e *Env) RunScript(t *testing.T, path string, cfg ScriptConfig) {
 	t.Helper()
 
@@ -251,7 +249,7 @@ type ContainerImage struct {
 	Repo, Tag string
 }
 
-func StartKind(t *testing.T, opts ...Option) *Kind {
+func StartKind(t *testing.T, k8sMinorVer string, opts ...Option) *Kind {
 	t.Helper()
 
 	invalidChars := []string{"/"}
@@ -266,7 +264,7 @@ func StartKind(t *testing.T, opts ...Option) *Kind {
 	k.Dir = t.TempDir()
 
 	kk := &k
-	if err := kk.Start(context.Background()); err != nil {
+	if err := kk.Start(context.Background(), k8sMinorVer); err != nil {
 		t.Fatal(err)
 	}
 	t.Cleanup(func() {
@@ -323,7 +321,7 @@ func (k *Kind) Kubeconfig() string {
 	return k.kubeconfig
 }
 
-func (k *Kind) Start(ctx context.Context) error {
+func (k *Kind) Start(ctx context.Context, k8sMinorVer string) error {
 	getNodes, err := k.CombinedOutput(k.kindGetNodesCmd(ctx, k.Name))
 	if err != nil {
 		return err
@@ -337,6 +335,8 @@ func (k *Kind) Start(ctx context.Context) error {
 			return err
 		}
 
+		image := images[k8sMinorVer]
+
 		kindConfig := []byte(fmt.Sprintf(`kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 name: %s
@@ -344,8 +344,10 @@ networking:
   apiServerAddress: 0.0.0.0
 nodes:
   - role: control-plane
+    image: %s
   - role: worker
-`, k.Name))
+    image: %s
+`, k.Name, image, image))
 
 		if err := os.WriteFile(f.Name(), kindConfig, 0644); err != nil {
 			return err