Turn the bug report template into a form with more context (#1401)

I believe this helps us focus on relatively more important issues like critical bug reports and highly-requested feature requests.

Co-authored-by: Callum Tait <15716903+toast-gear@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,36 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**Checks**
-
-- [ ] My actions-runner-controller version (v0.x.y) does support the feature
-- [ ] I'm using an unreleased version of the controller I built from HEAD of the default branch
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Environment (please complete the following information):**
- - Controller Version [e.g. 0.18.2]
- - Deployment Method [e.g. Helm and Kustomize ]
- - Helm Chart Version [e.g. 0.11.0, if applicable]
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,160 @@
+name: Bug Report
+description: File a bug report
+title: "Bug"
+labels: ["bug"]
+body:
+- type: input
+  id: controller-version
+  attributes:
+    label: Controller Version
+    description: Refer to semver-like release tags for controller versions. Any release tags prefixed with `actions-runner-controller-` are for chart releases
+    placeholder: ex. 0.18.2 or git commit ID
+  validations:
+    required: true
+- type: input
+  id: chart-version
+  attributes:
+    label: Helm Chart Version
+    description: Run `helm list` and see what's shown under CHART VERSION. Any release tags prefixed with `actions-runner-controller-` are for chart releases
+    placeholder: ex. 0.11.0
+- type: dropdown
+  id: deployment-method
+  attributes:
+    label: Deployment Method
+    description: Which deployment method did you use to install ARC?
+    options:
+      - Helm
+      - Kustomize
+      - ArgoCD
+      - Other
+  validations:
+    required: true
+- type: checkboxes
+  id: checks
+  attributes:
+    label: Checks
+    description: Please check the boxes below before submitting
+    options:
+    - label: This isn't a question or user support case (For Q&A and community support, go to [Discussions](https://github.com/actions-runner-controller/actions-runner-controller/discussions). It might also be a good idea to contract with any of contributors and maintainers if your business is so critical and therefore you need priority support
+      required: true
+    - label: I've read [releasenotes](https://github.com/actions-runner-controller/actions-runner-controller/tree/master/docs/releasenotes) before submitting this issue and I'm sure it's not due to any recently-introduced backward-incompatible changes
+      required: true
+    - label: My actions-runner-controller version (v0.x.y) does support the feature
+      required: true
+    - label: I've already upgraded ARC to the latest and it didn't fix the issue
+      required: true
+- type: textarea
+  id: resource-definitions
+  attributes:
+    label: Resource Definitions
+    description: "Add copy(s) of your resource definition(s) (RunnerDeployment or RunnerSet, and HorizontalRunnerAutoscaler. If RunnerSet, also include the StorageClass being used)"
+    render: yaml
+    placeholder: |
+      apiVersion: actions.summerwind.dev/v1alpha1
+      kind: RunnerDeployment
+      metadata:
+        name: example
+      spec:
+        #snip
+      ---
+      apiVersion: actions.summerwind.dev/v1alpha1
+      kind: RunnerSet
+      metadata:
+        name: example
+      spec:
+        #snip
+      ---
+      apiVersion: storage.k8s.io/v1
+      kind: StorageClass
+      metadata:
+        name: example
+      provisioner: ...
+      reclaimPolicy: ...
+      volumeBindingMode: ...
+      ---
+      apiVersion: actions.summerwind.dev/v1alpha1
+      kind: HorizontalRunnerAutoscaler
+      metadata:
+        name:
+      spec:
+        #snip
+  validations:
+    required: true
+- type: textarea
+  id: reproduction-steps
+  attributes:
+    label: To Reproduce
+    description: "Steps to reproduce the behavior"
+    render: markdown
+    placeholder: |
+      1. Go to '...'
+      2. Click on '....'
+      3. Scroll down to '....'
+      4. See error
+  validations:
+    required: true
+- type: textarea
+  id: actual-behavior
+  attributes:
+    label: Describe the bug
+    description: Also tell us, what did happen?
+    placeholder: A clear and concise description of what happened.
+  validations:
+    required: true
+- type: textarea
+  id: expected-behavior
+  attributes:
+    label: Describe the expected behavior
+    description: Also tell us, what did you expect to happen?
+    placeholder: A clear and concise description of what the expected behavior is.
+  validations:
+    required: true
+- type: textarea
+  id: controller-logs
+  attributes:
+    label: Controller Logs
+    description: "Include logs from `actions-runner-controller`'s controller-manager pod"
+    render: shell
+    placeholder: |
+      To grab controller logs:
+
+      # Set NS according to your setup
+      NS=actions-runner-system
+
+      # Grab the pod name and set it to $POD_NAME
+      kubectl -n $NS get po
+
+      kubectl -n $NS logs $POD_NAME > arc.log
+
+      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
+  validations:
+    required: true
+- type: textarea
+  id: runner-pod-logs
+  attributes:
+    label: Runner Pod Logs
+    description: "Include logs from runner pod(s)"
+    render: shell
+    placeholder: |
+      To grab the runner pod logs:
+
+      # Set NS according to your setup. It should match your RunnerDeployment's metadata.namespace.
+      NS=default
+
+      # Grab the name of the problematic runner pod and set it to $POD_NAME
+      kubectl -n $NS get po
+
+      kubectl -n $NS logs $POD_NAME -c runner > runnerpod_runner.log
+      kubectl -n $NS logs $POD_NAME -c docker > runnerpod_docker.log
+
+      Upload it to e.g. https://gist.github.com/ and paste the link to it here.
+  validations:
+    required: true
+- type: textarea
+  id: additional-context
+  attributes:
+    label: Additional Context
+    description: |
+      Add any other context about the problem here.
+
+      Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,15 @@
+# Blank issues are mainly for maintainers who are known to write complete issue descriptions without need to following a form
+blank_issues_enabled: true
+contact_links:
+- name: Sponsor ARC Maintainers
+  about: If your business relies on the continued maintainance of actions-runner-controller, please consider sponsoring the project and the maintainers.
+  url: https://github.com/actions-runner-controller/actions-runner-controller/tree/master/CODEOWNERS
+- name: Ideas and Feature Requests
+  about: Wanna request a feature? Create a discussion and collect :+1:s first.
+  url: https://github.com/actions-runner-controller/actions-runner-controller/discussions/new?category=ideas
+- name: Questions and User Support
+  about: Need support using ARC? We use Discussions as the place to provide community support.
+  url: https://github.com/actions-runner-controller/actions-runner-controller/discussions/new?category=questions
+- name: Need Paid Support?
+  about: Consider contracting with any of the actions-runner-controller maintainers and contributors.
+  url: https://github.com/actions-runner-controller/actions-runner-controller/tree/master/CODEOWNERS

.github/renovate.json5

@@ -18,6 +18,24 @@
       "matchStrings": ["RUNNER_VERSION: +(?<currentValue>.*?)\\n"],
       "depNameTemplate": "actions/runner",
       "datasourceTemplate": "github-releases"
+    },
+    {
+      "fileMatch": [
+        "runner/Makefile",
+        "Makefile"
+      ],
+      "matchStrings": ["RUNNER_VERSION \\?= +(?<currentValue>.*?)\\n"],
+      "depNameTemplate": "actions/runner",
+      "datasourceTemplate": "github-releases"
+    },
+    {
+      "fileMatch": [
+        "runner/Dockerfile",
+        "runner/Dockerfile.dindrunner"
+      ],
+      "matchStrings": ["RUNNER_VERSION=+(?<currentValue>.*?)\\n"],
+      "depNameTemplate": "actions/runner",
+      "datasourceTemplate": "github-releases"
     }
   ]
 }

.github/stale.yml

@@ -1,67 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 14
-
-# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
-  - pinned
-  - security
-  - enhancement
-  - refactor
-  - documentation
-  - chore
-  - bug
-  - dependencies
-  - needs-investigation
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking as stale
-staleLabel: stale
-
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-
-# Comment to post when removing the stale label.
-# unmarkComment: >
-#   Your comment here.
-
-# Comment to post when closing a stale Issue or Pull Request.
-# closeComment: >
-#   Your comment here.
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-# only: issues
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed

.github/workflows/on-push-lint-charts.yml

@@ -23,7 +23,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.0
+        uses: azure/setup-helm@v2.1
         with:
           version: ${{ env.HELM_VERSION }}
 

.github/workflows/on-push-master-publish-chart.yml

@@ -28,7 +28,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v2.0
+        uses: azure/setup-helm@v2.1
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -114,7 +114,7 @@ jobs:
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.3.0
+        uses: helm/chart-releaser-action@v1.4.0
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '^1.17.7'
+          go-version: '1.17.7'
 
       - name: Install tools
         run: |

.github/workflows/runners.yml

@@ -11,11 +11,12 @@ on:
       - 'master'
     paths:
       - 'runner/**'
+      - '!runner/Makefile'
       - .github/workflows/runners.yml
       - '!**.md'
 
 env:
-  RUNNER_VERSION: 2.289.1
+  RUNNER_VERSION: 2.290.1
   DOCKER_VERSION: 20.10.12
   DOCKERHUB_USERNAME: summerwind
 

.github/workflows/stale.yaml

@@ -0,0 +1,19 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    # 01:30 every day
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v5
+        with:
+          stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
+          # turn off stale for both issues and PRs
+          days-before-stale: -1
+          # turn stale back on for issues only
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          exempt-issue-labels: 'pinned,security,enhancement,refactor,documentation,chore,bug,dependencies,needs-investigation'

.github/workflows/test.yaml

@@ -24,7 +24,8 @@ jobs:
       uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: '^1.17.7'
+        go-version: '1.17.7'
+        check-latest: false
     - run: go version
     - uses: actions/cache@v3
       with:

CODEOWNERS

@@ -0,0 +1,2 @@
+# actions-runner-controller maintainers
+* @mumoshu @toast-gear

Makefile

@@ -5,6 +5,7 @@ else
 endif
 DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
 VERSION ?= latest
+RUNNER_VERSION ?= 2.290.1
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}
@@ -12,7 +13,7 @@ TEST_REPO ?= ${DOCKER_USER}/actions-runner-controller
 TEST_ORG ?=
 TEST_ORG_REPO ?=
 TEST_EPHEMERAL ?= false
-SYNC_PERIOD ?= 5m
+SYNC_PERIOD ?= 1m
 USE_RUNNERSET ?=
 RUNNER_FEATURE_FLAG_EPHEMERAL ?=
 KUBECONTEXT ?= kind-acceptance
@@ -109,13 +110,9 @@ vet:
 generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
 
-# Build the docker image
-docker-build:
-	docker build -t ${NAME}:${VERSION} .
-	docker build -t ${RUNNER_NAME}:${RUNNER_TAG} --build-arg TARGETPLATFORM=${TARGETPLATFORM} runner
-
 docker-buildx:
-	export DOCKER_CLI_EXPERIMENTAL=enabled
+	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
+	export DOCKER_BUILDKIT=1
 	@if ! docker buildx ls | grep -q container-builder; then\
 		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
 	fi

README.md

@@ -1,4 +1,4 @@
-# actions-runner-controller
+# actions-runner-controller (ARC)
 
 [![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners)
 
@@ -6,7 +6,8 @@ This controller operates self-hosted runners for GitHub Actions on your Kubernet
 
 ToC:
 
-- [Motivation](#motivation)
+- [Status](#status)
+- [About](#about)
 - [Installation](#installation)
   - [GitHub Enterprise Support](#github-enterprise-support)
 - [Setting Up Authentication with GitHub API](#setting-up-authentication-with-github-api)
@@ -19,6 +20,7 @@ ToC:
   - [Enterprise Runners](#enterprise-runners)
   - [RunnerDeployments](#runnerdeployments)
   - [RunnerSets](#runnersets)
+  - [Persistent Runners](#persistent-runners)  
   - [Autoscaling](#autoscaling)
     - [Anti-Flapping Configuration](#anti-flapping-configuration)
     - [Pull Driven Scaling](#pull-driven-scaling)
@@ -32,25 +34,32 @@ ToC:
   - [Runner Groups](#runner-groups)
   - [Runner Entrypoint Features](#runner-entrypoint-features)
   - [Using IRSA (IAM Roles for Service Accounts) in EKS](#using-irsa-iam-roles-for-service-accounts-in-eks)
-  - [Persistent Runners](#persistent-runners)
   - [Software Installed in the Runner Image](#software-installed-in-the-runner-image)
   - [Using without cert-manager](#using-without-cert-manager)
 - [Troubleshooting](#troubleshooting)
 - [Contributing](#contributing)
 
-## Motivation
+
+## Status
+
+Even though actions-runner-controller is used in production environments, it is still in its early stage of development, hence versioned 0.x.
+
+actions-runner-controller complies to Semantic Versioning 2.0.0 in which v0.x means that there could be backward-incompatible changes for every release.
+
+The documentation is kept inline with master@HEAD, we do our best to highlight any features that require a specific ARC version or higher however this is not always easily done due to there being many moving parts. Additionally, we actively do not retain compatibly with every GitHub Enterprise Server version nor every Kubernetes version so you will need to ensure you stay current within a reasonable timespan.
+
+## About
 
 [GitHub Actions](https://github.com/features/actions) is a very useful tool for automating development. GitHub Actions jobs are run in the cloud by default, but you may want to run your jobs in your environment. [Self-hosted runner](https://github.com/actions/runner) can be used for such use cases, but requires the provisioning and configuration of a virtual machine instance. Instead if you already have a Kubernetes cluster, it makes more sense to run the self-hosted runner on top of it.
 
 **actions-runner-controller** makes that possible. Just create a *Runner* resource on your Kubernetes, and it will run and operate the self-hosted runner for the specified repository. Combined with Kubernetes RBAC, you can also build simple Self-hosted runners as a Service.
-
 ## Installation
 
-By default, actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) for certificate management of Admission Webhook. Make sure you have already installed cert-manager before you install. The installation instructions for cert-manager can be found below.
+By default, actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) for certificate management of Admission Webhook. Make sure you have already installed cert-manager before you install. The installation instructions for the cert-manager can be found below.
 
 - [Installing cert-manager on Kubernetes](https://cert-manager.io/docs/installation/kubernetes/)
 
-Subsequent to this, install the custom resource definitions and actions-runner-controller with `kubectl` or `helm`. This will create actions-runner-system namespace in your Kubernetes and deploy the required resources.
+Subsequent to this, install the custom resource definitions and actions-runner-controller with `kubectl` or `helm`. This will create an actions-runner-system namespace in your Kubernetes and deploy the required resources.
 
 **Kubectl Deployment:**
 
@@ -81,7 +90,7 @@ When deploying the solution for a GHES environment you need to provide an additi
 kubectl set env deploy controller-manager -c manager GITHUB_ENTERPRISE_URL=<GHEC/S URL> --namespace actions-runner-system
 ```
 
-**_Note: The repository maintainers do not have an enterprise environment (cloud or server). Support for the enterprise specific feature set is community driven and on a best effort basis. PRs from the community are welcomed to add features and maintain support._**
+**_Note: The repository maintainers do not have an enterprise environment (cloud or server). Support for the enterprise specific feature set is community driven and on a best effort basis. PRs from the community are welcome to add features and maintain support._**
 
 ## Setting Up Authentication with GitHub API
 
@@ -90,7 +99,7 @@ There are two ways for actions-runner-controller to authenticate with the GitHub
 1. Using a GitHub App (not supported for enterprise level runners due to lack of support from GitHub)
 2. Using a PAT
 
-Functionality wise, there isn't much of a difference between the 2 authentication methods. The primarily benefit of authenticating via a GitHub App is an [increased API quota](https://docs.github.com/en/developers/apps/rate-limits-for-github-apps).
+Functionality wise, there isn't much of a difference between the 2 authentication methods. The primary benefit of authenticating via a GitHub App is an [increased API quota](https://docs.github.com/en/developers/apps/rate-limits-for-github-apps).
 
 If you are deploying the solution for a GHES environment you are able to [configure your rate limit settings](https://docs.github.com/en/enterprise-server@3.0/admin/configuration/configuring-rate-limits) making the main benefit irrelevant. If you're deploying the solution for a GHEC or regular GitHub environment and you run into rate limit issues, consider deploying the solution using the GitHub App authentication method instead.
 
@@ -156,7 +165,7 @@ When the installation is complete, you will be taken to a URL in one of the foll
 - `https://github.com/organizations/eventreactor/settings/installations/${INSTALLATION_ID}`
 
 
-Finally, register the App ID (`APP_ID`), Installation ID (`INSTALLATION_ID`), and downloaded private key file (`PRIVATE_KEY_FILE_PATH`) to Kubernetes as Secret.
+Finally, register the App ID (`APP_ID`), Installation ID (`INSTALLATION_ID`), and the downloaded private key file (`PRIVATE_KEY_FILE_PATH`) to Kubernetes as a secret.
 
 **Kubectl Deployment:**
 
@@ -196,9 +205,9 @@ Log-in to a GitHub account that has `admin` privileges for the repository, and [
 
 * admin:enterprise (manage_runners:enterprise)
 
-_Note: When you deploy enterprise runners they will get access to organizations, however, access to the repositories themselves is **NOT** allowed by default. Each GitHub organization must allow enterprise runner groups to be used in repositories as an initial one time configuration step, this  only needs to be done once after which it is permanent for that runner group._
+_Note: When you deploy enterprise runners they will get access to organizations, however, access to the repositories themselves is **NOT** allowed by default. Each GitHub organization must allow enterprise runner groups to be used in repositories as an initial one-time configuration step, this only needs to be done once after which it is permanent for that runner group._
 
-_Note: GitHub do not document exactly what permissions you get with each PAT scope beyond a vague description. The best documentation they provide on the topic can be found [here](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps) if you wish to review. The docs target OAuth apps and so are incomplete and amy not be 100% accurate._ 
+_Note: GitHub does not document exactly what permissions you get with each PAT scope beyond a vague description. The best documentation they provide on the topic can be found [here](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps) if you wish to review. The docs target OAuth apps and so are incomplete and may not be 100% accurate._ 
 
 ---
 
@@ -220,7 +229,7 @@ Configure your values.yaml, see the chart's [README](./charts/actions-runner-con
 
 > This feature requires controller version => [v0.18.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.18.0)
 
-**_Note: Be aware when using this feature that CRDs are cluster wide and so you should upgrade all of your controllers (and your CRDs) as the same time if you are doing an upgrade. Do not mix and match CRD versions with different controller versions. Doing so risks out of control scaling._**
+**_Note: Be aware when using this feature that CRDs are cluster-wide and so you should upgrade all of your controllers (and your CRDs) at the same time if you are doing an upgrade. Do not mix and match CRD versions with different controller versions. Doing so risks out of control scaling._**
 
 By default the controller will look for runners in all namespaces, the watch namespace feature allows you to restrict the controller to monitoring a single namespace. This then lets you deploy multiple controllers in a single cluster. You may want to do this either because you wish to scale beyond the API rate limit of a single PAT / GitHub App configuration or you wish to support multiple GitHub organizations with runners installed at the organization level in a single cluster.
 
@@ -231,11 +240,11 @@ You can deploy multiple controllers either in a single shared namespace, or in a
 If you plan on installing all instances of the controller stack into a single namespace there are a few things you need to do for this to work.
 
 1. All resources per stack must have a unique, in the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
-2. `authSecret.name` needs be unique per stack when each stack is tied to runners in different GitHub organizations and repositories AND you want your GitHub credentials to narrowly scoped.
+2. `authSecret.name` needs to be unique per stack when each stack is tied to runners in different GitHub organizations and repositories AND you want your GitHub credentials to be narrowly scoped.
 3. `leaderElectionId` needs to be unique per stack. If this is not unique to the stack the controller tries to race onto the leader election lock resulting in only one stack working concurrently. Your controller will be stuck with a log message something like this `attempting to acquire leader lease arc-controllers/actions-runner-controller...`
-4. The MutatingWebhookConfiguration in each stack must include a namespace selector for that stacks corresponding runners namespace, this is already configured in the helm chart.
+4. The MutatingWebhookConfiguration in each stack must include a namespace selector for that stack's corresponding runner namespace, this is already configured in the helm chart.
 
-Alternatively, you can install each controller stack into a unique namespace (relative to other controller stacks in the cluster), avoiding these potential pitfalls.
+Alternatively, you can install each controller stack into a unique namespace (relative to other controller stacks in the cluster). Implementing ARC this way avoids the first, second and third pitfalls (you still need to set the corresponding namespace selector for each stack's mutating webhook)
 
 ## Usage
 
@@ -251,7 +260,7 @@ There are two ways to use this controller:
 
 ### Repository Runners
 
-To launch a single self-hosted runner, you need to create a manifest file includes `Runner` resource as follows. This example launches a self-hosted runner with name *example-runner* for the *actions-runner-controller/actions-runner-controller* repository.
+To launch a single self-hosted runner, you need to create a manifest file that includes a `Runner` resource as follows. This example launches a self-hosted runner with name *example-runner* for the *actions-runner-controller/actions-runner-controller* repository.
 
 ```yaml
 # runner.yaml
@@ -367,7 +376,9 @@ example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 
 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
 
-For scenarios where you require the advantages of a `StatefulSet`, for example persistent storage, ARC implements a runner based on Kubernete's StatefulSets, the RunnerSet.
+_Ensure you see the limitations before using this kind!!!!!_
+
+For scenarios where you require the advantages of a `StatefulSet`, for example persistent storage, ARC implements a runner based on Kubernetes' `StatefulSets`, the `RunnerSet`.
 
 A basic `RunnerSet` would look like this:
 
@@ -391,7 +402,7 @@ spec:
         app: example
 ```
 
-As it is based on `StatefulSet`, `selector` and `template.medatada.labels` needs to be defined and have the exact same set of labels. `serviceName` must be set to some non-empty string as it is also required by `StatefulSet`.
+As it is based on `StatefulSet`, `selector` and `template.medatada.labels` it needs to be defined and have the exact same set of labels. `serviceName` must be set to some non-empty string as it is also required by `StatefulSet`.
 
 Runner-related fields like `ephemeral`, `repository`, `organization`, `enterprise`, and so on should be written directly under `spec`.
 
@@ -399,7 +410,7 @@ Fields like `volumeClaimTemplates` that originates from `StatefulSet` should als
 
 Pod-related fields like security contexts and volumes are written under `spec.template.spec` like `StatefulSet`.
 
-Similarly, container-related fields like resource requests and limits, container image names and tags, security context, and so on are written under `spec.template.spec.containers`. There are two reserved container `name`, `runner` and `docker`. The former is for the container that runs [actions runner](https://github.com/actions/runner) and the latter is for the container that runs a dockerd.
+Similarly, container-related fields like resource requests and limits, container image names and tags, security context, and so on are written under `spec.template.spec.containers`. There are two reserved container `name`, `runner` and `docker`. The former is for the container that runs [actions runner](https://github.com/actions/runner) and the latter is for the container that runs a `dockerd`.
 
 For a more complex example, see the below:
 
@@ -445,29 +456,49 @@ spec:
 
 You can also read the design and usage documentation written in the original pull request that introduced `RunnerSet` for more information [#629](https://github.com/actions-runner-controller/actions-runner-controller/pull/629).
 
-Under the hood, `RunnerSet` relies on Kubernetes's `StatefulSet` and Mutating Webhook. A statefulset is used to create a number of pods that has stable names and dynamically provisioned persistent volumes, so that each statefulset-managed pod gets the same persistent volume even after restarting. A mutating webhook is used to dynamically inject a runner's "registration token" which is used to call GitHub's "Create Runner" API.
+Under the hood, `RunnerSet` relies on Kubernetes's `StatefulSet` and Mutating Webhook. A `statefulset` is used to create a number of pods that has stable names and dynamically provisioned persistent volumes, so that each `statefulset-managed` pod gets the same persistent volume even after restarting. A mutating webhook is used to dynamically inject a runner's "registration token" which is used to call GitHub's "Create Runner" API.
 
 **Limitations**
 
 * For autoscaling the `RunnerSet` kind only supports pull driven scaling or the `workflow_job` event for webhook driven scaling.
+* Whilst `RunnerSets` support all runner modes as well as autoscaling, currently PVs are **NOT** automatically cleaned up as they are still bound to their respective PVCs when a runner is deleted by the controller. This has **major** implications when using `RunnerSets` in the standard runner mode, `ephemeral: true`, see [persistent runners](#persistent-runners) for more details. As a result of this, using the default ephemeral configuration or implementing autoscaling for your `RunnerSets`, you will get a build-up of PVCs and PVs without some sort of custom solution for cleaning up.
+
+### Persistent Runners
+
+Every runner managed by ARC is "ephemeral" by default. The life of an ephemeral runner managed by ARC looks like this- ARC creates a runner pod for the runner. As it's an ephemeral runner, the `--ephemeral` flag is passed to the `actions/runner` agent that runs within the `runner` container of the runner pod.
+
+`--ephemeral` is an `actions/runner` feature that instructs the runner to stop and de-register itself after the first job run.
+
+Once the ephemeral runner has completed running a workflow job, it stops with a status code of 0, hence the runner pod is marked as completed, removed by ARC.
+
+As it's removed after a workflow job run, the runner pod is never reused across multiple GitHub Actions workflow jobs, providing you a clean environment per each workflow job.
+
+Although not generally recommended, it's possible to disable the passing of the `--ephemeral` flag by explicitly setting `ephemeral: false` in the `RunnerDeployment` or `RunnerSet` spec. When disabled, your runner becomes "persistent". A persistent runner does not stop after workflow job ends, and in this mode `actions/runner` is known to clean only runner's work dir after each job. Whilst this can seem helpful it creates a non-deterministic environment which is not ideal for a CI/CD environment. Between runs, your actions cache, docker images stored in the `dind` and layer cache, globally installed packages etc are retained across multiple workflow job runs which can cause issues that are hard to debug and inconsistent.
+
+Persistent runners are available as an option for some edge cases however they are not preferred as they can create challenges around providing a deterministic and secure environment.
 
 ### Autoscaling
 
-> Since the release of GitHub's [`workflow_job` webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job), webhook driven scaling is the preferred way of autoscaling as it enables targeted scaling of your `RunnerDeployment` / `RunnerSet` as it includes the `runs-on` information needed to scale the appropriate runners for that workflow run. More broadly, webhook driven scaling is the preferred scaling option as it is far quicker compared to the pull driven scaling and is easy to setup.
+> Since the release of GitHub's [`workflow_job` webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job), webhook driven scaling is the preferred way of autoscaling as it enables targeted scaling of your `RunnerDeployment` / `RunnerSet` as it includes the `runs-on` information needed to scale the appropriate runners for that workflow run. More broadly, webhook driven scaling is the preferred scaling option as it is far quicker compared to the pull driven scaling and is easy to set up.
 
 > If you are using controller version < [v0.22.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.22.0) and you are not using GHES, and so can't set your rate limit budget, it is recommended that you use 100 replicas or fewer to prevent being rate limited.
 
-A `RunnerDeployment` or `RunnerSet` can scale the number of runners between `minReplicas` and `maxReplicas` fields driven by either pull based scaling metrics or via a webhook event (see limitations section of [stateful runners](#stateful-runners) for cavaets of this kind). Whether the autoscaling is driven from a webhook event or pull based metrics it is implemented by backing a `RunnerDeployment` or `RunnerSet` kind with a `HorizontalRunnerAutoscaler` kind.
+A `RunnerDeployment` or `RunnerSet` can scale the number of runners between `minReplicas` and `maxReplicas` fields driven by either pull based scaling metrics or via a webhook event (see limitations section of [RunnerSets](#runnersets) for caveats of this kind). Whether the autoscaling is driven from a webhook event or pull based metrics it is implemented by backing a `RunnerDeployment` or `RunnerSet` kind with a `HorizontalRunnerAutoscaler` kind.
 
 **_Important!!! If you opt to configure autoscaling, ensure you remove the `replicas:` attribute in the `RunnerDeployment` / `RunnerSet` kinds that are configured for autoscaling [#206](https://github.com/actions-runner-controller/actions-runner-controller/issues/206#issuecomment-748601907)_**
 
 #### Anti-Flapping Configuration
 
-For both pull driven or webhook driven scaling an anti-flapping implementation is included, by default a runner won't be scaled down within 10 minutes of it having been scaled up. This delay is configurable by including the attribute `scaleDownDelaySecondsAfterScaleOut:` in a `HorizontalRunnerAutoscaler` kind's `spec:`.
+For both pull driven or webhook driven scaling an anti-flapping implementation is included, by default a runner won't be scaled down within 10 minutes of it having been scaled up. 
 
-This configuration has the final say on if a runner can be scaled down or not regardless of the chosen scaling method. Depending on your requirements, you may want to consider adjusting this by setting the `scaleDownDelaySecondsAfterScaleOut:` attribute.
+This anti-flap configuration also has the final say on if a runner can be scaled down or not regardless of the chosen scaling method.
 
-Below is a complete basic example with one of the pull driven scaling metrics.
+This delay is configurable via 2 methods:
+
+1. By setting a new default via the controller's `--default-scale-down-delay` flag
+2. By setting by setting the attribute `scaleDownDelaySecondsAfterScaleOut:` in a `HorizontalRunnerAutoscaler` kind's `spec:`.
+
+Below is a complete basic example of one of the pull driven scaling metrics.
 
 ```yaml
 apiVersion: actions.summerwind.dev/v1alpha1
@@ -505,7 +536,9 @@ spec:
 
 > To configure webhook driven scaling see the [Webhook Driven Scaling](#webhook-driven-scaling) section
 
-The pull based metrics are configured in the `metrics` attribute of a HRA (see snippet below). The period between polls is defined by the controller's `--sync-period` flag. If this flag isn't provided then the controller defaults to a sync period of 10 minutes. The default value is set to 10 minutes to prevent default deployments rate limiting themselves from the GitHub API, you will most likely want to adjust this.
+The pull based metrics are configured in the `metrics` attribute of a HRA (see snippet below). The period between polls is defined by the controller's `--sync-period` flag. If this flag isn't provided then the controller defaults to a sync period of `1m`, this can be configured in seconds or minutes. 
+
+Be aware that the shorter the sync period the quicker you will consume your rate limit budget, depending on your environment this may or may not be a risk. Consider monitoring ARCs rate limit budget when configuring this feature to find the optimal performance sync period.
 
 ```yaml
 apiVersion: actions.summerwind.dev/v1alpha1
@@ -532,14 +565,13 @@ The `TotalNumberOfQueuedAndInProgressWorkflowRuns` metric polls GitHub for all p
 
 **Benefits of this metric**
 1. Supports named repositories allowing you to restrict the runner to a specified set of repositories server-side.
-2. Scales the runner count based on the depth of the job queue meaning a more 1:1 scaling of runners to queued jobs (caveat, see drawback #4)
+2. Scales the runner count based on the depth of the job queue meaning a 1:1 scaling of runners to queued jobs.
 3. Like all scaling metrics, you can manage workflow allocation to the RunnerDeployment through the use of [GitHub labels](#runner-labels).
 
 **Drawbacks of this metric**
 1. A list of repositories must be included within the scaling metric. Maintaining a list of repositories may not be viable in larger environments or self-serve environments.
-2. May not scale quick enough for some users needs. This metric is pull based and so the queue depth is polled as configured by the sync period, as a result scaling performance is bound by this sync period meaning there is a lag to scaling activity.
+2. May not scale quickly enough for some users' needs. This metric is pull based and so the queue depth is polled as configured by the sync period, as a result scaling performance is bound by this sync period meaning there is a lag to scaling activity.
-3. Relatively large amounts of API requests required to maintain this metric, you may run in API rate limit issues depending on the size of your environment and how aggressive your sync period configuration is.
+3. Relatively large amounts of API requests are required to maintain this metric, you may run into API rate limit issues depending on the size of your environment and how aggressive your sync period configuration is.
-4. The GitHub API doesn't provide a way to filter workflow jobs to just those targeting self-hosted runners. If your environment's workflows target both self-hosted and GitHub hosted runners then the queue depth this metric scales against isn't a true 1:1 mapping of queue depth to required runner count. As a result of this, this metric may scale too aggressively for your actual self-hosted runner count needs.
 
 Example `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`:
 
@@ -581,7 +613,7 @@ The `HorizontalRunnerAutoscaler` will poll GitHub for the number of runners in t
 4. Supports scaling desired runner count on both a percentage increase / decrease basis as well as on a fixed increase / decrease count basis [#223](https://github.com/actions-runner-controller/actions-runner-controller/pull/223) [#315](https://github.com/actions-runner-controller/actions-runner-controller/pull/315)
 
 **Drawbacks of this metric**
-1. May not scale quick enough for some users needs. This metric is pull based and so the number of busy runners are polled as configured by the sync period, as a result scaling performance is bound by this sync period meaning there is a lag to scaling activity.
+1. May not scale quickly enough for some users' needs. This metric is pull based and so the number of busy runners is polled as configured by the sync period, as a result scaling performance is bound by this sync period meaning there is a lag to scaling activity.
 2. We are scaling up and down based on indicative information rather than a count of the actual number of queued jobs and so the desired runner count is likely to under provision new runners or overprovision them relative to actual job queue depth, this may or may not be a problem for you.
 
 Examples of each scaling type implemented with a `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`:
@@ -632,10 +664,10 @@ spec:
 
 > To configure pull driven scaling see the [Pull Driven Scaling](#pull-driven-scaling) section
 
-Webhooks are processed by a seperate webhook server. The webhook server receives GitHub Webhook events and scales
+Webhooks are processed by a separate webhook server. The webhook server receives GitHub Webhook events and scales
 [`RunnerDeployments`](#runnerdeployments) by updating corresponding [`HorizontalRunnerAutoscalers`](#autoscaling).
 
-Today, the Webhook server can be configured to respond GitHub `check_run`, `workflow_job`, `pull_request` and `push`  events
+Today, the Webhook server can be configured to respond to GitHub's `check_run`, `workflow_job`, `pull_request`, and `push`  events
 by scaling up the matching `HorizontalRunnerAutoscaler` by N replica(s), where `N` is configurable within `HorizontalRunnerAutoscaler`'s `spec:`.
 
 More concretely, you can configure the targeted GitHub event types and the `N` in `scaleUpTriggers`:
@@ -658,17 +690,17 @@ spec:
 
 With the above example, the webhook server scales `example-runners` by `1` replica for 5 minutes on each `check_run` event with the type of `created` and the status of `queued` received.
 
-Of note is the `HRA.spec.scaleUpTriggers[].duration` attribute. This attribute is used to calculate if the replica number added via the trigger is expired or not. On each reconcilation loop, the controller sums up all the non-expiring replica numbers from previous scale up triggers. It then compares the summed desired replica number against the current replica number. If the summed desired replica number > the current number then it means the replica count needs to scale up.
+Of note is the `HRA.spec.scaleUpTriggers[].duration` attribute. This attribute is used to calculate if the replica number added via the trigger is expired or not. On each reconciliation loop, the controller sums up all the non-expiring replica numbers from previous scale-up triggers. It then compares the summed desired replica number against the current replica number. If the summed desired replica number > the current number then it means the replica count needs to scale up.
 
 As mentioned previously, the `scaleDownDelaySecondsAfterScaleOut` property has the final say still. If the latest scale-up time + the anti-flapping duration is later than the current time, it doesn’t immediately scale up and instead retries the calculation again later to see if it needs to scale yet.
 
 ---
 
-The primary benefit of autoscaling on Webhook compared to the pull driven scaling is that it is far quicker as it allows you to immediately add runners resource rather than waiting for the next sync period.
+The primary benefit of autoscaling on Webhooks compared to the pull driven scaling is that it is far quicker as it allows you to immediately add runner resources rather than waiting for the next sync period.
 
 > You can learn the implementation details in [#282](https://github.com/actions-runner-controller/actions-runner-controller/pull/282)
 
-To enable this feature, you firstly need to install the webhook server, currently, only our Helm chart has the ability install it:
+To enable this feature, you first need to install the GitHub webhook server. To install via our Helm chart,
 _[see the values documentation for all configuration options](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/charts/actions-runner-controller/README.md)_
 
 ```console
@@ -678,7 +710,7 @@ $ helm upgrade --install --namespace actions-runner-system --create-namespace \
 ```
 
 The above command will result in exposing the node port 33080 for Webhook events. Usually, you need to create an
-external loadbalancer targeted to the node port, and register the hostname or the IP address of the external loadbalancer
+external load balancer targeted to the node port, and register the hostname or the IP address of the external load balancer
 to the GitHub Webhook.
 
 Once you were able to confirm that the Webhook server is ready and running from GitHub - this is usually verified by the
@@ -690,13 +722,13 @@ by learning the following configuration examples.
 - [Example 3: Scale on each `pull_request` event against a given set of branches](#example-3-scale-on-each-pull_request-event-against-a-given-set-of-branches)
 - [Example 4: Scale on each `push` event](#example-4-scale-on-each-push-event)
 
-**Note:** All these examples should have **minReplicas** & **maxReplicas** as mandatory parameter even for webhook driven scaling. 
+**Note:** All these examples should have **minReplicas** & **maxReplicas** as mandatory parameters even for webhook driven scaling. 
 
 ##### Example 1: Scale on each `workflow_job` event
 
 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
 
-_Note: GitHub does not include the runner group information of a repository in the payload of `workflow_job` event in the initial `queued` event. The runner group information is only include for `workflow_job` events when the job has already been allocated to a runner (events with a status of `in_progress` or `completed`). Please do raise feature requests against [GitHub](https://support.github.com/tickets/personal/0) for this information to be included in the initial `queued` event if this would improve autoscaling runners for you._
+_Note: GitHub does not include the runner group information of a repository in the payload of `workflow_job` event in the initial `queued` event. The runner group information is only included for `workflow_job` events when the job has already been allocated to a runner (events with a status of `in_progress` or `completed`). Please do raise feature requests against [GitHub](https://support.github.com/tickets/personal/0) for this information to be included in the initial `queued` event if this would improve autoscaling runners for you._
 
 The most flexible webhook GitHub offers is the `workflow_job` webhook, it includes the `runs-on` information in the payload allowing scaling based on runner labels.
 
@@ -718,7 +750,8 @@ spec:
     # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
     #kind: RunnerSet
   scaleUpTriggers:
-  - githubEvent: {}
+  - githubEvent:
+      workflowJob: {}
     duration: "30m"
 ```
 
@@ -726,7 +759,7 @@ This webhook requires you to explicitly set the labels in the RunnerDeployment /
 
 You can configure your GitHub webhook settings to only include `Workflows Job` events, so that it sends us three kinds of `workflow_job` events per a job run.
 
-Each kind has a `status` of `queued`, `in_progress` and `completed`. With the above configuration, `actions-runner-controller` adds one runner for a `workflow_job` event whose `status` is `queued`. Similarly, it removes one runner for a `workflow_job` event whose `status` is `completed`. The cavaet to this to remember is that this the scale down is within the bounds of your `scaleDownDelaySecondsAfterScaleOut` configuration, if this time hasn't past the scale down will be defered.
+Each kind has a `status` of `queued`, `in_progress` and `completed`. With the above configuration, `actions-runner-controller` adds one runner for a `workflow_job` event whose `status` is `queued`. Similarly, it removes one runner for a `workflow_job` event whose `status` is `completed`. The caveat to this to remember is that this scale-down is within the bounds of your `scaleDownDelaySecondsAfterScaleOut` configuration, if this time hasn't passed the scale down will be deferred.
 
 ##### Example 2: Scale up on each `check_run` event
 
@@ -853,17 +886,17 @@ The main use case for scaling from 0 is with the `HorizontalRunnerAutoscaler` ki
 - `PercentageRunnersBusy` + Webhook-based autoscaling
 - Webhook-based autoscaling only
 
-`PercentageRunnersBusy` can't be used alone as, by its definition, it needs one or more GitHub runners to become `busy` to be able to scale. If there isn't a runner to pick up a job and enter a `busy` state then the controller will never know to provision a runner to begin with as this metric has no knowledge of the job queue and is relying using the number of busy runners as a means for calculating the desired replica count.
+`PercentageRunnersBusy` can't be used alone as, by its definition, it needs one or more GitHub runners to become `busy` to be able to scale. If there isn't a runner to pick up a job and enter a `busy` state then the controller will never know to provision a runner to begin with as this metric has no knowledge of the job queue and is relying on using the number of busy runners as a means for calculating the desired replica count.
 
 If a HorizontalRunnerAutoscaler is configured with a secondary metric of `TotalNumberOfQueuedAndInProgressWorkflowRuns` then be aware that the controller will check the primary metric of `PercentageRunnersBusy` first and will only use the secondary metric to calculate the desired replica count if the primary metric returns 0 desired replicas.
 
-Webhook-based autoscaling is the best option as it is relatively easy to configure and also it can scale scale quickly.
+Webhook-based autoscaling is the best option as it is relatively easy to configure and also it can scale quickly.
 
 #### Scheduled Overrides
 
 > This feature requires controller version => [v0.19.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.19.0)
 
-`Scheduled Overrides` allows you to configure `HorizontalRunnerAutoscaler` so that its `spec:` gets updated only during a certain period of time. This feature is usually used for following scenarios:
+`Scheduled Overrides` allows you to configure `HorizontalRunnerAutoscaler` so that its `spec:` gets updated only during a certain period of time. This feature is usually used for the following scenarios:
 
 - You want to reduce your infrastructure costs by scaling your Kubernetes nodes down outside a given period
 - You want to scale for scheduled spikes in workloads
@@ -914,7 +947,7 @@ spec:
   minReplicas: 1
 ```
 
- A recurring override is initially active between `startTime` and `endTime`, and then it repeatedly get activated after a certain period of time denoted by `frequency`.
+ A recurring override is initially active between `startTime` and `endTime`, and then it repeatedly gets activated after a certain period of time denoted by `frequency`.
 
 `frequecy` can take one of the following values:
 
@@ -923,21 +956,21 @@ spec:
 - `Monthly`
 - `Yearly`
 
-By default, a scheduled override repeats forever. If you want it to repeat until a specific point in time, define `untilTime`. The controller create the last recurrence of the override until the recurrence's `startTime` is equal or earlier than `untilTime`.
+By default, a scheduled override repeats forever. If you want it to repeat until a specific point in time, define `untilTime`. The controller creates the last recurrence of the override until the recurrence's `startTime` is equal or earlier than `untilTime`.
 
 Do ensure that you have enough slack for `untilTime` so that a delayed or offline `actions-runner-controller` is much less likely to miss the last recurrence. For example, you might want to set `untilTime` to `M` minutes after the last recurrence's `startTime`, so that `actions-runner-controller` being offline up to `M` minutes doesn't miss the last recurrence.
 
 **Combining Multiple Scheduled Overrides**:
 
-In case you have a more complex scenarios, try writing two or more entries under `scheduledOverrides`.
+In case you have a more complex scenario, try writing two or more entries under `scheduledOverrides`.
 
-The earlier entry is prioritized higher than later entries. So you usually define one-time overrides in the top of your list, then yearly, monthly, weekly, and lastly daily overrides.
+The earlier entry is prioritized higher than later entries. So you usually define one-time overrides at the top of your list, then yearly, monthly, weekly, and lastly daily overrides.
 
 A common use case for this may be to have 1 override to scale to 0 during the week outside of core business hours and another override to scale to 0 during all hours of the weekend.
 
 ### Runner with DinD
 
-When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
+When using the default runner, the runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
 
 ```yaml
 # dindrunnerdeployment.yaml
@@ -1042,7 +1075,7 @@ spec:
       # false (default) = Docker support is provided by a sidecar container deployed in the runner pod.
       # true = No docker sidecar container is deployed in the runner pod but docker can be used within the runner container instead. The image summerwind/actions-runner-dind is used by default.
       dockerdWithinRunnerContainer: true
-      #Optional environement variables for docker container
+      #Optional environment variables for docker container
       # Valid only when dockerdWithinRunnerContainer=false
       dockerEnv:
         - name: HTTP_PROXY
@@ -1084,7 +1117,7 @@ spec:
         - mountPath: /var/lib/docker
           name: docker-extra
       # You can mount some of the shared volumes to the runner container using volumeMounts.
-      # NOTE: Do not try to mount the volume onto the runner workdir itself as it will not work. You could mount it however on a sub directory in the runner workdir
+      # NOTE: Do not try to mount the volume onto the runner workdir itself as it will not work. You could mount it however on a subdirectory in the runner workdir
       # Please see https://github.com/actions-runner-controller/actions-runner-controller/issues/630#issuecomment-862087323 for more information.
       volumeMounts:
         - mountPath: /home/runner/work/repo
@@ -1108,7 +1141,7 @@ spec:
 ```
 
 ### Custom Volume mounts
-You can configure your own custom volume mounts. For example to have the work/docker data in memory or on NVME ssd, for
+You can configure your own custom volume mounts. For example to have the work/docker data in memory or on NVME SSD, for
 i/o intensive builds. Other custom volume mounts should be possible as well, see [kubernetes documentation](https://kubernetes.io/docs/concepts/storage/volumes/)
 
 **RAM Disk Runner**<br />
@@ -1139,9 +1172,9 @@ spec:
 
 **NVME SSD Runner**<br />
 In this example we provide NVME backed storage for the workdir, docker sidecar and /tmp within the runner.
-Here we use a working example on GKE, which will provide the NVME disk at /mnt/disks/ssd0.  We will be placing the respective volumes in subdirs here and in order to be able to run multiple runners we will use the pod name as prefix for subdirectories. Also the disk will fill up over time and disk space will not be freed until the node is removed.
+Here we use a working example on GKE, which will provide the NVME disk at /mnt/disks/ssd0.  We will be placing the respective volumes in subdirs here and in order to be able to run multiple runners we will use the pod name as a prefix for subdirectories. Also the disk will fill up over time and disk space will not be freed until the node is removed.
 
-**Beware** that running these persistent backend volumes **leave data behind** between 2 different jobs on the workdir and /tmp with emphemeral: false.
+**Beware** that running these persistent backend volumes **leave data behind** between 2 different jobs on the workdir and `/tmp` with `emphemeral: false`.
 
 ```yaml
 kind: RunnerDeployment
@@ -1241,11 +1274,11 @@ spec:
       group: NewGroup
 ```
 
-GitHub supports custom visilibity in a Runner Group to make it available to a specific set of repositories only. By default if no GitHub
+GitHub supports custom visibility in a Runner Group to make it available to a specific set of repositories only. By default if no GitHub
 authentication is included in the webhook server ARC will be assumed that all runner groups to be usable in all repositories.
-Currently, GitHub do not include the repository runner group membership information in the workflow_job event (or any webhook). To make the ARC "runner group aware" additional GitHub API calls are needed to find out what runner groups are visible to the webhook's repository. This behaviour will impact your rate-limit budget and so the option needs to be explicitly configured by the end user.
+Currently, GitHub does not include the repository runner group membership information in the workflow_job event (or any webhook). To make the ARC "runner group aware" additional GitHub API calls are needed to find out what runner groups are visible to the webhook's repository. This behaviour will impact your rate-limit budget and so the option needs to be explicitly configured by the end user.
 
-This option will be enabled when proper GitHub authentication options (token, app or basic auth) is provided in the webhook server and `useRunnerGroupsVisibility` is set to true, e.g.
+This option will be enabled when proper GitHub authentication options (token, app or basic auth) are provided in the webhook server and `useRunnerGroupsVisibility` is set to true, e.g.
 
 ```yaml
 githubWebhookServer:
@@ -1291,10 +1324,10 @@ spec:
 
 > This feature requires controller version => [v0.15.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.15.0)
 
-As similar as for regular pods and deployments, you firstly need an existing service account with the IAM role associated.
+Similar to regular pods and deployments, you firstly need an existing service account with the IAM role associated.
 Create one using e.g. `eksctl`. You can refer to [the EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) for more details.
 
-Once you set up the service account, all you need is to add `serviceAccountName` and `fsGroup` to any pods that uses the IAM-role enabled service account.
+Once you set up the service account, all you need is to add `serviceAccountName` and `fsGroup` to any pods that use the IAM-role enabled service account.
 
 For `RunnerDeployment`, you can set those two fields under the runner spec at `RunnerDeployment.Spec.Template`:
 
@@ -1311,28 +1344,13 @@ spec:
       securityContext:
         fsGroup: 1000
 ```
-
-### Persistent Runners
-
-Every runner managed by ARC is "ephemeral" by default. The life of an ephemeral runner managed by ARC looks like this- ARC creates a runner pod for the runner. As it's an ephemeral runner, the `--ephemeral` flag is passed to the `actions/runner` agent that runs within the `runner` container of the runner pod.
-
-`--ephemeral` is an `actions/runner` feature that instructs the runner to stop and de-register itself after the first job run.
-
-Once the ephemeral runner has completed running a workflow job, it stops with a status code of 0, hence the runner pod is marked as completed, removed by ARC.
-
-As it's removed after a workflow job run, the runner pod is never reused across multiple GitHub Actions workflow jobs, providing you a clean environment per each workflow job.
-
-Although not recommended, it's possible to disable passing `--ephemeral` flag by explicitly setting `ephemeral: false` in the `RunnerDeployment` or `RunnerSet` spec. When disabled, your runner becomes "persistent". A persistent runner does not stop after workflow job ends, and in this mode `actions/runner` is known to clean only runner's work dir after each job. That means your runner's environment, including various actions cache, docker images stored in the `dind` and layer cache, is retained across multiple workflow job runs.
-
-Persistent runners are available as an option for some edge cases however they are not preferred as they can create challenges around providing a deterministic and secure environment.
-
 ### Software Installed in the Runner Image
 
 **Cloud Tooling**<br />
 The project supports being deployed on the various cloud Kubernetes platforms (e.g. EKS), it does not however aim to go beyond that. No cloud specific tooling is bundled in the base runner, this is an active decision to keep the overhead of maintaining the solution manageable.
 
 **Bundled Software**<br />
-The GitHub hosted runners include a large amount of pre-installed software packages. GitHub maintain a list in README files at <https://github.com/actions/virtual-environments/tree/main/images/linux>
+The GitHub hosted runners include a large amount of pre-installed software packages. GitHub maintains a list in README files at <https://github.com/actions/virtual-environments/tree/main/images/linux>
 
 This solution maintains a few runner images with `latest` aligning with GitHub's Ubuntu version, these images do not contain all of the software installed on the GitHub runners. The images contain the following subset of packages from the GitHub runners:
 
@@ -1343,7 +1361,7 @@ This solution maintains a few runner images with `latest` aligning with GitHub's
 
 The virtual environments from GitHub contain a lot more software packages (different versions of Java, Node.js, Golang, .NET, etc) which are not provided in the runner image. Most of these have dedicated setup actions which allow the tools to be installed on-demand in a workflow, for example: `actions/setup-java` or `actions/setup-node`
 
-If there is a need to include packages in the runner image for which there is no setup action, then this can be achieved by building a custom container image for the runner. The easiest way is to start with the `summerwind/actions-runner` image and installing the extra dependencies directly in the docker image:
+If there is a need to include packages in the runner image for which there is no setup action, then this can be achieved by building a custom container image for the runner. The easiest way is to start with the `summerwind/actions-runner` image and then install the extra dependencies directly in the docker image:
 
 ```shell
 FROM summerwind/actions-runner:latest
@@ -1396,7 +1414,7 @@ $ helm --upgrade install actions-runner-controller/actions-runner-controller \
 
 # Troubleshooting
 
-See [troubleshooting guide](TROUBLESHOOTING.md) for solutions to various problems people have ran into consistently.
+See [troubleshooting guide](TROUBLESHOOTING.md) for solutions to various problems people have run into consistently.
 
 # Contributing
 

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -72,10 +72,12 @@ type GitHubEventScaleUpTriggerSpec struct {
 	CheckRun    *CheckRunSpec    `json:"checkRun,omitempty"`
 	PullRequest *PullRequestSpec `json:"pullRequest,omitempty"`
 	Push        *PushSpec        `json:"push,omitempty"`
+	WorkflowJob *WorkflowJobSpec `json:"workflowJob,omitempty"`
 }
 
 // https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run
 type CheckRunSpec struct {
+	// One of: created, rerequested, or completed
 	Types  []string `json:"types,omitempty"`
 	Status string   `json:"status,omitempty"`
 
@@ -90,6 +92,10 @@ type CheckRunSpec struct {
 	Repositories []string `json:"repositories,omitempty"`
 }
 
+// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job
+type WorkflowJobSpec struct {
+}
+
 // https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
 type PullRequestSpec struct {
 	Types    []string `json:"types,omitempty"`

api/v1alpha1/runner_types.go

@@ -145,7 +145,7 @@ type RunnerPodSpec struct {
 	HostAliases []corev1.HostAlias `json:"hostAliases,omitempty"`
 
 	// +optional
-	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraint,omitempty"`
+	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
 
 	// RuntimeClassName is the container runtime configuration that containers should run under.
 	// More info: https://kubernetes.io/docs/concepts/containers/runtime-class
@@ -153,7 +153,7 @@ type RunnerPodSpec struct {
 	RuntimeClassName *string `json:"runtimeClassName,omitempty"`
 
 	// +optional
-	DnsConfig []corev1.PodDNSConfig `json:"dnsConfig,omitempty"`
+	DnsConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"`
 }
 
 // ValidateRepository validates repository field.
@@ -181,6 +181,9 @@ func (rs *RunnerSpec) ValidateRepository() error {
 
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
+	// Turns true only if the runner pod is ready.
+	// +optional
+	Ready bool `json:"ready"`
 	// +optional
 	Registration RunnerStatusRegistration `json:"registration"`
 	// +optional

api/v1alpha1/zz_generated.deepcopy.go

@@ -108,6 +108,11 @@ func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTri
 		*out = new(PushSpec)
 		**out = **in
 	}
+	if in.WorkflowJob != nil {
+		in, out := &in.WorkflowJob, &out.WorkflowJob
+		*out = new(WorkflowJobSpec)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubEventScaleUpTriggerSpec.
@@ -733,10 +738,8 @@ func (in *RunnerPodSpec) DeepCopyInto(out *RunnerPodSpec) {
 	}
 	if in.DnsConfig != nil {
 		in, out := &in.DnsConfig, &out.DnsConfig
-		*out = make([]v1.PodDNSConfig, len(*in))
+		*out = new(v1.PodDNSConfig)
-		for i := range *in {
+		(*in).DeepCopyInto(*out)
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
 	}
 }
 
@@ -1122,3 +1125,18 @@ func (in *ScheduledOverride) DeepCopy() *ScheduledOverride {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WorkflowJobSpec) DeepCopyInto(out *WorkflowJobSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkflowJobSpec.
+func (in *WorkflowJobSpec) DeepCopy() *WorkflowJobSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(WorkflowJobSpec)
+	in.DeepCopyInto(out)
+	return out
+}

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.17.1
+version: 0.17.3
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.22.1
+appVersion: 0.22.3
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/README.md

@@ -4,9 +4,9 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 
 ## Values
 
-**_The values are documented as of HEAD, to review the configuration options for your chart version ensure you view this file at the relevent [tag](https://github.com/actions-runner-controller/actions-runner-controller/tags)_**
+**_The values are documented as of HEAD, to review the configuration options for your chart version ensure you view this file at the relevant [tag](https://github.com/actions-runner-controller/actions-runner-controller/tags)_**
 
-> _Default values are the defaults set in the charts values.yaml, some properties have default configurations in the code for when the property is omitted or invalid_
+> _Default values are the defaults set in the charts `values.yaml`, some properties have default configurations in the code for when the property is omitted or invalid_
 
 | Key                                                      | Description                                                                                                                | Default                                                              |
 |----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------|

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -141,6 +141,7 @@ spec:
                               status:
                                 type: string
                               types:
+                                description: 'One of: created, rerequested, or completed'
                                 items:
                                   type: string
                                 type: array
@@ -160,6 +161,9 @@ spec:
                           push:
                             description: PushSpec is the condition for triggering scale-up on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
                             type: object
+                          workflowJob:
+                            description: https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job
+                            type: object
                         type: object
                     type: object
                   type: array

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml

@@ -1354,7 +1354,6 @@ spec:
                             type: object
                           type: array
                         dnsConfig:
-                          items:
                           description: PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.
                           properties:
                             nameservers:
@@ -1380,7 +1379,6 @@ spec:
                                 type: string
                               type: array
                           type: object
-                          type: array
                         dockerEnabled:
                           type: boolean
                         dockerEnv:
@@ -4157,7 +4155,7 @@ spec:
                                 type: string
                             type: object
                           type: array
-                        topologySpreadConstraint:
+                        topologySpreadConstraints:
                           items:
                             description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
                             properties:

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml

@@ -1351,7 +1351,6 @@ spec:
                             type: object
                           type: array
                         dnsConfig:
-                          items:
                           description: PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.
                           properties:
                             nameservers:
@@ -1377,7 +1376,6 @@ spec:
                                 type: string
                               type: array
                           type: object
-                          type: array
                         dockerEnabled:
                           type: boolean
                         dockerEnv:
@@ -4154,7 +4152,7 @@ spec:
                                 type: string
                             type: object
                           type: array
-                        topologySpreadConstraint:
+                        topologySpreadConstraints:
                           items:
                             description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
                             properties:

charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml

@@ -1292,7 +1292,6 @@ spec:
                     type: object
                   type: array
                 dnsConfig:
-                  items:
                   description: PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.
                   properties:
                     nameservers:
@@ -1318,7 +1317,6 @@ spec:
                         type: string
                       type: array
                   type: object
-                  type: array
                 dockerEnabled:
                   type: boolean
                 dockerEnv:
@@ -4095,7 +4093,7 @@ spec:
                         type: string
                     type: object
                   type: array
-                topologySpreadConstraint:
+                topologySpreadConstraints:
                   items:
                     description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
                     properties:
@@ -5126,6 +5124,9 @@ spec:
                   type: string
                 phase:
                   type: string
+                ready:
+                  description: Turns true only if the runner pod is ready.
+                  type: boolean
                 reason:
                   type: string
                 registration:

charts/actions-runner-controller/docs/UPGRADING.md

@@ -18,11 +18,11 @@ Due to the above you can't just do a `helm upgrade` to release the latest versio
 
 ## Steps
 
-1. Upgrade CRDs
+1. Upgrade CRDs, this isn't optional, the CRDs you are using must be those that correspond with the version of the controller you are installing
 
 ```shell
 # REMEMBER TO UPDATE THE CHART_VERSION TO RELEVANT CHART VERISON!!!!
-CHART_VERSION=0.17.0
+CHART_VERSION=0.18.0
 
 curl -L https://github.com/actions-runner-controller/actions-runner-controller/releases/download/actions-runner-controller-${CHART_VERSION}/actions-runner-controller-${CHART_VERSION}.tgz | tar zxv --strip 1 actions-runner-controller/crds
 

charts/actions-runner-controller/templates/deployment.yaml

@@ -45,6 +45,7 @@ spec:
         - "--leader-election-id={{ .Values.leaderElectionId }}"
         {{- end }}
         - "--sync-period={{ .Values.syncPeriod }}"
+        - "--default-scale-down-delay={{ .Values.defaultScaleDownDelay }}"
         - "--docker-image={{ .Values.image.dindSidecarRepositoryAndTag }}"
         - "--runner-image={{ .Values.image.actionsRunnerRepositoryAndTag }}"
         {{- range .Values.image.actionsRunnerImagePullSecrets }}

charts/actions-runner-controller/values.yaml

@@ -6,7 +6,8 @@ labels: {}
 
 replicaCount: 1
 
-syncPeriod: 10m
+syncPeriod: 1m
+defaultScaleDownDelay: 10m
 
 enableLeaderElection: true
 # Specifies the controller id for leader election.

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -141,6 +141,7 @@ spec:
                               status:
                                 type: string
                               types:
+                                description: 'One of: created, rerequested, or completed'
                                 items:
                                   type: string
                                 type: array
@@ -160,6 +161,9 @@ spec:
                           push:
                             description: PushSpec is the condition for triggering scale-up on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
                             type: object
+                          workflowJob:
+                            description: https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job
+                            type: object
                         type: object
                     type: object
                   type: array

config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml

@@ -1354,7 +1354,6 @@ spec:
                             type: object
                           type: array
                         dnsConfig:
-                          items:
                           description: PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.
                           properties:
                             nameservers:
@@ -1380,7 +1379,6 @@ spec:
                                 type: string
                               type: array
                           type: object
-                          type: array
                         dockerEnabled:
                           type: boolean
                         dockerEnv:
@@ -4157,7 +4155,7 @@ spec:
                                 type: string
                             type: object
                           type: array
-                        topologySpreadConstraint:
+                        topologySpreadConstraints:
                           items:
                             description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
                             properties:

config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml

@@ -1351,7 +1351,6 @@ spec:
                             type: object
                           type: array
                         dnsConfig:
-                          items:
                           description: PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.
                           properties:
                             nameservers:
@@ -1377,7 +1376,6 @@ spec:
                                 type: string
                               type: array
                           type: object
-                          type: array
                         dockerEnabled:
                           type: boolean
                         dockerEnv:
@@ -4154,7 +4152,7 @@ spec:
                                 type: string
                             type: object
                           type: array
-                        topologySpreadConstraint:
+                        topologySpreadConstraints:
                           items:
                             description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
                             properties:

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -1292,7 +1292,6 @@ spec:
                     type: object
                   type: array
                 dnsConfig:
-                  items:
                   description: PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.
                   properties:
                     nameservers:
@@ -1318,7 +1317,6 @@ spec:
                         type: string
                       type: array
                   type: object
-                  type: array
                 dockerEnabled:
                   type: boolean
                 dockerEnv:
@@ -4095,7 +4093,7 @@ spec:
                         type: string
                     type: object
                   type: array
-                topologySpreadConstraint:
+                topologySpreadConstraints:
                   items:
                     description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
                     properties:
@@ -5126,6 +5124,9 @@ spec:
                   type: string
                 phase:
                   type: string
+                ready:
+                  description: Turns true only if the runner pod is ready.
+                  type: boolean
                 reason:
                   type: string
                 registration:

config/crd/patches/webhook_in_runners.yaml

@@ -8,6 +8,7 @@ spec:
   conversion:
     strategy: Webhook
     webhook:
+      conversionReviewVersions: ["v1","v1beta1"]
       clientConfig:
         # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
         # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)

config/default/gh-webhook-server-auth-proxy-patch.yaml

@@ -0,0 +1,23 @@
+# This patch injects an HTTP proxy sidecar container that performs RBAC
+# authorization against the Kubernetes API using SubjectAccessReviews.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: github-webhook-server
+spec:
+  template:
+    spec:
+      containers:
+        - name: kube-rbac-proxy
+          image: quay.io/brancz/kube-rbac-proxy:v0.10.0
+          args:
+            - '--secure-listen-address=0.0.0.0:8443'
+            - '--upstream=http://127.0.0.1:8080/'
+            - '--logtostderr=true'
+            - '--v=10'
+          ports:
+            - containerPort: 8443
+              name: https
+        - name: github-webhook-server
+          args:
+            - '--metrics-addr=127.0.0.1:8080'

config/default/kustomization.yaml

@@ -22,17 +22,20 @@ bases:
 - ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
+# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
+#- ../github-webhook-server
 
 patchesStrategicMerge:
-  # Protect the /metrics endpoint by putting it behind auth.
+# Protect the /metrics endpoint by putting it behind auth.
-  # Only one of manager_auth_proxy_patch.yaml and
+# Only one of manager_auth_proxy_patch.yaml and
-  # manager_prometheus_metrics_patch.yaml should be enabled.
+# manager_prometheus_metrics_patch.yaml should be enabled.
 - manager_auth_proxy_patch.yaml
-  # If you want your controller-manager to expose the /metrics
+
-  # endpoint w/o any authn/z, uncomment the following line and
+# If you want your controller-manager to expose the /metrics
-  # comment manager_auth_proxy_patch.yaml.
+# endpoint w/o any authn/z, uncomment the following line and
-  # Only one of manager_auth_proxy_patch.yaml and
+# comment manager_auth_proxy_patch.yaml.
-  # manager_prometheus_metrics_patch.yaml should be enabled.
+# Only one of manager_auth_proxy_patch.yaml and
+# manager_prometheus_metrics_patch.yaml should be enabled.
 #- manager_prometheus_metrics_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
@@ -43,6 +46,10 @@ patchesStrategicMerge:
 # 'CERTMANAGER' needs to be enabled to use ca injection
 - webhookcainjection_patch.yaml
 
+# [GH_WEBHOOK_SERVER] To enable the GitHub webhook server, uncomment all sections with 'GH_WEBHOOK_SERVER'.
+# Protect the GitHub webhook server metrics endpoint by putting it behind auth.
+# - gh-webhook-server-auth-proxy-patch.yaml
+
 # the following config is for teaching kustomize how to do var substitution
 vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.

config/default/manager_auth_proxy_patch.yaml

@@ -23,4 +23,3 @@ spec:
         args:
         - "--metrics-addr=127.0.0.1:8080"
         - "--enable-leader-election"
-        - "--sync-period=10m"

config/github-webhook-server/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: github-webhook-server
+      app.kubernetes.io/part-of: actions-runner-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: github-webhook-server
+        app.kubernetes.io/part-of: actions-runner-controller
+    spec:
+      containers:
+        - name: github-webhook-server
+          image: controller:latest
+          command:
+            - '/github-webhook-server'
+          env:
+            - name: GITHUB_WEBHOOK_SECRET_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  key: github_webhook_secret_token
+                  name: github-webhook-server
+                  optional: true
+          ports:
+            - containerPort: 8000
+              name: http
+              protocol: TCP
+      serviceAccountName: github-webhook-server
+      terminationGracePeriodSeconds: 10

config/github-webhook-server/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+images:
+  - name: controller
+    newName: summerwind/actions-runner-controller
+    newTag: latest
+
+resources:
+  - deployment.yaml
+  - rbac.yaml
+  - service.yaml

config/github-webhook-server/rbac.yaml

@@ -0,0 +1,113 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+rules:
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - horizontalrunnerautoscalers
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - horizontalrunnerautoscalers/finalizers
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - horizontalrunnerautoscalers/status
+    verbs:
+      - get
+      - patch
+      - update
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnersets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnerdeployments
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnerdeployments/finalizers
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - actions.summerwind.dev
+    resources:
+      - runnerdeployments/status
+    verbs:
+      - get
+      - patch
+      - update
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: github-webhook-server
+subjects:
+  - kind: ServiceAccount
+    name: github-webhook-server

config/github-webhook-server/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller
+  name: github-webhook-server
+spec:
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/component: github-webhook-server
+    app.kubernetes.io/part-of: actions-runner-controller

controllers/autoscaling.go

@@ -29,10 +29,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTa
 	metrics := hra.Spec.Metrics
 	numMetrics := len(metrics)
 	if numMetrics == 0 {
-		if len(hra.Spec.ScaleUpTriggers) == 0 {
+		// We don't default to anything since ARC 0.23.0
-			return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(st, hra, nil)
+		// See https://github.com/actions-runner-controller/actions-runner-controller/issues/728
-		}
-
 		return nil, nil
 	} else if numMetrics > 2 {
 		return nil, fmt.Errorf("too many autoscaling metrics configured: It must be 0 to 2, but got %d", numMetrics)
@@ -140,7 +138,29 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 		if len(allJobs) == 0 {
 			fallback_cb()
 		} else {
+		JOB:
 			for _, job := range allJobs {
+				runnerLabels := make(map[string]struct{}, len(st.labels))
+				for _, l := range st.labels {
+					runnerLabels[l] = struct{}{}
+				}
+
+				if len(job.Labels) == 0 {
+					// This shouldn't usually happen
+					r.Log.Info("Detected job with no labels, which is not supported by ARC. Skipping anyway.", "labels", job.Labels, "run_id", job.GetRunID(), "job_id", job.GetID())
+					continue JOB
+				}
+
+				for _, l := range job.Labels {
+					if l == "self-hosted" {
+						continue
+					}
+
+					if _, ok := runnerLabels[l]; !ok {
+						continue JOB
+					}
+				}
+
 				switch job.GetStatus() {
 				case "completed":
 					// We add a case for `completed` so it is not counted in `unknown`.

controllers/autoscaling_test.go

@@ -41,8 +41,12 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 	metav1Now := metav1.Now()
 	testcases := []struct {
+		description string
+
 		repo   string
 		org    string
+		labels []string
+
 		fixed     *int
 		max       *int
 		min       *int
@@ -68,6 +72,19 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
 			want:                     3,
 		},
+		// Explicitly speified the default `self-hosted` label which is ignored by the simulator,
+		// as we assume that GitHub Actions automatically associates the `self-hosted` label to every self-hosted runner.
+		// 3 demanded, max at 3
+		{
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
+		},
 		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
 		{
 			repo:                     "test/valid",
@@ -152,9 +169,40 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			want:                     3,
 		},
 
-		// Job-level autoscaling
-		// 5 requested from 3 workflows
 		{
+			description:              "Job-level autoscaling with no explicit runner label (runners have implicit self-hosted, requested self-hosted, 5 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"completed", "labels":["self-hosted"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with no explicit runner label (runners have implicit self-hosted, requested self-hosted+custom, 0 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with no label (runners have implicit self-hosted, jobs had no labels, 0 jobs from 3 workflows)",
 			repo:                     "test/valid",
 			min:                      intPtr(2),
 			max:                      intPtr(10),
@@ -166,6 +214,91 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
 				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
 			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with default runner label (runners have self-hosted only, requested self-hosted+custom, 0 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with custom runner label (runners have custom2, requested self-hosted+custom, 0 jobs from 5 workflows",
+			repo:                     "test/valid",
+			labels:                   []string{"custom2"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with default runner label (runners have self-hosted, requested managed-runner-label, 0 jobs from 3 runs)",
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["managed-runner-label"]}, {"status":"queued", "labels":["managed-runner-label"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["managed-runner-label"]}, {"status":"completed", "labels":["managed-runner-label"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["managed-runner-label"]}, {"status":"queued", "labels":["managed-runner-label"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Job-level autoscaling with default + custom runner label (runners have self-hosted+custom, requested self-hosted+custom, 5 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted", "custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Job-level autoscaling with custom runner label (runners have custom, requested self-hosted+custom, 5 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			labels:                   []string{"custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
 			want: 5,
 		},
 	}
@@ -181,7 +314,12 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		_ = clientgoscheme.AddToScheme(scheme)
 		_ = v1alpha1.AddToScheme(scheme)
 
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+		testName := fmt.Sprintf("case %d", i)
+		if tc.description != "" {
+			testName = tc.description
+		}
+
+		t.Run(testName, func(t *testing.T) {
 			server := fake.NewServer(
 				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
 				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
@@ -194,6 +332,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 				Log:                   log,
 				GitHubClient:          client,
 				Scheme:                scheme,
+				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
 
 			rd := v1alpha1.RunnerDeployment{
@@ -206,6 +345,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 						Spec: v1alpha1.RunnerSpec{
 							RunnerConfig: v1alpha1.RunnerConfig{
 								Repository: tc.repo,
+								Labels:     tc.labels,
 							},
 						},
 					},
@@ -220,6 +360,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 				Spec: v1alpha1.HorizontalRunnerAutoscalerSpec{
 					MaxReplicas: tc.max,
 					MinReplicas: tc.min,
+					Metrics: []v1alpha1.MetricSpec{
+						{
+							Type: "TotalNumberOfQueuedAndInProgressWorkflowRuns",
+						},
+					},
 				},
 				Status: v1alpha1.HorizontalRunnerAutoscalerStatus{
 					DesiredReplicas:            tc.sReplicas,
@@ -258,8 +403,12 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 	metav1Now := metav1.Now()
 	testcases := []struct {
+		description string
+
 		repos  []string
 		org    string
+		labels []string
+
 		fixed     *int
 		max       *int
 		min       *int
@@ -399,9 +548,43 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			err:                      "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
 		},
 
-		// Job-level autoscaling
-		// 5 requested from 3 workflows
 		{
+			description:              "Job-level autoscaling (runners have implicit self-hosted, requested self-hosted, 5 jobs from 3 runs)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"completed", "labels":["self-hosted"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Job-level autoscaling (runners have explicit self-hosted, requested self-hosted, 5 jobs from 3 runs)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"completed", "labels":["self-hosted"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (jobs lack labels, 0 requested from 3 workflows)",
 			org:                      "test",
 			repos:                    []string{"valid"},
 			min:                      intPtr(2),
@@ -414,8 +597,97 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
 				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
 			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (runners have valid and implicit self-hosted, requested self-hosted+custom, 0 jobs from 3 runs)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (runners have self-hosted, requested self-hosted+custom, 0 jobs from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Job-level autoscaling (runners have custom, requested self-hosted+custom, 5 requested from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
 			want: 5,
 		},
+
+		{
+			description:              "Job-level autoscaling (runners have custom, requested custom, 5 requested from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["custom"]}, {"status":"queued", "labels":["custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["custom"]}, {"status":"completed", "labels":["custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["custom"]}, {"status":"queued", "labels":["custom"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (specified custom2, 0 requested from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"custom2"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
 	}
 
 	for i := range testcases {
@@ -429,7 +701,12 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		_ = clientgoscheme.AddToScheme(scheme)
 		_ = v1alpha1.AddToScheme(scheme)
 
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+		testName := fmt.Sprintf("case %d", i)
+		if tc.description != "" {
+			testName = tc.description
+		}
+
+		t.Run(testName, func(t *testing.T) {
 			t.Helper()
 
 			server := fake.NewServer(
@@ -444,6 +721,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 				Log:                   log,
 				Scheme:                scheme,
 				GitHubClient:          client,
+				DefaultScaleDownDelay: DefaultScaleDownDelay,
 			}
 
 			rd := v1alpha1.RunnerDeployment{
@@ -465,6 +743,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 						Spec: v1alpha1.RunnerSpec{
 							RunnerConfig: v1alpha1.RunnerConfig{
 								Organization: tc.org,
+								Labels:       tc.labels,
 							},
 						},
 					},

controllers/horizontal_runner_autoscaler_webhook.go

@@ -242,18 +242,23 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.Respons
 				enterpriseSlug,
 				labels,
 			)
+			if target == nil {
+				break
+			}
 
-			if target != nil {
 			if e.GetAction() == "queued" {
 				target.Amount = 1
-				} else if e.GetAction() == "completed" {
+				break
+			} else if e.GetAction() == "completed" && e.GetWorkflowJob().GetConclusion() != "skipped" {
 				// A nagative amount is processed in the tryScale func as a scale-down request,
 				// that erasese the oldest CapacityReservation with the same amount.
 				// If the first CapacityReservation was with Replicas=1, this negative scale target erases that,
 				// so that the resulting desired replicas decreases by 1.
 				target.Amount = -1
+				break
 			}
-			}
+			// If the conclusion is "skipped", we will ignore it and fallthrough to the default case.
+			fallthrough
 		default:
 			ok = true
 
@@ -663,16 +668,29 @@ HRA:
 
 		if len(hra.Spec.ScaleUpTriggers) > 1 {
 			autoscaler.Log.V(1).Info("Skipping this HRA as it has too many ScaleUpTriggers to be used in workflow_job based scaling", "hra", hra.Name)
+			continue
+		}
+
+		if len(hra.Spec.ScaleUpTriggers) == 0 {
+			autoscaler.Log.V(1).Info("Skipping this HRA as it has no ScaleUpTriggers configured", "hra", hra.Name)
+			continue
+		}
+
+		scaleUpTrigger := hra.Spec.ScaleUpTriggers[0]
+
+		if scaleUpTrigger.GitHubEvent == nil {
+			autoscaler.Log.V(1).Info("Skipping this HRA as it has no `githubEvent` scale trigger configured", "hra", hra.Name)
 
 			continue
 		}
 
-		var duration metav1.Duration
+		if scaleUpTrigger.GitHubEvent.WorkflowJob == nil {
+			autoscaler.Log.V(1).Info("Skipping this HRA as it has no `githubEvent.workflowJob` scale trigger configured", "hra", hra.Name)
 
-		if len(hra.Spec.ScaleUpTriggers) > 0 {
+			continue
-			duration = hra.Spec.ScaleUpTriggers[0].Duration
 		}
 
+		duration := scaleUpTrigger.Duration
 		if duration.Duration <= 0 {
 			// Try to release the reserved capacity after at least 10 minutes by default,
 			// we won't end up in the reserved capacity remained forever in case GitHub somehow stopped sending us "completed" workflow_job events.

controllers/horizontal_runner_autoscaler_webhook_test.go

@@ -138,6 +138,13 @@ func TestWebhookWorkflowJob(t *testing.T) {
 				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
 					Name: "test-name",
 				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
 			},
 		}
 
@@ -177,6 +184,13 @@ func TestWebhookWorkflowJob(t *testing.T) {
 				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
 					Name: "test-name",
 				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
 			},
 		}
 
@@ -217,6 +231,13 @@ func TestWebhookWorkflowJob(t *testing.T) {
 				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
 					Name: "test-name",
 				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
 			},
 		}
 
@@ -277,6 +298,13 @@ func TestWebhookWorkflowJobWithSelfHostedLabel(t *testing.T) {
 				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
 					Name: "test-name",
 				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
 			},
 		}
 
@@ -316,6 +344,13 @@ func TestWebhookWorkflowJobWithSelfHostedLabel(t *testing.T) {
 				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
 					Name: "test-name",
 				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
 			},
 		}
 
@@ -356,6 +391,13 @@ func TestWebhookWorkflowJobWithSelfHostedLabel(t *testing.T) {
 				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
 					Name: "test-name",
 				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
 			},
 		}
 

controllers/horizontalrunnerautoscaler_controller.go

@@ -51,8 +51,8 @@ type HorizontalRunnerAutoscalerReconciler struct {
 	Log                   logr.Logger
 	Recorder              record.EventRecorder
 	Scheme                *runtime.Scheme
-
 	CacheDuration         time.Duration
+	DefaultScaleDownDelay time.Duration
 	Name                  string
 }
 
@@ -159,6 +159,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(ctx context.Context, re
 			org:        rs.Spec.Organization,
 			repo:       rs.Spec.Repository,
 			replicas:   replicas,
+			labels:     rs.Spec.RunnerConfig.Labels,
 			getRunnerMap: func() (map[string]struct{}, error) {
 				// return the list of runners in namespace. Horizontal Runner Autoscaler should only be responsible for scaling resources in its own ns.
 				var runnerPodList corev1.PodList
@@ -251,6 +252,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) scaleTargetFromRD(ctx context.Con
 		org:        rd.Spec.Template.Spec.Organization,
 		repo:       rd.Spec.Template.Spec.Repository,
 		replicas:   rd.Spec.Replicas,
+		labels:     rd.Spec.Template.Spec.RunnerConfig.Labels,
 		getRunnerMap: func() (map[string]struct{}, error) {
 			// return the list of runners in namespace. Horizontal Runner Autoscaler should only be responsible for scaling resources in its own ns.
 			var runnerList v1alpha1.RunnerList
@@ -293,6 +295,7 @@ type scaleTarget struct {
 	st, kind              string
 	enterprise, repo, org string
 	replicas              *int
+	labels                []string
 
 	getRunnerMap func() (map[string]struct{}, error)
 }
@@ -497,7 +500,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr
 	if hra.Spec.ScaleDownDelaySecondsAfterScaleUp != nil {
 		scaleDownDelay = time.Duration(*hra.Spec.ScaleDownDelaySecondsAfterScaleUp) * time.Second
 	} else {
-		scaleDownDelay = DefaultScaleDownDelay
+		scaleDownDelay = r.DefaultScaleDownDelay
 	}
 
 	var scaleDownDelayUntil *time.Time

controllers/integration_test.go

@@ -537,6 +537,106 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 			}
 		})
 
+		It("should create and scale organization's repository runners on workflow_job event", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+									Group:      "baz",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, rd, "test RunnerDeployment")
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 1 replica via ScaleUpTriggers.GitHubEvent.WorkflowJob based scaling
+			{
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(5),
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+								},
+								Amount:   1,
+								Duration: metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler")
+
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 2 replicas on first workflow_job.queued webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"self-hosted"})
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
+			}
+
+			// Scale-up to 3 replicas on second workflow_job.queued webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"self-hosted"})
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after second webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
+			}
+
+			// Do not scale-up on third workflow_job.queued webhook event
+			// repo "example" doesn't match our Spec
+			{
+				env.SendWorkflowJobEvent("test", "example", "queued", []string{"self-hosted"})
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after third webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
+			}
+		})
+
 		It("should create and scale organization's repository runners only on check_run event", func() {
 			name := "example-runnerdeploy"
 
@@ -581,9 +681,7 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
 			}
 
-			// Scale-up to 3 replicas by the default TotalNumberOfQueuedAndInProgressWorkflowRuns-based scaling
+			// Scale-up to 1 replica via ScaleUpTriggers.GitHubEvent.CheckRun based scaling
-			// See workflowRunsFor3Replicas_queued and workflowRunsFor3Replicas_in_progress for GitHub List-Runners API responses
-			// used while testing.
 			{
 				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
 					ObjectMeta: metav1.ObjectMeta{
@@ -1130,7 +1228,9 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
 						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
 							{
-								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{},
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+								},
 								Amount:   1,
 								Duration: metav1.Duration{Duration: time.Minute},
 							},
@@ -1150,7 +1250,7 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 			// Scale-up to 2 replicas on first workflow_job webhook event
 			{
-				env.SendWorkflowJobEvent("test", "valid", "pending", "created", []string{"self-hosted"})
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"self-hosted"})
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
 				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
@@ -1212,7 +1312,9 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
 						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
 							{
-								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{},
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+								},
 								Amount:   1,
 								Duration: metav1.Duration{Duration: time.Minute},
 							},
@@ -1232,7 +1334,7 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 			// Scale-up to 2 replicas on first workflow_job webhook event
 			{
-				env.SendWorkflowJobEvent("test", "valid", "pending", "created", []string{"custom-label"})
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"custom-label"})
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
 				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
@@ -1313,6 +1415,30 @@ func (env *testEnvironment) SendOrgCheckRunEvent(org, repo, status, action strin
 	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
 }
 
+func (env *testEnvironment) SendWorkflowJobEvent(org, repo, statusAndAction string, labels []string) {
+	resp, err := sendWebhook(env.webhookServer, "workflow_job", &github.WorkflowJobEvent{
+		WorkflowJob: &github.WorkflowJob{
+			Status: &statusAndAction,
+			Labels: labels,
+		},
+		Org: &github.Organization{
+			Login: github.String(org),
+		},
+		Repo: &github.Repository{
+			Name: github.String(repo),
+			Owner: &github.User{
+				Login: github.String(org),
+				Type:  github.String("Organization"),
+			},
+		},
+		Action: github.String(statusAndAction),
+	})
+
+	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to send workflow_job event")
+
+	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
+}
+
 func (env *testEnvironment) SendUserPullRequestEvent(owner, repo, branch, action string) {
 	resp, err := sendWebhook(env.webhookServer, "pull_request", &github.PullRequestEvent{
 		PullRequest: &github.PullRequest{
@@ -1355,28 +1481,6 @@ func (env *testEnvironment) SendUserCheckRunEvent(owner, repo, status, action st
 	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
 }
 
-func (env *testEnvironment) SendWorkflowJobEvent(owner, repo, status, action string, labels []string) {
-	resp, err := sendWebhook(env.webhookServer, "workflow_job", &github.WorkflowJobEvent{
-		Org: &github.Organization{
-			Name: github.String(owner),
-		},
-		WorkflowJob: &github.WorkflowJob{
-			Labels: labels,
-		},
-		Action: github.String("queued"),
-		Repo: &github.Repository{
-			Name: github.String(repo),
-			Owner: &github.User{
-				Login: github.String(owner),
-				Type:  github.String("Organization"),
-			},
-		},
-	})
-
-	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to send check_run event")
-
-	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
-}
 func (env *testEnvironment) SyncRunnerRegistrations() {
 	var runnerList actionsv1alpha1.RunnerList
 

controllers/new_runner_pod_test.go

@@ -0,0 +1,940 @@
+package controllers
+
+import (
+	"testing"
+
+	arcv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+)
+
+func TestNewRunnerPod(t *testing.T) {
+	type testcase struct {
+		description string
+
+		template corev1.Pod
+		config   arcv1alpha1.RunnerConfig
+		want     corev1.Pod
+	}
+
+	base := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels: map[string]string{
+				"actions-runner-controller/inject-registration-token": "true",
+				"runnerset-name": "runner",
+			},
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "runner",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+				{
+					Name: "work",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+				{
+					Name: "certs-client",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name:  "runner",
+					Image: "default-runner-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "RUNNER_ORG",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_REPO",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_ENTERPRISE",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_LABELS",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_GROUP",
+							Value: "",
+						},
+						{
+							Name:  "DOCKER_ENABLED",
+							Value: "true",
+						},
+						{
+							Name:  "DOCKERD_IN_RUNNER",
+							Value: "false",
+						},
+						{
+							Name:  "GITHUB_URL",
+							Value: "api.github.com",
+						},
+						{
+							Name:  "RUNNER_WORKDIR",
+							Value: "/runner/_work",
+						},
+						{
+							Name:  "RUNNER_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "DOCKER_HOST",
+							Value: "tcp://localhost:2376",
+						},
+						{
+							Name:  "DOCKER_TLS_VERIFY",
+							Value: "1",
+						},
+						{
+							Name:  "DOCKER_CERT_PATH",
+							Value: "/certs/client",
+						},
+						{
+							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
+							Value: "true",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+						{
+							Name:      "work",
+							MountPath: "/runner/_work",
+						},
+						{
+							Name:      "certs-client",
+							MountPath: "/certs/client",
+							ReadOnly:  true,
+						},
+					},
+					ImagePullPolicy: corev1.PullAlways,
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: func() *bool { v := false; return &v }(),
+					},
+				},
+				{
+					Name:  "docker",
+					Image: "default-docker-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "DOCKER_TLS_CERTDIR",
+							Value: "/certs",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+						{
+							Name:      "certs-client",
+							MountPath: "/certs/client",
+						},
+						{
+							Name:      "work",
+							MountPath: "/runner/_work",
+						},
+					},
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: func(b bool) *bool { return &b }(true),
+					},
+				},
+			},
+			RestartPolicy: corev1.RestartPolicyOnFailure,
+		},
+	}
+
+	boolPtr := func(v bool) *bool {
+		return &v
+	}
+
+	dinrBase := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels: map[string]string{
+				"actions-runner-controller/inject-registration-token": "true",
+				"runnerset-name": "runner",
+			},
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "runner",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name:  "runner",
+					Image: "default-runner-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "RUNNER_ORG",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_REPO",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_ENTERPRISE",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_LABELS",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_GROUP",
+							Value: "",
+						},
+						{
+							Name:  "DOCKER_ENABLED",
+							Value: "true",
+						},
+						{
+							Name:  "DOCKERD_IN_RUNNER",
+							Value: "true",
+						},
+						{
+							Name:  "GITHUB_URL",
+							Value: "api.github.com",
+						},
+						{
+							Name:  "RUNNER_WORKDIR",
+							Value: "/runner/_work",
+						},
+						{
+							Name:  "RUNNER_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
+							Value: "true",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+					},
+					ImagePullPolicy: corev1.PullAlways,
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: boolPtr(true),
+					},
+				},
+			},
+			RestartPolicy: corev1.RestartPolicyOnFailure,
+		},
+	}
+
+	dockerDisabled := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels: map[string]string{
+				"actions-runner-controller/inject-registration-token": "true",
+				"runnerset-name": "runner",
+			},
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "runner",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name:  "runner",
+					Image: "default-runner-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "RUNNER_ORG",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_REPO",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_ENTERPRISE",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_LABELS",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_GROUP",
+							Value: "",
+						},
+						{
+							Name:  "DOCKER_ENABLED",
+							Value: "false",
+						},
+						{
+							Name:  "DOCKERD_IN_RUNNER",
+							Value: "false",
+						},
+						{
+							Name:  "GITHUB_URL",
+							Value: "api.github.com",
+						},
+						{
+							Name:  "RUNNER_WORKDIR",
+							Value: "/runner/_work",
+						},
+						{
+							Name:  "RUNNER_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
+							Value: "true",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+					},
+					ImagePullPolicy: corev1.PullAlways,
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: boolPtr(false),
+					},
+				},
+			},
+			RestartPolicy: corev1.RestartPolicyOnFailure,
+		},
+	}
+
+	newTestPod := func(base corev1.Pod, f func(*corev1.Pod)) corev1.Pod {
+		pod := base.DeepCopy()
+		if f != nil {
+			f(pod)
+		}
+		return *pod
+	}
+
+	testcases := []testcase{
+		{
+			description: "it should have unprivileged runner and privileged sidecar docker container",
+			template:    corev1.Pod{},
+			config:      arcv1alpha1.RunnerConfig{},
+			want:        newTestPod(base, nil),
+		},
+		{
+			description: "dockerdWithinRunnerContainer=true should set privileged=true and omit the dind sidecar container",
+			template:    corev1.Pod{},
+			config: arcv1alpha1.RunnerConfig{
+				DockerdWithinRunnerContainer: boolPtr(true),
+			},
+			want: newTestPod(dinrBase, nil),
+		},
+		{
+			description: "in the default config you should provide both dockerdWithinRunnerContainer=true and runnerImage",
+			template:    corev1.Pod{},
+			config: arcv1alpha1.RunnerConfig{
+				DockerdWithinRunnerContainer: boolPtr(true),
+				Image:                        "dind-runner-image",
+			},
+			want: newTestPod(dinrBase, func(p *corev1.Pod) {
+				p.Spec.Containers[0].Image = "dind-runner-image"
+			}),
+		},
+		{
+			description: "dockerEnabled=false should have no effect when dockerdWithinRunnerContainer=true",
+			template:    corev1.Pod{},
+			config: arcv1alpha1.RunnerConfig{
+				DockerdWithinRunnerContainer: boolPtr(true),
+				DockerEnabled:                boolPtr(false),
+			},
+			want: newTestPod(dinrBase, nil),
+		},
+		{
+			description: "dockerEnabled=false should omit the dind sidecar and set privileged=false and envvars DOCKER_ENABLED=false and DOCKERD_IN_RUNNER=false",
+			template:    corev1.Pod{},
+			config: arcv1alpha1.RunnerConfig{
+				DockerEnabled: boolPtr(false),
+			},
+			want: newTestPod(dockerDisabled, nil),
+		},
+		{
+			description: "TODO: dockerEnabled=false results in privileged=false by default but you can override it",
+			template: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "runner",
+							SecurityContext: &corev1.SecurityContext{
+								Privileged: boolPtr(true),
+							},
+						},
+					},
+				},
+			},
+			config: arcv1alpha1.RunnerConfig{
+				DockerEnabled: boolPtr(false),
+			},
+			want: newTestPod(dockerDisabled, func(p *corev1.Pod) {
+				// TODO
+				// p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+			}),
+		},
+	}
+
+	var (
+		defaultRunnerImage            = "default-runner-image"
+		defaultRunnerImagePullSecrets = []string{}
+		defaultDockerImage            = "default-docker-image"
+		defaultDockerRegistryMirror   = ""
+		githubBaseURL                 = "api.github.com"
+	)
+
+	for i := range testcases {
+		tc := testcases[i]
+		t.Run(tc.description, func(t *testing.T) {
+			got, err := newRunnerPod("runner", tc.template, tc.config, defaultRunnerImage, defaultRunnerImagePullSecrets, defaultDockerImage, defaultDockerRegistryMirror, githubBaseURL, false)
+			require.NoError(t, err)
+			require.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestNewRunnerPodFromRunnerController(t *testing.T) {
+	type testcase struct {
+		description string
+
+		runner arcv1alpha1.Runner
+		want   corev1.Pod
+	}
+
+	boolPtr := func(v bool) *bool {
+		return &v
+	}
+
+	base := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "runner",
+			Labels: map[string]string{
+				"actions-runner-controller/inject-registration-token": "true",
+				"pod-template-hash": "8857b86c7",
+				"runnerset-name":    "runner",
+			},
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion:         "actions.summerwind.dev/v1alpha1",
+					Kind:               "Runner",
+					Name:               "runner",
+					Controller:         boolPtr(true),
+					BlockOwnerDeletion: boolPtr(true),
+				},
+			},
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "runner",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+				{
+					Name: "work",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+				{
+					Name: "certs-client",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name:  "runner",
+					Image: "default-runner-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "RUNNER_ORG",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_REPO",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_ENTERPRISE",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_LABELS",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_GROUP",
+							Value: "",
+						},
+						{
+							Name:  "DOCKER_ENABLED",
+							Value: "true",
+						},
+						{
+							Name:  "DOCKERD_IN_RUNNER",
+							Value: "false",
+						},
+						{
+							Name:  "GITHUB_URL",
+							Value: "api.github.com",
+						},
+						{
+							Name:  "RUNNER_WORKDIR",
+							Value: "/runner/_work",
+						},
+						{
+							Name:  "RUNNER_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "DOCKER_HOST",
+							Value: "tcp://localhost:2376",
+						},
+						{
+							Name:  "DOCKER_TLS_VERIFY",
+							Value: "1",
+						},
+						{
+							Name:  "DOCKER_CERT_PATH",
+							Value: "/certs/client",
+						},
+						{
+							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "RUNNER_NAME",
+							Value: "runner",
+						},
+						{
+							Name:  "RUNNER_TOKEN",
+							Value: "",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+						{
+							Name:      "work",
+							MountPath: "/runner/_work",
+						},
+						{
+							Name:      "certs-client",
+							MountPath: "/certs/client",
+							ReadOnly:  true,
+						},
+					},
+					ImagePullPolicy: corev1.PullAlways,
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: func() *bool { v := false; return &v }(),
+					},
+				},
+				{
+					Name:  "docker",
+					Image: "default-docker-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "DOCKER_TLS_CERTDIR",
+							Value: "/certs",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+						{
+							Name:      "certs-client",
+							MountPath: "/certs/client",
+						},
+						{
+							Name:      "work",
+							MountPath: "/runner/_work",
+						},
+					},
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: func(b bool) *bool { return &b }(true),
+					},
+				},
+			},
+			RestartPolicy: corev1.RestartPolicyOnFailure,
+		},
+	}
+
+	dinrBase := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "runner",
+			Labels: map[string]string{
+				"actions-runner-controller/inject-registration-token": "true",
+				"pod-template-hash": "8857b86c7",
+				"runnerset-name":    "runner",
+			},
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion:         "actions.summerwind.dev/v1alpha1",
+					Kind:               "Runner",
+					Name:               "runner",
+					Controller:         boolPtr(true),
+					BlockOwnerDeletion: boolPtr(true),
+				},
+			},
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "runner",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name:  "runner",
+					Image: "default-runner-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "RUNNER_ORG",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_REPO",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_ENTERPRISE",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_LABELS",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_GROUP",
+							Value: "",
+						},
+						{
+							Name:  "DOCKER_ENABLED",
+							Value: "true",
+						},
+						{
+							Name:  "DOCKERD_IN_RUNNER",
+							Value: "true",
+						},
+						{
+							Name:  "GITHUB_URL",
+							Value: "api.github.com",
+						},
+						{
+							Name:  "RUNNER_WORKDIR",
+							Value: "/runner/_work",
+						},
+						{
+							Name:  "RUNNER_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "RUNNER_NAME",
+							Value: "runner",
+						},
+						{
+							Name:  "RUNNER_TOKEN",
+							Value: "",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+					},
+					ImagePullPolicy: corev1.PullAlways,
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: boolPtr(true),
+					},
+				},
+			},
+			RestartPolicy: corev1.RestartPolicyOnFailure,
+		},
+	}
+
+	dockerDisabled := corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "runner",
+			Labels: map[string]string{
+				"actions-runner-controller/inject-registration-token": "true",
+				"pod-template-hash": "8857b86c7",
+				"runnerset-name":    "runner",
+			},
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion:         "actions.summerwind.dev/v1alpha1",
+					Kind:               "Runner",
+					Name:               "runner",
+					Controller:         boolPtr(true),
+					BlockOwnerDeletion: boolPtr(true),
+				},
+			},
+		},
+		Spec: corev1.PodSpec{
+			Volumes: []corev1.Volume{
+				{
+					Name: "runner",
+					VolumeSource: corev1.VolumeSource{
+						EmptyDir: &corev1.EmptyDirVolumeSource{},
+					},
+				},
+			},
+			Containers: []corev1.Container{
+				{
+					Name:  "runner",
+					Image: "default-runner-image",
+					Env: []corev1.EnvVar{
+						{
+							Name:  "RUNNER_ORG",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_REPO",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_ENTERPRISE",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_LABELS",
+							Value: "",
+						},
+						{
+							Name:  "RUNNER_GROUP",
+							Value: "",
+						},
+						{
+							Name:  "DOCKER_ENABLED",
+							Value: "false",
+						},
+						{
+							Name:  "DOCKERD_IN_RUNNER",
+							Value: "false",
+						},
+						{
+							Name:  "GITHUB_URL",
+							Value: "api.github.com",
+						},
+						{
+							Name:  "RUNNER_WORKDIR",
+							Value: "/runner/_work",
+						},
+						{
+							Name:  "RUNNER_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "RUNNER_FEATURE_FLAG_EPHEMERAL",
+							Value: "true",
+						},
+						{
+							Name:  "RUNNER_NAME",
+							Value: "runner",
+						},
+						{
+							Name:  "RUNNER_TOKEN",
+							Value: "",
+						},
+					},
+					VolumeMounts: []corev1.VolumeMount{
+						{
+							Name:      "runner",
+							MountPath: "/runner",
+						},
+					},
+					ImagePullPolicy: corev1.PullAlways,
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: boolPtr(false),
+					},
+				},
+			},
+			RestartPolicy: corev1.RestartPolicyOnFailure,
+		},
+	}
+
+	newTestPod := func(base corev1.Pod, f func(*corev1.Pod)) corev1.Pod {
+		pod := base.DeepCopy()
+		if f != nil {
+			f(pod)
+		}
+		return *pod
+	}
+
+	testcases := []testcase{
+		{
+			description: "it should have unprivileged runner and privileged sidecar docker container",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerConfig: arcv1alpha1.RunnerConfig{},
+				},
+			},
+			want: newTestPod(base, nil),
+		},
+		{
+			description: "dockerdWithinRunnerContainer=true should set privileged=true and omit the dind sidecar container",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerConfig: arcv1alpha1.RunnerConfig{
+						DockerdWithinRunnerContainer: boolPtr(true),
+					},
+				},
+			},
+			want: newTestPod(dinrBase, nil),
+		},
+		{
+			description: "in the default config you should provide both dockerdWithinRunnerContainer=true and runnerImage",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerConfig: arcv1alpha1.RunnerConfig{
+						DockerdWithinRunnerContainer: boolPtr(true),
+						Image:                        "dind-runner-image",
+					},
+				},
+			},
+			want: newTestPod(dinrBase, func(p *corev1.Pod) {
+				p.Spec.Containers[0].Image = "dind-runner-image"
+			}),
+		},
+		{
+			description: "dockerEnabled=false should have no effect when dockerdWithinRunnerContainer=true",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerConfig: arcv1alpha1.RunnerConfig{
+						DockerdWithinRunnerContainer: boolPtr(true),
+						DockerEnabled:                boolPtr(false),
+					},
+				},
+			},
+			want: newTestPod(dinrBase, nil),
+		},
+		{
+			description: "dockerEnabled=false should omit the dind sidecar and set privileged=false and envvars DOCKER_ENABLED=false and DOCKERD_IN_RUNNER=false",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerConfig: arcv1alpha1.RunnerConfig{
+						DockerEnabled: boolPtr(false),
+					},
+				},
+			},
+			want: newTestPod(dockerDisabled, nil),
+		},
+		{
+			description: "TODO: dockerEnabled=false results in privileged=false by default but you can override it",
+			runner: arcv1alpha1.Runner{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "runner",
+				},
+				Spec: arcv1alpha1.RunnerSpec{
+					RunnerConfig: arcv1alpha1.RunnerConfig{
+						DockerEnabled: boolPtr(false),
+					},
+					RunnerPodSpec: arcv1alpha1.RunnerPodSpec{
+						Containers: []corev1.Container{
+							{
+								Name: "runner",
+								SecurityContext: &corev1.SecurityContext{
+									Privileged: boolPtr(true),
+								},
+							},
+						},
+					},
+				},
+			},
+
+			want: newTestPod(dockerDisabled, func(p *corev1.Pod) {
+				// p.Spec.Containers[0].SecurityContext.Privileged = boolPtr(true)
+			}),
+		},
+	}
+
+	var (
+		defaultRunnerImage            = "default-runner-image"
+		defaultRunnerImagePullSecrets = []string{}
+		defaultDockerImage            = "default-docker-image"
+		defaultDockerRegistryMirror   = ""
+		githubBaseURL                 = "api.github.com"
+	)
+
+	scheme := runtime.NewScheme()
+	_ = clientgoscheme.AddToScheme(scheme)
+	_ = arcv1alpha1.AddToScheme(scheme)
+
+	for i := range testcases {
+		tc := testcases[i]
+		t.Run(tc.description, func(t *testing.T) {
+			r := &RunnerReconciler{
+				RunnerImage:            defaultRunnerImage,
+				RunnerImagePullSecrets: defaultRunnerImagePullSecrets,
+				DockerImage:            defaultDockerImage,
+				DockerRegistryMirror:   defaultDockerRegistryMirror,
+				GitHubClient:           &github.Client{GithubBaseURL: githubBaseURL},
+				Scheme:                 scheme,
+			}
+			got, err := r.newPod(tc.runner)
+			require.NoError(t, err)
+			require.Equal(t, tc.want, got)
+		})
+	}
+}

controllers/runner_controller.go

@@ -106,15 +106,16 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			return ctrl.Result{}, nil
 		}
 	} else {
+		// Request to remove a runner. DeletionTimestamp was set in the runner - we need to unregister runner
 		var pod corev1.Pod
 		if err := r.Get(ctx, req.NamespacedName, &pod); err != nil {
 			if !kerrors.IsNotFound(err) {
 				log.Info(fmt.Sprintf("Retrying soon as we failed to get runner pod: %v", err))
 				return ctrl.Result{Requeue: true}, nil
 			}
+			// Pod was not found
+			return r.processRunnerDeletion(runner, ctx, log, nil)
 		}
-
-		// Request to remove a runner. DeletionTimestamp was set in the runner - we need to unregister runner
 		return r.processRunnerDeletion(runner, ctx, log, &pod)
 	}
 
@@ -132,7 +133,9 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		phase = "Created"
 	}
 
-	if runner.Status.Phase != phase {
+	ready := runnerPodReady(&pod)
+
+	if runner.Status.Phase != phase || runner.Status.Ready != ready {
 		if pod.Status.Phase == corev1.PodRunning {
 			// Seeing this message, you can expect the runner to become `Running` soon.
 			log.V(1).Info(
@@ -143,6 +146,7 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 		updated := runner.DeepCopy()
 		updated.Status.Phase = phase
+		updated.Status.Ready = ready
 		updated.Status.Reason = pod.Status.Reason
 		updated.Status.Message = pod.Status.Message
 
@@ -155,6 +159,18 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 	return ctrl.Result{}, nil
 }
 
+func runnerPodReady(pod *corev1.Pod) bool {
+	for _, c := range pod.Status.Conditions {
+		if c.Type != corev1.PodReady {
+			continue
+		}
+
+		return c.Status == corev1.ConditionTrue
+	}
+
+	return false
+}
+
 func runnerContainerExitCode(pod *corev1.Pod) *int32 {
 	for _, status := range pod.Status.ContainerStatuses {
 		if status.Name != containerName {
@@ -172,7 +188,7 @@ func runnerContainerExitCode(pod *corev1.Pod) *int32 {
 func runnerPodOrContainerIsStopped(pod *corev1.Pod) bool {
 	// If pod has ended up succeeded we need to restart it
 	// Happens e.g. when dind is in runner and run completes
-	stopped := pod.Status.Phase == corev1.PodSucceeded
+	stopped := pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed
 
 	if !stopped {
 		if pod.Status.Phase == corev1.PodRunning {
@@ -181,7 +197,7 @@ func runnerPodOrContainerIsStopped(pod *corev1.Pod) bool {
 					continue
 				}
 
-				if status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 {
+				if status.State.Terminated != nil {
 					stopped = true
 				}
 			}
@@ -437,6 +453,10 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		pod.Spec.HostAliases = runnerSpec.HostAliases
 	}
 
+	if runnerSpec.DnsConfig != nil {
+		pod.Spec.DNSConfig = runnerSpec.DnsConfig
+	}
+
 	if runnerSpec.RuntimeClassName != nil {
 		pod.Spec.RuntimeClassName = runnerSpec.RuntimeClassName
 	}

docs/releasenotes/0.23.md

@@ -0,0 +1,89 @@
+# actions-runner-controller v0.23.0
+
+All changes in this release can be found in the milestone https://github.com/actions-runner-controller/actions-runner-controller/milestone/3
+
+This log documents breaking and major enhancements
+## BREAKING CHANGE : Workflow job webhooks require an explicit field set
+
+Previously the webhook event workflow job was set as the default if no `githubEvent` was set.
+
+**Migration Steps**
+
+Change this:
+
+```yaml
+  scaleUpTriggers:
+  - githubEvent: {}
+    duration: "30m"
+```
+
+To this:
+
+```yaml
+  scaleUpTriggers:
+  - githubEvent:
+      workflowJob: {}
+    duration: "30m"
+```
+
+## BREAKING CHANGE : topologySpreadConstraints renamed to topologySpreadConstraint
+
+Previously to use the pod `topologySpreadConstraint:` attribute in your runners you had to set `topologySpreadConstraints:` instead, this was a typo and has been corrected.
+
+**Migration Steps**
+
+Update your runners to use `topologySpreadConstraints:` instead
+
+## BREAKING CHANGE : Default sync period is now 1 minute instead of 10 minutes
+
+Since caching as been implemented the default sync period of 10 minutes is unnecessarily conservative and gives a poor out of the box user experience. If you need a 10 minute sync period ensure you explicitly set this value.
+
+**Migration Steps**
+
+Update your sync period, how this is done will depend on how you've deployed ARC.
+
+## BREAKING CHANGE : A metric is set by default
+
+Previously if no metric was provided and you were using pull based scaling the `TotalNumberOfQueuedAndInProgressWorkflowRuns` was metric applied. No default is set now.
+
+**Migration Steps**
+
+Add in the `TotalNumberOfQueuedAndInProgressWorkflowRuns` metric where you are currenty relying on it
+
+```yaml
+
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runner-deployment
+spec:
+  template:
+    spec:
+      organisation: my-awesome-organisation
+      labels:
+        - my-awesome-runner
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    name: example-runner-deployment
+  minReplicas: 1
+  maxReplicas: 5
+  metrics:
+  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
+    repositoryNames:
+    - owner/my-awesome-repo-1
+    - owner/my-awesome-repo-2
+    - owner/my-awesome-repo-3
+```
+
+## ENHANCEMENT : Find runner groups that visible to repository using a single API call
+
+GitHub has contributed code to utilise a new API to enable us to get a repositories runner groups with a single API call. This enables us to scale runners based on the requesting repositories runner group membership without a series of expensive API queries.
+
+This is an opt-in feature currently as it's a significant change in behaviour if enabled, additionally, whilst scaling based on the repositories runner group membership is supported in both GHES and github.com, only github.com currently has access to the new raate-limit budget friendly API.
+
+To enable this set deploy via Helm and set `githubWebhookServer.useRunnerGroupsVisibility` to `true`.

github/github.go

@@ -153,8 +153,18 @@ func (c *Client) GetRegistrationToken(ctx context.Context, enterprise, org, repo
 	key := getRegistrationKey(org, repo, enterprise)
 	rt, ok := c.regTokens[key]
 
-	// we like to give runners a chance that are just starting up and may miss the expiration date by a bit
+	// We'd like to allow the runner just starting up to miss the expiration date by a bit.
-	runnerStartupTimeout := 3 * time.Minute
+	// Note that this means that we're going to cache Creation Registraion Token API response longer than the
+	// recommended cache duration.
+	//
+	// https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-a-repository
+	// https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-an-organization
+	// https://docs.github.com/en/rest/reference/actions#create-a-registration-token-for-an-enterprise
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#conditional-requests
+	//
+	// This is currently set to 30 minutes as the result of the discussion took place at the following issue:
+	// https://github.com/actions-runner-controller/actions-runner-controller/issues/1295
+	runnerStartupTimeout := 30 * time.Minute
 
 	if ok && rt.GetExpiresAt().After(time.Now().Add(runnerStartupTimeout)) {
 		return rt, nil
@@ -255,6 +265,29 @@ func (c *Client) ListOrganizationRunnerGroups(ctx context.Context, org string) (
 	return runnerGroups, nil
 }
 
+// ListOrganizationRunnerGroupsForRepository returns all the runner groups defined in the organization and
+// inherited to the organization from an enterprise.
+// We can remove this when google/go-github library is updated to support this.
+func (c *Client) ListOrganizationRunnerGroupsForRepository(ctx context.Context, org, repo string) ([]*github.RunnerGroup, error) {
+	var runnerGroups []*github.RunnerGroup
+
+	opts := github.ListOptions{PerPage: 100}
+	for {
+		list, res, err := c.listOrganizationRunnerGroupsVisibleToRepo(ctx, org, repo, &opts)
+		if err != nil {
+			return runnerGroups, fmt.Errorf("failed to list organization runner groups: %w", err)
+		}
+
+		runnerGroups = append(runnerGroups, list.RunnerGroups...)
+		if res.NextPage == 0 {
+			break
+		}
+		opts.Page = res.NextPage
+	}
+
+	return runnerGroups, nil
+}
+
 func (c *Client) ListRunnerGroupRepositoryAccesses(ctx context.Context, org string, runnerGroupId int64) ([]*github.Repository, error) {
 	var repos []*github.Repository
 
@@ -276,6 +309,42 @@ func (c *Client) ListRunnerGroupRepositoryAccesses(ctx context.Context, org stri
 	return repos, nil
 }
 
+// listOrganizationRunnerGroupsVisibleToRepo lists all self-hosted runner groups configured in an organization which can be used by the repository.
+//
+// GitHub API docs: https://docs.github.com/en/rest/reference/actions#list-self-hosted-runner-groups-for-an-organization
+func (c *Client) listOrganizationRunnerGroupsVisibleToRepo(ctx context.Context, org, repo string, opts *github.ListOptions) (*github.RunnerGroups, *github.Response, error) {
+	repoName := repo
+	parts := strings.Split(repo, "/")
+	if len(parts) == 2 {
+		repoName = parts[1]
+	}
+
+	u := fmt.Sprintf("orgs/%v/actions/runner-groups?visible_to_repository=%v", org, repoName)
+
+	if opts != nil {
+		if opts.PerPage > 0 {
+			u = fmt.Sprintf("%v&per_page=%v", u, opts.PerPage)
+		}
+
+		if opts.Page > 0 {
+			u = fmt.Sprintf("%v&page=%v", u, opts.Page)
+		}
+	}
+
+	req, err := c.Client.NewRequest("GET", u, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	groups := &github.RunnerGroups{}
+	resp, err := c.Client.Do(ctx, req, &groups)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return groups, resp, nil
+}
+
 // cleanup removes expired registration tokens.
 func (c *Client) cleanup() {
 	c.mu.Lock()

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v1.2.0
-	github.com/google/go-cmp v0.5.7
+	github.com/google/go-cmp v0.5.8
 	github.com/google/go-github/v39 v39.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
@@ -15,14 +15,14 @@ require (
 	github.com/onsi/gomega v1.17.0
 	github.com/prometheus/client_golang v1.12.1
 	github.com/stretchr/testify v1.7.0
-	github.com/teambition/rrule-go v1.7.2
+	github.com/teambition/rrule-go v1.8.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a
 	gomodules.xyz/jsonpatch/v2 v2.2.0
-	k8s.io/api v0.23.4
+	k8s.io/api v0.23.5
-	k8s.io/apimachinery v0.23.4
+	k8s.io/apimachinery v0.23.5
-	k8s.io/client-go v0.23.4
+	k8s.io/client-go v0.23.5
-	sigs.k8s.io/controller-runtime v0.11.1
+	sigs.k8s.io/controller-runtime v0.11.2
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -68,8 +68,8 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.23.0 // indirect
+	k8s.io/apiextensions-apiserver v0.23.5 // indirect
-	k8s.io/component-base v0.23.0 // indirect
+	k8s.io/component-base v0.23.5 // indirect
 	k8s.io/klog/v2 v2.30.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect

go.sum

@@ -224,6 +224,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
 github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ=
@@ -461,6 +463,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/teambition/rrule-go v1.7.2 h1:goEajFWYydfCgavn2m/3w5U+1b3PGqPUHx/fFSVfTy0=
 github.com/teambition/rrule-go v1.7.2/go.mod h1:mBJ1Ht5uboJ6jexKdNUJg2NcwP8uUMNvStWXlJD3MvU=
+github.com/teambition/rrule-go v1.8.0 h1:a/IX5s56hGkFF+nRlJUooZU/45OTeeldBGL29nDKIHw=
+github.com/teambition/rrule-go v1.8.0/go.mod h1:Ieq5AbrKGciP1V//Wq8ktsTXwSwJHDD5mD/wLBGl3p4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -947,18 +951,30 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg=
 k8s.io/api v0.23.4 h1:85gnfXQOWbJa1SiWGpE9EEtHs0UVvDyIsSMpEtl2D4E=
 k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI=
+k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA=
+k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
 k8s.io/apiextensions-apiserver v0.23.0 h1:uii8BYmHYiT2ZTAJxmvc3X8UhNYMxl2A0z0Xq3Pm+WY=
 k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4=
+k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI=
+k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ=
 k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc=
 k8s.io/apimachinery v0.23.4 h1:fhnuMd/xUL3Cjfl64j5ULKZ1/J9n8NuQEgNL+WXWfdM=
 k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
+k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0=
+k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
 k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4=
+k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw=
 k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
 k8s.io/client-go v0.23.4 h1:YVWvPeerA2gpUudLelvsolzH7c2sFoXXR5wM/sWqNFU=
 k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0=
+k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8=
+k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
 k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
+k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
 k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8=
 k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=
+k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE=
+k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
@@ -974,8 +990,11 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
 sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
 sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
+sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA=
+sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=

main.go

@@ -74,6 +74,7 @@ func main() {
 		syncPeriod           time.Duration
 
 		gitHubAPICacheDuration time.Duration
+		defaultScaleDownDelay  time.Duration
 
 		runnerImage            string
 		runnerImagePullSecrets stringSlice
@@ -111,7 +112,8 @@ func main() {
 	flag.StringVar(&c.BasicauthPassword, "github-basicauth-password", c.BasicauthPassword, "Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
 	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
 	flag.DurationVar(&gitHubAPICacheDuration, "github-api-cache-duration", 0, "DEPRECATED: The duration until the GitHub API cache expires. Setting this to e.g. 10m results in the controller tries its best not to make the same API call within 10m to reduce the chance of being rate-limited. Defaults to mostly the same value as sync-period. If you're tweaking this in order to make autoscaling more responsive, you'll probably want to tweak sync-period, too")
-	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change.")
+	flag.DurationVar(&defaultScaleDownDelay, "default-scale-down-delay", controllers.DefaultScaleDownDelay, "The approximate delay for a scale down followed by a scale up, used to prevent flapping (down->up->down->... loop)")
+	flag.DurationVar(&syncPeriod, "sync-period", 1*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled.")
 	flag.Var(&commonRunnerLabels, "common-runner-labels", "Runner labels in the K1=V1,K2=V2,... format that are inherited all the runners created by the controller. See https://github.com/actions-runner-controller/actions-runner-controller/issues/321 for more information")
 	flag.StringVar(&namespace, "watch-namespace", "", "The namespace to watch for custom resources. Set to empty for letting it watch for all namespaces.")
 	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
@@ -212,6 +214,7 @@ func main() {
 	log.Info(
 		"Initializing actions-runner-controller",
 		"github-api-cache-duration", gitHubAPICacheDuration,
+		"default-scale-down-delay", defaultScaleDownDelay,
 		"sync-period", syncPeriod,
 		"runner-image", runnerImage,
 		"docker-image", dockerImage,
@@ -225,6 +228,7 @@ func main() {
 		Scheme:                mgr.GetScheme(),
 		GitHubClient:          ghClient,
 		CacheDuration:         gitHubAPICacheDuration,
+		DefaultScaleDownDelay: defaultScaleDownDelay,
 	}
 
 	runnerPodReconciler := &controllers.RunnerPodReconciler{

runner/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.287.1
+ARG RUNNER_VERSION=2.290.1
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
@@ -10,10 +10,10 @@ RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update -y \
-    && apt install -y software-properties-common \
+    && apt-get install -y software-properties-common \
     && add-apt-repository -y ppa:git-core/ppa \
-    && apt update -y \
+    && apt-get update -y \
-    && apt install -y --no-install-recommends \
+    && apt-get install -y --no-install-recommends \
     build-essential \
     curl \
     ca-certificates \
@@ -66,7 +66,6 @@ RUN set -vx; \
     && usermod -aG docker runner \
     && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
 
-ENV RUNNER_ASSETS_DIR=/runnertmp
 ENV HOME=/home/runner
 
 # Uncomment the below COPY to use your own custom build of actions-runner.
@@ -83,7 +82,7 @@ ENV HOME=/home/runner
 #
 # If you're willing to uncomment the following line, you'd also need to comment-out the
 #   && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
-# line in the next `RUN` command in this Dockerfile, to avoid overwiding this runner.tar.gz with a remote one.
+# line in the next `RUN` command in this Dockerfile, to avoid overwiting this runner.tar.gz with a remote one.
 
 # COPY actions-runner-linux-x64-2.280.3.tar.gz /runnertmp/runner.tar.gz
 
@@ -92,6 +91,7 @@ ENV HOME=/home/runner
 # libyaml-dev is required for ruby/setup-ruby action.
 # It is installed after installdependencies.sh and before removing /var/lib/apt/lists
 # to avoid rerunning apt-update on its own.
+ENV RUNNER_ASSETS_DIR=/runnertmp
 RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
     && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "x86_64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x64 ; fi \
     && mkdir -p "$RUNNER_ASSETS_DIR" \
@@ -110,7 +110,9 @@ RUN mkdir /opt/hostedtoolcache \
     && chgrp docker /opt/hostedtoolcache \
     && chmod g+rwx /opt/hostedtoolcache
 
-COPY entrypoint.sh /
+# We place the scripts in `/usr/bin` so that users who extend this image can
+# override them with scripts of the same name placed in `/usr/local/bin`.
+COPY entrypoint.sh logger.bash /usr/bin/
 
 # Add the Python "User Script Directory" to the PATH
 ENV PATH="${PATH}:${HOME}/.local/bin"
@@ -122,4 +124,4 @@ RUN echo "PATH=${PATH}" > /etc/environment \
 USER runner
 
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
-CMD ["/entrypoint.sh"]
+CMD ["entrypoint.sh"]

runner/Dockerfile.dindrunner

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.287.1
+ARG RUNNER_VERSION=2.290.1
 ARG DOCKER_CHANNEL=stable
 ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
@@ -10,10 +10,10 @@ RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update -y \
-    && apt install -y software-properties-common \
+    && apt-get install -y software-properties-common \
     && add-apt-repository -y ppa:git-core/ppa \
-    && apt update -y \
+    && apt-get update -y \
-    && apt install -y --no-install-recommends \
+    && apt-get install -y --no-install-recommends \
     build-essential \
     curl \
     ca-certificates \
@@ -74,7 +74,6 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 	dockerd --version; \
 	docker --version
 
-ENV RUNNER_ASSETS_DIR=/runnertmp
 ENV HOME=/home/runner
 
 # Runner download supports amd64 as x64
@@ -82,6 +81,7 @@ ENV HOME=/home/runner
 # libyaml-dev is required for ruby/setup-ruby action.
 # It is installed after installdependencies.sh and before removing /var/lib/apt/lists
 # to avoid rerunning apt-update on its own.
+ENV RUNNER_ASSETS_DIR=/runnertmp
 RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
     && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "x86_64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x64 ; fi \
     && mkdir -p "$RUNNER_ASSETS_DIR" \
@@ -98,12 +98,11 @@ RUN mkdir /opt/hostedtoolcache \
     && chgrp docker /opt/hostedtoolcache \
     && chmod g+rwx /opt/hostedtoolcache
 
-COPY modprobe startup.sh /usr/local/bin/
+# We place the scripts in `/usr/bin` so that users who extend this image can
+# override them with scripts of the same name placed in `/usr/local/bin`.
+COPY entrypoint.sh logger.bash startup.sh /usr/bin/
 COPY supervisor/ /etc/supervisor/conf.d/
-COPY logger.sh /opt/bash-utils/logger.sh
+RUN chmod +x /usr/bin/startup.sh /usr/bin/entrypoint.sh
-COPY entrypoint.sh /usr/local/bin/
-
-RUN chmod +x /usr/local/bin/startup.sh /usr/local/bin/entrypoint.sh /usr/local/bin/modprobe
 
 # arch command on OS X reports "i386" for Intel CPUs regardless of bitness
 RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \

runner/Makefile

@@ -4,7 +4,7 @@ DIND_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind
 TAG ?= latest
 TARGETPLATFORM ?= $(shell arch)
 
-RUNNER_VERSION ?= 2.287.1
+RUNNER_VERSION ?= 2.290.1
 DOCKER_VERSION ?= 20.10.12
 
 # default list of platforms for which multiarch image is built
@@ -33,7 +33,8 @@ docker-push-ubuntu:
 	docker push ${DIND_RUNNER_NAME}:${TAG}
 
 docker-buildx-ubuntu:
-	export DOCKER_CLI_EXPERIMENTAL=enabled
+	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
+    export DOCKER_BUILDKIT=1
 	@if ! docker buildx ls | grep -q container-builder; then\
 		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
 	fi

runner/entrypoint.sh

@@ -1,51 +1,35 @@
 #!/bin/bash
+source logger.bash
 
 RUNNER_ASSETS_DIR=${RUNNER_ASSETS_DIR:-/runnertmp}
 RUNNER_HOME=${RUNNER_HOME:-/runner}
 
-LIGHTGREEN="\e[0;32m"
-LIGHTRED="\e[0;31m"
-WHITE="\e[0;97m"
-RESET="\e[0m"
-
-log(){
-  printf "${WHITE}${@}${RESET}\n" 1>&2
-}
-
-success(){
-  printf "${LIGHTGREEN}${@}${RESET}\n" 1>&2
-}
-
-error(){
-  printf "${LIGHTRED}${@}${RESET}\n" 1>&2
-}
-
 if [ ! -z "${STARTUP_DELAY_IN_SECONDS}" ]; then
-  log "Delaying startup by ${STARTUP_DELAY_IN_SECONDS} seconds"
+  log.notice "Delaying startup by ${STARTUP_DELAY_IN_SECONDS} seconds"
   sleep ${STARTUP_DELAY_IN_SECONDS}
 fi
 
 if [[ "${DISABLE_WAIT_FOR_DOCKER}" != "true" ]] && [[ "${DOCKER_ENABLED}" == "true" ]]; then
-    log "Docker enabled runner detected and Docker daemon wait is enabled"
+    log.debug 'Docker enabled runner detected and Docker daemon wait is enabled'
-    log "Waiting until Docker is avaliable or the timeout is reached"
+    log.debug 'Waiting until Docker is available or the timeout is reached'
     timeout 120s bash -c 'until docker ps ;do sleep 1; done'
 else
-  log "Docker wait check skipped. Either Docker is disabled or the wait is disabled, continuing with entrypoint"
+  log.notice 'Docker wait check skipped. Either Docker is disabled or the wait is disabled, continuing with entrypoint'
 fi
 
 if [ -z "${GITHUB_URL}" ]; then
-  log "Working with public GitHub"
+  log.debug 'Working with public GitHub'
   GITHUB_URL="https://github.com/"
 else
   length=${#GITHUB_URL}
   last_char=${GITHUB_URL:length-1:1}
 
   [[ $last_char != "/" ]] && GITHUB_URL="$GITHUB_URL/"; :
-  log "Github endpoint URL ${GITHUB_URL}"
+  log.debug "Github endpoint URL ${GITHUB_URL}"
 fi
 
 if [ -z "${RUNNER_NAME}" ]; then
-  error "RUNNER_NAME must be set"
+  log.error 'RUNNER_NAME must be set'
   exit 1
 fi
 
@@ -58,12 +42,12 @@ elif [ -n "${RUNNER_REPO}" ]; then
 elif [ -n "${RUNNER_ENTERPRISE}" ]; then
   ATTACH="enterprises/${RUNNER_ENTERPRISE}"
 else
-  error "At least one of RUNNER_ORG or RUNNER_REPO or RUNNER_ENTERPRISE must be set"
+  log.error 'At least one of RUNNER_ORG, RUNNER_REPO, or RUNNER_ENTERPRISE must be set'
   exit 1
 fi
 
 if [ -z "${RUNNER_TOKEN}" ]; then
-  error "RUNNER_TOKEN must be set"
+  log.error 'RUNNER_TOKEN must be set'
   exit 1
 fi
 
@@ -73,7 +57,7 @@ fi
 
 # Hack due to https://github.com/actions-runner-controller/actions-runner-controller/issues/252#issuecomment-758338483
 if [ ! -d "${RUNNER_HOME}" ]; then
-  error "${RUNNER_HOME} should be an emptyDir mount. Please fix the pod spec."
+  log.error "$RUNNER_HOME should be an emptyDir mount. Please fix the pod spec."
   exit 1
 fi
 
@@ -94,16 +78,16 @@ cd ${RUNNER_HOME}
 config_args=()
 if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" == "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
   config_args+=(--ephemeral)
-  echo "Passing --ephemeral to config.sh to enable the ephemeral runner."
+  log.debug 'Passing --ephemeral to config.sh to enable the ephemeral runner.'
 fi
 if [ "${DISABLE_RUNNER_UPDATE:-}" == "true" ]; then
   config_args+=(--disableupdate)
-  echo "Passing --disableupdate to config.sh to disable automatic runner updates."
+  log.debug 'Passing --disableupdate to config.sh to disable automatic runner updates.'
 fi
 
 retries_left=10
 while [[ ${retries_left} -gt 0 ]]; do
-  log "Configuring the runner."
+  log.debug 'Configuring the runner.'
   ./config.sh --unattended --replace \
     --name "${RUNNER_NAME}" \
     --url "${GITHUB_URL}${ATTACH}" \
@@ -113,18 +97,18 @@ while [[ ${retries_left} -gt 0 ]]; do
     --work "${RUNNER_WORKDIR}" "${config_args[@]}"
 
   if [ -f .runner ]; then
-    success "Runner successfully configured."
+    log.debug 'Runner successfully configured.'
     break
   fi
 
-  error "Configuration failed. Retrying"
+  log.debug 'Configuration failed. Retrying'
   retries_left=$((retries_left - 1))
   sleep 1
 done
 
 if [ ! -f .runner ]; then
   # we couldn't configure and register the runner; no point continuing
-  error "Configuration failed!"
+  log.error 'Configuration failed!'
   exit 2
 fi
 
@@ -151,7 +135,7 @@ cat .runner
 #     https://api.github.com/repos/USER/REPO/actions/runners/171
 
 if [ -z "${UNITTEST:-}" ]; then
-  mkdir ./externals
+  mkdir -p ./externals
   # Hack due to the DinD volumes
   mv ./externalstmp/* ./externals/
 fi
@@ -159,8 +143,10 @@ fi
 args=()
 if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" != "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
   args+=(--once)
-  echo "[WARNING] Passing --once is deprecated and will be removed as an option from the image and ARC at the release of 0.24.0."
+  log.warning 'Passing --once is deprecated and will be removed as an option' \
-  echo "[WARNING] Upgrade to GHES => 3.3 to continue using actions-runner-controller. If you are using github.com ignore this warning."
+    'from the image and actions-runner-controller at the release of 0.24.0.' \
+    'Upgrade to GHES => 3.3 to continue using actions-runner-controller. If' \
+    'you are using github.com ignore this warning.'
 fi
 
 # Unset entrypoint environment variables so they don't leak into the runner environment

runner/logger.bash

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# We are not using `set -Eeuo pipefail` here because this file is sourced by
+# other scripts that might not be ready for a strict Bash setup. The functions
+# in this file do not require it, because they are not handling signals, have
+# no external calls that can fail (printf as well as date failures are ignored),
+# are not using any variables that need to be set, and are not using any pipes.
+
+# This logger implementation can be replaced with another logger implementation
+# by placing a script called `logger.bash` in `/usr/local/bin` of the image. The
+# only requirement for the script is that it defines the following functions:
+#
+# - `log.debug`
+# - `log.notice`
+# - `log.warning`
+# - `log.error`
+# - `log.success`
+#
+# Each function **MUST** accept an arbitrary amount of arguments that make up
+# the (unstructured) logging message.
+#
+# Additionally the following environment variables **SHOULD** be supported to
+# disable their corresponding log entries, the value of the variables **MUST**
+# not matter the mere fact that they are set is all that matters:
+#
+# - `LOG_DEBUG_DISABLED`
+# - `LOG_NOTICE_DISABLED`
+# - `LOG_WARNING_DISABLED`
+# - `LOG_ERROR_DISABLED`
+# - `LOG_SUCCESS_DISABLED`
+
+# The log format is constructed in a way that it can easily be parsed with
+# standard tools and simple string manipulations; pattern and example:
+#
+#     YYYY-MM-DD hh:mm:ss.SSS  $level --- $message
+#     2022-03-19 10:01:23.172  NOTICE --- example message
+#
+# This function is an implementation detail and **MUST NOT** be called from
+# outside this script (which is possible if the file is sourced).
+__log() {
+  local color instant level
+
+  color=${1:?missing required <color> argument}
+  shift
+
+  level=${FUNCNAME[1]} # `main` if called from top-level
+  level=${level#log.} # substring after `log.`
+  level=${level^^} # UPPERCASE
+
+  if [[ ! -v "LOG_${level}_DISABLED" ]]; then
+    instant=$(date '+%F %T.%-3N' 2>/dev/null || :)
+
+    # https://no-color.org/
+    if [[ -v NO_COLOR ]]; then
+      printf -- '%s  %s --- %s\n' "$instant" "$level" "$*" 1>&2 || :
+    else
+      printf -- '\033[0;%dm%s  %s --- %s\033[0m\n' "$color" "$instant" "$level" "$*" 1>&2 || :
+    fi
+  fi
+}
+
+# To log with a dynamic level use standard Bash capabilities:
+#
+#     level=notice
+#     command || level=error
+#     "log.$level" message
+#
+# @formatter:off
+log.debug   () { __log 37 "$@"; } # white
+log.notice  () { __log 34 "$@"; } # blue
+log.warning () { __log 33 "$@"; } # yellow
+log.error   () { __log 31 "$@"; } # red
+log.success () { __log 32 "$@"; } # green
+# @formatter:on

runner/logger.sh

@@ -1,24 +0,0 @@
-#!/bin/sh
-# Logger from this post http://www.cubicrace.com/2016/03/log-tracing-mechnism-for-shell-scripts.html
-
-function INFO(){
-    local function_name="${FUNCNAME[1]}"
-    local msg="$1"
-    timeAndDate=`date`
-    echo "[$timeAndDate] [INFO] [${0}] $msg"
-}
-
-
-function DEBUG(){
-    local function_name="${FUNCNAME[1]}"
-    local msg="$1"
-    timeAndDate=`date`
-    echo "[$timeAndDate] [DEBUG] [${0}] $msg"
-}
-
-function ERROR(){
-    local function_name="${FUNCNAME[1]}"
-    local msg="$1"
-    timeAndDate=`date`
-    echo "[$timeAndDate] [ERROR]  $msg"
-}

runner/modprobe

@@ -1,21 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-# "modprobe" without modprobe
-# https://twitter.com/lucabruno/status/902934379835662336
-
-# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
-
-# Docker often uses "modprobe -va foo bar baz"
-# so we ignore modules that start with "-"
-for module; do
-	if [ "${module#-}" = "$module" ]; then
-		ip link show "$module" || true
-		lsmod | grep "$module" || true
-	fi
-done
-
-# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
-export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
-exec modprobe "$@"

runner/startup.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
-source /opt/bash-utils/logger.sh
+source logger.bash
 
 function wait_for_process () {
     local max_time_wait=30
     local process_name="$1"
     local waited_sec=0
     while ! pgrep "$process_name" >/dev/null && ((waited_sec < max_time_wait)); do
-        INFO "Process $process_name is not running yet. Retrying in 1 seconds"
+        log.debug "Process $process_name is not running yet. Retrying in 1 seconds"
-        INFO "Waited $waited_sec seconds of $max_time_wait seconds"
+        log.debug "Waited $waited_sec seconds of $max_time_wait seconds"
         sleep 1
         ((waited_sec=waited_sec+1))
         if ((waited_sec >= max_time_wait)); then
@@ -33,29 +33,32 @@ jq ".\"registry-mirrors\"[0] = \"${DOCKER_REGISTRY_MIRROR}\"" /etc/docker/daemon
 fi
 SCRIPT
 
-INFO "Using /etc/docker/daemon.json with the following content"
+dump() {
+  local path=${1:?missing required <path> argument}
+  shift
+  printf -- "%s\n---\n" "${*//\{path\}/"$path"}" 1>&2
+  cat "$path" 1>&2
+  printf -- '---\n' 1>&2
+}
 
-cat /etc/docker/daemon.json
+for config in /etc/docker/daemon.json /etc/supervisor/conf.d/dockerd.conf; do
+  dump "$config" 'Using {path} with the following content:'
+done
 
-INFO "Using /etc/supervisor/conf.d/dockerd.conf with the following content"
+log.debug 'Starting supervisor daemon'
-
-cat /etc/supervisor/conf.d/dockerd.conf
-
-INFO "Starting supervisor"
 sudo /usr/bin/supervisord -n >> /dev/null 2>&1 &
 
-INFO "Waiting for processes to be running"
+log.debug 'Waiting for processes to be running...'
 processes=(dockerd)
 
 for process in "${processes[@]}"; do
     wait_for_process "$process"
     if [ $? -ne 0 ]; then
-        ERROR "$process is not running after max time"
+        log.error "$process is not running after max time"
-        ERROR "Dumping /var/log/dockerd.err.log to help investigation"
+        dump /var/log/dockerd.err.log 'Dumping {path} to aid investigation'
-        cat /var/log/dockerd.err.log
         exit 1
     else
-        INFO "$process is running"
+        log.debug "$process is running"
     fi
 done
 

simulator/runnergroup_visibility.go

@@ -18,6 +18,22 @@ func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org,
 		panic(fmt.Sprintf("BUG: owner should not be empty in this context. repo=%v", repo))
 	}
 
+	if c.Client.GithubBaseURL == "https://github.com/" {
+		runnerGroups, err := c.Client.ListOrganizationRunnerGroupsForRepository(ctx, org, repo)
+		if err != nil {
+			return visible, err
+		}
+
+		for _, runnerGroup := range runnerGroups {
+			ref := NewRunnerGroupFromGitHub(runnerGroup)
+
+			if !managed.Includes(ref) {
+				continue
+			}
+
+			visible.Add(ref)
+		}
+	} else {
 		runnerGroups, err := c.Client.ListOrganizationRunnerGroups(ctx, org)
 		if err != nil {
 			return visible, err
@@ -43,6 +59,7 @@ func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org,
 
 			visible.Add(ref)
 		}
+	}
 
 	return visible, nil
 }

test/e2e/e2e_test.go

@@ -37,12 +37,22 @@ var (
 		},
 		{
 			Dockerfile: "../../runner/Dockerfile",
-			Args:       []testing.BuildArg{},
+			Args: []testing.BuildArg{
+				{
+					Name:  "RUNNER_VERSION",
+					Value: "2.289.2",
+				},
+			},
 			Image: runnerImage,
 		},
 		{
 			Dockerfile: "../../runner/Dockerfile.dindrunner",
-			Args:       []testing.BuildArg{},
+			Args: []testing.BuildArg{
+				{
+					Name:  "RUNNER_VERSION",
+					Value: "2.289.2",
+				},
+			},
 			Image: runnerDindImage,
 		},
 	}
@@ -58,7 +68,7 @@ var (
 	}
 
 	commonScriptEnv = []string{
-		"SYNC_PERIOD=" + "10s",
+		"SYNC_PERIOD=" + "30m",
 		"NAME=" + controllerImageRepo,
 		"VERSION=" + controllerImageTag,
 		"RUNNER_TAG=" + runnerImageTag,