Make logging more concise (#410)

This makes logging more concise by changing logger names to something like `controllers.Runner` to `actions-runner-controller.runner` after the standard `controller-rutime.controller` and reducing redundant logs by removing unnecessary requeues. I have also tweaked log messages so that their style is more consistent, which will also help readability. Also, runnerreplicaset-controller lacked useful logs so I have enhanced it.

.github/workflows/build-and-release-runners.yml

@@ -1,10 +1,12 @@
+name: Build and Release Runners
+
 on:
   pull_request:
     branches:
       - '**'
     paths:
       - 'runner/**'
-      - .github/workflows/build-runner.yml
+      - .github/workflows/build-and-release-runners.yml
   push:
     branches:
       - master
@@ -13,8 +15,8 @@ on:
       - runner/Dockerfile
       - runner/dindrunner.Dockerfile
       - runner/entrypoint.sh
-      - .github/workflows/build-runner.yml
-name: Runner
+      - .github/workflows/build-and-release-runners.yml
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -27,10 +29,14 @@ jobs:
           - name: actions-runner-dind
             dockerfile: dindrunner.Dockerfile
     env:
-      RUNNER_VERSION: 2.275.1
+      RUNNER_VERSION: 2.277.1
       DOCKER_VERSION: 19.03.12
       DOCKERHUB_USERNAME: ${{ github.repository_owner }}
     steps:
+      - name: Set outputs
+        id: vars
+        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
+
       - name: Checkout
         uses: actions/checkout@v2
 
@@ -44,12 +50,12 @@ jobs:
 
       - name: Login to DockerHub
         uses: docker/login-action@v1
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name == 'push' || github.event_name == 'release' }}
         with:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
-      - name: Build [and Push]
+      - name: Build and Push
         uses: docker/build-push-action@v2
         with:
           context: ./runner
@@ -61,4 +67,5 @@ jobs:
             DOCKER_VERSION=${{ env.DOCKER_VERSION }}
           tags: |
             ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ steps.vars.outputs.sha_short }}
             ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:latest

.github/workflows/on-push-lint-charts.yml

@@ -0,0 +1,75 @@
+name: Lint and Test Charts
+
+on:
+  push:
+    paths:
+      - 'charts/**'
+      - '.github/**'
+  workflow_dispatch:
+
+env:
+  KUBE_SCORE_VERSION: 1.10.0
+  HELM_VERSION: v3.4.1
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: ${{ env.HELM_VERSION }}
+
+      - name: Set up kube-score
+        run: |
+          wget https://github.com/zegl/kube-score/releases/download/v${{ env.KUBE_SCORE_VERSION }}/kube-score_${{ env.KUBE_SCORE_VERSION }}_linux_amd64 -O kube-score
+          chmod 755 kube-score
+
+      - name: Kube-score generated manifests
+        run: helm template  --values charts/.ci/values-kube-score.yaml charts/* | ./kube-score score -
+              --ignore-test pod-networkpolicy
+              --ignore-test deployment-has-poddisruptionbudget
+              --ignore-test deployment-has-host-podantiaffinity
+              --ignore-test container-security-context
+              --ignore-test pod-probes
+              --ignore-test container-image-tag
+              --enable-optional-test container-security-context-privileged
+              --enable-optional-test container-security-context-readonlyrootfilesystem
+
+      # python is a requirement for the chart-testing action below (supports yamllint among other tests)
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.0.1
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config charts/.ci/ct-config.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Run chart-testing (lint)
+        run: ct lint --config charts/.ci/ct-config.yaml
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1.0.0
+        if: steps.list-changed.outputs.changed == 'true'
+
+      # We need cert-manager already installed in the cluster because we assume the CRDs exist
+      - name: Install cert-manager
+        run: |
+          helm repo add jetstack https://charts.jetstack.io --force-update
+          helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
+        if: steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart-testing (install)
+        run: ct install --config charts/.ci/ct-config.yaml

.github/workflows/on-push-master-publish-chart.yml

@@ -0,0 +1,101 @@
+name: Publish helm chart
+
+on:
+  push:
+    branches:
+      - master
+      - main # assume that the branch name may change in future
+    paths:
+      - 'charts/**'
+      - '.github/**'
+  workflow_dispatch:
+
+env:
+  KUBE_SCORE_VERSION: 1.10.0
+  HELM_VERSION: v3.4.1
+
+jobs:
+  lint-chart:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: ${{ env.HELM_VERSION }}
+
+      - name: Set up kube-score
+        run: |
+          wget https://github.com/zegl/kube-score/releases/download/v${{ env.KUBE_SCORE_VERSION }}/kube-score_${{ env.KUBE_SCORE_VERSION }}_linux_amd64 -O kube-score
+          chmod 755 kube-score
+
+      - name: Kube-score generated manifests
+        run: helm template  --values charts/.ci/values-kube-score.yaml charts/* | ./kube-score score -
+              --ignore-test pod-networkpolicy
+              --ignore-test deployment-has-poddisruptionbudget
+              --ignore-test deployment-has-host-podantiaffinity
+              --ignore-test container-security-context
+              --ignore-test pod-probes
+              --ignore-test container-image-tag
+              --enable-optional-test container-security-context-privileged
+              --enable-optional-test container-security-context-readonlyrootfilesystem
+
+      # python is a requirement for the chart-testing action below (supports yamllint among other tests)
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.0.1
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config charts/.ci/ct-config.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Run chart-testing (lint)
+        run: ct lint --config charts/.ci/ct-config.yaml
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1.0.0
+        if: steps.list-changed.outputs.changed == 'true'
+
+      # We need cert-manager already installed in the cluster because we assume the CRDs exist
+      - name: Install cert-manager
+        run: |
+          helm repo add jetstack https://charts.jetstack.io --force-update
+          helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
+        if: steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart-testing (install)
+        run: ct install --config charts/.ci/ct-config.yaml
+        if: steps.list-changed.outputs.changed == 'true'
+
+  publish-chart:
+
+    runs-on: ubuntu-latest
+    needs: lint-chart
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.1.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+

.github/workflows/release.yml

@@ -9,6 +9,10 @@ jobs:
     env:
       DOCKERHUB_USERNAME: ${{ github.repository_owner }}
     steps:
+      - name: Set outputs
+        id: vars
+        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
+
       - name: Checkout
         uses: actions/checkout@v2
 
@@ -52,5 +56,8 @@ jobs:
           file: Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }} 
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
 

.github/workflows/test.yaml

@@ -6,6 +6,7 @@ on:
       - master
     paths-ignore:
     - 'runner/**'
+    - .github/workflows/build-and-release-runners.yml
 
 jobs:
   test:

.github/workflows/wip.yml

@@ -30,11 +30,13 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
+      # Considered unstable builds
+      # See Issue #285, PR #286, and PR #323 for more information
       - name: Build and Push
         uses: docker/build-push-action@v2
         with:
           file: Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest 
-
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary

.gitignore

@@ -26,3 +26,6 @@ bin
 
 .envrc
 *.pem
+
+# OS
+.DS_STORE

Dockerfile

@@ -22,7 +22,8 @@ COPY . .
 RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
   export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
   GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \
-  go build -a -o manager main.go
+  go build -a -o manager main.go && \
+  go build -a -o github-webhook-server ./cmd/githubwebhookserver
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -31,6 +32,7 @@ FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 
 COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/github-webhook-server .
 
 USER nonroot:nonroot
 

README.md

@@ -1,7 +1,33 @@
 # actions-runner-controller
 
+[![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners)
+
 This controller operates self-hosted runners for GitHub Actions on your Kubernetes cluster.
 
+ToC:
+
+- [Motivation](#motivation)
+- [Installation](#installation)
+  - [GitHub Enterprise support](#github-enterprise-support)
+- [Setting up authentication with GitHub API](#setting-up-authentication-with-github-api)
+  - [Deploying using GitHub App Authentication](#deploying-using-github-app-authentication)
+  - [Deploying using PAT Authentication](#deploying-using-pat-authentication)
+- [Usage](#usage)
+  - [Repository Runners](#repository-runners)
+  - [Organization Runners](#organization-runners)
+  - [Runner Deployments](#runnerdeployments)
+    - [Autoscaling](#autoscaling)
+      - [Faster Autoscaling with GitHub Webhook](#faster-autoscaling-with-github-webhook)
+  - [Runner with DinD](#runner-with-dind)
+  - [Additional tweaks](#additional-tweaks)
+  - [Runner labels](#runner-labels)
+  - [Runner groups](#runner-groups)
+  - [Using EKS IAM role for service accounts](#using-eks-iam-role-for-service-accounts)
+  - [Software installed in the runner image](#software-installed-in-the-runner-image)
+  - [Common errors](#common-errors)
+- [Developing](#developing)
+- [Alternatives](#alternatives)
+
 ## Motivation
 
 [GitHub Actions](https://github.com/features/actions) is a very useful tool for automating development. GitHub Actions jobs are run in the cloud by default, but you may want to run your jobs in your environment. [Self-hosted runner](https://github.com/actions/runner) can be used for such use cases, but requires the provisioning and configuration of a virtual machine instance. Instead if you already have a Kubernetes cluster, it makes more sense to run the self-hosted runner on top of it.
@@ -14,38 +40,81 @@ actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/insta
 
 - [Installing cert-manager on Kubernetes](https://cert-manager.io/docs/installation/kubernetes/)
 
-Install the custom resource and actions-runner-controller itself. This will create actions-runner-system namespace in your Kubernetes and deploy the required resources.
+Install the custom resource and actions-runner-controller with `kubectl` or `helm`. This will create actions-runner-system namespace in your Kubernetes and deploy the required resources.
 
+`kubectl`:
+
+```shell
+# REPLACE "v0.17.0" with the version you wish to deploy
+kubectl apply -f https://github.com/summerwind/actions-runner-controller/releases/download/v0.17.0/actions-runner-controller.yaml
 ```
-kubectl apply -f https://github.com/summerwind/actions-runner-controller/releases/latest/download/actions-runner-controller.yaml
+
+`helm`:
+
+```shell
+helm repo add actions-runner-controller https://summerwind.github.io/actions-runner-controller
+helm upgrade --install -n actions-runner-system actions-runner-controller/actions-runner-controller
 ```
 
 ### Github Enterprise support
 
-If you use either Github Enterprise Cloud or Server (and have recent enought version supporting Actions), you can use **actions-runner-controller**  with those, too. Authentication works same way as with public Github (repo and organization level).
+If you use either Github Enterprise Cloud or Server, you can use **actions-runner-controller**  with those, too.
+Authentication works same way as with public Github (repo and organization level).
+The minimum version of Github Enterprise Server is 3.0.0 (or rc1/rc2).
+__**NOTE : The maintainers do not have an Enterprise environment to be able to test changes and so are reliant on the community for testing, support is a best endeavors basis only and is community driven**__
 
 ```shell
 kubectl set env deploy controller-manager -c manager GITHUB_ENTERPRISE_URL=<GHEC/S URL> --namespace actions-runner-system
 ```
 
-[Enterprise level](https://docs.github.com/en/enterprise-server@2.22/actions/hosting-your-own-runners/adding-self-hosted-runners#adding-a-self-hosted-runner-to-an-enterprise) runners are not working yet as there's no API definition for those.
+#### Enterprise runners usage
+
+In order to use enterprise runners you must have Admin access to Github Enterprise and you should do Personal Access Token (PAT)
+with `enterprise:admin` access. Enterprise runners are not possible to run with Github APP or any other permission.
+
+When you use enterprise runners those will get access to Github Organisations. However, access to the repositories is **NOT**
+allowed by default. Each Github Organisation must allow Enterprise runner groups to be used in repositories.
+This is needed only one time and is permanent after that.
+
+Example:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: ghe-runner-deployment
+spec:
+  replicas: 2
+  template:
+    spec:
+      enterprise: your-enterprise-name
+      dockerdWithinRunnerContainer: true
+      resources:
+        limits:
+          cpu: "4000m"
+          memory: "2Gi"
+        requests:
+          cpu: "200m"
+          memory: "200Mi"
+      volumeMounts:
+      - mountPath: /runner
+        name: runner
+      volumes:
+      - name: runner
+        emptyDir: {}
+
+```
 
 ## Setting up authentication with GitHub API
 
-There are two ways for actions-runner-controller to authenticate with the GitHub API:
+There are two ways for actions-runner-controller to authenticate with the GitHub API (only 1 can be configured at a time however):
 
 1. Using GitHub App.
 2. Using Personal Access Token.
 
-Regardless of which authentication method you use, the same permissions are required, those permissions are:
-- Repository: Administration (read/write)
-- Repository: Actions (read)
-- Organization: Self-hosted runners (read/write)
+Functionality wise there isn't a difference between the 2 authentication methods. There are however some benefits to using a GitHub App for authentication over a PAT such as an [increased API quota](https://docs.github.com/en/developers/apps/rate-limits-for-github-apps), if you run into rate limiting consider deploying this solution using GitHub App authentication instead.
 
-
-**NOTE: It is extremely important to only follow one of the sections below and not both.**
-
-### Using GitHub App
+### Deploying using GitHub App Authentication
 
 You can create a GitHub App for either your account or any organization. If you want to create a GitHub App for your account, open the following link to the creation page, enter any unique name in the "GitHub App name" field, and hit the "Create GitHub App" button at the bottom of the page.
 
@@ -82,19 +151,29 @@ $ kubectl create secret generic controller-manager \
     --from-file=github_app_private_key=${PRIVATE_KEY_FILE_PATH}
 ```
 
-### Using Personal Access Token
+### Deploying using PAT Authentication
 
-From an account that has `admin` privileges for the repository, create a [personal access token](https://github.com/settings/tokens) with `repo` scope. This token is used to register a self-hosted runner by *actions-runner-controller*.
+Personal Acess Token can be used to register a self-hosted runner by *actions-runner-controller*.
 
-Self-hosted runners in GitHub can either be connected to a single repository, or to a GitHub organization (so they are available to all repositories in the organization). This token is used to register a self-hosted runner by *actions-runner-controller*.
+Self-hosted runners in GitHub can either be connected to a single repository, or to a GitHub organization (so they are available to all repositories in the organization). How you plan on using the runner will affect what scopes are needed for the token.
 
-For adding a runner to a repository, the token should have `repo` scope. If the runner should be added to an organization, the token should have `admin:org` scope. Note that to use a Personal Access Token, you must issue the token with an account that has `admin` privileges (on the repository and/or the organization).
+Log-in to a GitHub account that has `admin` privileges for the repository, and [create a personal access token](https://github.com/settings/tokens/new) with the appropriate scopes listed below:
 
-Open the Create Token page from the following link, grant the `repo` and/or `admin:org` scope, and press the "Generate Token" button at the bottom of the page to create the token.
+**Scopes for a Repository Runner**
 
-- [Create personal access token](https://github.com/settings/tokens/new)
+* repo (Full control)
 
-Register the created token (`GITHUB_TOKEN`) as a Kubernetes secret.
+**Scopes for a Organisation Runner**
+
+* repo (Full control)
+* admin:org (Full control)
+* admin:public_key - read:public_key
+* admin:repo_hook - read:repo_hook
+* admin:org_hook
+* notifications
+* workflow
+
+Once you have created the appropriate token, deploy it as a secret to your kubernetes cluster that you are going to deploy the solution on:
 
 ```shell
 kubectl create secret generic controller-manager \
@@ -109,7 +188,7 @@ There are two ways to use this controller:
 - Manage runners one by one with `Runner`.
 - Manage a set of runners with `RunnerDeployment`.
 
-### Repository runners
+### Repository Runners
 
 To launch a single self-hosted runner, you need to create a manifest file includes *Runner* resource as follows. This example launches a self-hosted runner with name *example-runner* for the *summerwind/actions-runner-controller* repository.
 
@@ -207,9 +286,33 @@ example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 
 #### Autoscaling
 
-`RunnerDeployment` can scale the number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
+A `RunnerDeployment` can scale the number of runners between `minReplicas` and `maxReplicas` fields based the chosen scaling metric as defined in the `metrics` attribute
+
+**Scaling Metrics**
+
+**TotalNumberOfQueuedAndInProgressWorkflowRuns**
+
+In the below example, `actions-runner` will poll GitHub for all pending workflows with the poll period defined by the sync period configuration. It will then scale to e.g. 3 if there're 3 pending jobs at sync time.
+With this scaling metric we are required to define a list of repositories within our metric.
+
+The scale out performance is controlled via the manager containers startup `--sync-period` argument. The default value is set to 10 minutes to prevent default deployments rate limiting themselves from the GitHub API.
+
+**Kustomize Config :** The period can be customised in the `config/default/manager_auth_proxy_patch.yaml` patch<br />
+**Helm Config :** `syncPeriod`
+
+**Benefits of this metric**
+1. Supports named repositories allowing you to restrict the runner to a specified set of repositories server side.
+2. Scales the runner count based on the actual queue depth of the jobs meaning a more 1:1 scaling of runners to queued jobs.
+3. Like all scaling metrics, you can manage workflow allocation to the RunnerDeployment through the use of [Github labels](#runner-labels).
+
+**Drawbacks of this metric**
+1. Repositories must be named within the scaling metric, maintaining a list of repositories may not be viable in larger environments or self-serve environments.
+2. May not scale quick enough for some users needs. This metric is pull based and so the queue depth is polled as configured by the sync period, as a result scaling performance is bound by this sync period meaning there is a lag to scaling activity.
+3. Relatively large amounts of API requests required to maintain this metric, you may run in API rate limiting issues depending on the size of your environment and how aggressive your sync period configuration is
+
+
+Example `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`
 
-In the below example, `actions-runner` checks for pending workflow runs for each sync period, and scale to e.g. 3 if there're 3 pending jobs at sync time.
 
 ```yaml
 apiVersion: actions.summerwind.dev/v1alpha1
@@ -236,38 +339,34 @@ spec:
     - summerwind/actions-runner-controller
 ```
 
-The scale out performance is controlled via the manager containers startup `--sync-period` argument. The default value is 10 minutes to prevent unconfigured deployments rate limiting themselves from the GitHub API. The period can be customised in the `config/default/manager_auth_proxy_patch.yaml` patch for those that are building the solution via the kustomize setup.
-
-Additionally, the autoscaling feature has an anti-flapping option that prevents periodic loop of scaling up and down.
-By default, it doesn't scale down until the grace period of 10 minutes passes after a scale up. The grace period can be configured by setting `scaleDownDelaySecondsAfterScaleUp`:
+Additionally, the `HorizontalRunnerAutoscaler` also has an anti-flapping option that prevents periodic loop of scaling up and down.
+By default, it doesn't scale down until the grace period of 10 minutes passes after a scale up. The grace period can be configured however by adding the setting `scaleDownDelaySecondsAfterScaleOut` in the `HorizontalRunnerAutoscaler` `spec`:
 
 ```yaml
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerDeployment
-metadata:
-  name: example-runner-deployment
 spec:
-  template:
-    spec:
-      repository: summerwind/actions-runner-controller
----
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: HorizontalRunnerAutoscaler
-metadata:
-  name: example-runner-deployment-autoscaler
-spec:
-  scaleTargetRef:
-    name: example-runner-deployment
-  minReplicas: 1
-  maxReplicas: 3
   scaleDownDelaySecondsAfterScaleOut: 60
-  metrics:
-  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
-    repositoryNames:
-    - summerwind/actions-runner-controller
 ```
 
-If you do not want to manage an explicit list of repositories to scale, an alternate autoscaling scheme that can be applied is the PercentageRunnersBusy scheme. The number of desired pods are evaulated by checking how many runners are currently busy and applying a scaleup or scale down factor if certain thresholds are met. By setting the metric type to PercentageRunnersBusy, the HorizontalRunnerAutoscaler will query github for the number of busy runners which live in the RunnerDeployment namespace. Scaleup and scaledown thresholds are the percentage of busy runners at which the number of desired runners are re-evaluated. Scaleup and scaledown factors are the multiplicative factor applied to the current number of runners used to calculate the number of desired runners. This scheme is also especially useful if you want multiple controllers in various clusters, each responsible for scaling their own runner pods per namespace.
+**PercentageRunnersBusy**
+
+The `HorizontalRunnerAutoscaler` will poll GitHub based on the configuration sync period for the number of busy runners which live in the RunnerDeployment's namespace and scale based on the settings
+
+**Kustomize Config :** The period can be customised in the `config/default/manager_auth_proxy_patch.yaml` patch<br />
+**Helm Config :** `syncPeriod`
+
+**Benefits of this metric**
+1. Supports named repositories server side the same as the `TotalNumberOfQueuedAndInProgressWorkflowRuns` metric [#313](https://github.com/summerwind/actions-runner-controller/pull/313)
+2. Supports GitHub organisation wide scaling without maintaining an explicit list of repositories, this is especially useful for those that are working at a larger scale. [#223](https://github.com/summerwind/actions-runner-controller/pull/223)
+3. Like all scaling metrics, you can manage workflow allocation to the RunnerDeployment through the use of [Github labels](#runner-labels)
+4. Supports scaling desired runner count on both a percentage increase / decrease basis as well as on a fixed increase / decrease count basis [#223](https://github.com/summerwind/actions-runner-controller/pull/223) [#315](https://github.com/summerwind/actions-runner-controller/pull/315)
+
+**Drawbacks of this metric**
+1. May not scale quick enough for some users needs. This metric is pull based and so the number of busy runners are polled as configured by the sync period, as a result scaling performance is bound by this sync period meaning there is a lag to scaling activity.
+2. We are scaling up and down based on indicative information rather than a count of the actual number of queued jobs and so the desired runner count is likely to under provision new runners or overprovision them relative to actual job queue depth, this may or may not be a problem for you.
+
+
+Examples of each scaling type implemented with a `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`:
+
 
 ```yaml
 ---
@@ -280,16 +379,153 @@ spec:
     name: example-runner-deployment
   minReplicas: 1
   maxReplicas: 3
-  scaleDownDelaySecondsAfterScaleOut: 60
   metrics:
   - type: PercentageRunnersBusy
-    scaleUpThreshold: '0.75'
-    scaleDownThreshold: '0.3'
-    scaleUpFactor: '1.4'
-    scaleDownFactor: '0.7'
+    scaleUpThreshold: '0.75'    # The percentage of busy runners at which the number of desired runners are re-evaluated to scale up
+    scaleDownThreshold: '0.3'   # The percentage of busy runners at which the number of desired runners are re-evaluated to scale down
+    scaleUpFactor: '1.4'        # The scale up multiplier factor applied to desired count
+    scaleDownFactor: '0.7'      # The scale down multiplier factor applied to desired count
 ```
 
-## Runner with DinD
+```yaml
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    name: example-runner-deployment
+  minReplicas: 1
+  maxReplicas: 3
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'    # The percentage of busy runners at which the number of desired runners are re-evaluated to scale up
+    scaleDownThreshold: '0.3'   # The percentage of busy runners at which the number of desired runners are re-evaluated to scale down
+    ScaleUpAdjustment: '2'      # The scale up runner count added to desired count
+    ScaleDownAdjustment: '1'    # The scale down runner count subtracted from the desired count
+```
+
+Like the previous metric, the scale down factor respects the anti-flapping configuration is applied to the `HorizontalRunnerAutoscaler` as mentioned previously:
+
+```yaml
+spec:
+  scaleDownDelaySecondsAfterScaleOut: 60
+```
+
+#### Faster Autoscaling with GitHub Webhook
+
+> This feature is an ADVANCED feature which may require more work to set up.
+> Please get prepared to put some time and effort to learn and leverage this feature!
+
+`actions-runner-controller` has an optional Webhook server that receives GitHub Webhook events and scale
+[`RunnerDeployment`s](#runnerdeployments) by updating corresponding [`HorizontalRunnerAutoscaler`s](#autoscaling).
+
+Today, the Webhook server can be configured to respond GitHub `check_run`, `pull_request`, and `push` events
+by scaling up the matching `HorizontalRunnerAutoscaler` by N replica(s), where `N` is configurable within
+`HorizontalRunerAutoscaler`'s `Spec`.
+
+More concretely, you can configure the targeted GitHub event types and the `N` in
+`scaleUpTriggers`:
+
+```yaml
+kind: HorizontalRunnerAutoscaler
+spec:
+  scaleTargetRef:
+    name: myrunners
+  scaleUpTriggers:
+  - githubEvent:
+      checkRun:
+        types: ["created"]
+        status: "queued"
+    amount: 1
+    duration: "5m"
+```
+
+With the above example, the webhook server scales `myrunners` by `1` replica for 5 minutes on each `check_run` event
+with the type of `created` and the status of `queued` received.
+
+The primary benefit of autoscaling on Webhook compared to the standard autoscaling is that this one allows you to
+immediately add "resource slack" for future GitHub Actions job runs.
+
+In contrast, the standard autoscaling requires you to wait next sync period to add
+insufficient runners. You can definitely shorten the sync period to make the standard autoscaling more responsive.
+But doing so eventually result in the controller not functional due to GitHub API rate limit.
+
+> You can learn the implementation details in #282
+
+To enable this feature, you firstly need to install the webhook server.
+
+Currently, only our Helm chart has the ability install it.
+
+```console
+$ helm --upgrade install actions-runner-controller/actions-runner-controller \
+  githubWebhookServer.enabled=true \
+  githubWebhookServer.ports[0].nodePort=33080
+```
+
+The above command will result in exposing the node port 33080 for Webhook events. Usually, you need to create an
+external loadbalancer targeted to the node port, and register the hostname or the IP address of the external loadbalancer
+to the GitHub Webhook.
+
+Once you were able to confirm that the Webhook server is ready and running from GitHub - this is usually verified by the
+GitHub sending PING events to the Webhook server - create or update your `HorizontalRunnerAutoscaler` resources
+by learning the following configuration examples.
+
+- [Example 1: Scale up on each `check_run` event](#example-1-scale-up-on-each-check_run-event)
+- [Example 2: Scale on each `pull_request` event against `develop` or `main` branches](#example-2-scale-on-each-pull_request-event-against-develop-or-main-branches)
+
+##### Example 1: Scale up on each `check_run` event
+
+> Note: This should work almost like https://github.com/philips-labs/terraform-aws-github-runner
+
+To scale up replicas of the runners for `example/myrepo` by 1 for 5 minutes on each `check_run`, you write manifests like the below:
+
+```yaml
+kind: RunnerDeployment
+metadata:
+   name: myrunners
+spec:
+  repository: example/myrepo
+---
+kind: HorizontalRunnerAutoscaler
+spec:
+  scaleTargetRef:
+    name: myrunners
+  scaleUpTriggers:
+  - githubEvent:
+      checkRun:
+        types: ["created"]
+        status: "queued"
+    amount: 1
+    duration: "5m"
+```
+
+###### Example 2: Scale on each `pull_request` event against `develop` or `main` branches
+
+```yaml
+kind: RunnerDeployment:
+metadata:
+   name: myrunners
+spec:
+  repository: example/myrepo
+---
+kind: HorizontalRunnerAutoscaler
+spec:
+  scaleTargetRef:
+    name: myrunners
+  scaleUpTriggers:
+  - githubEvent:
+      pullRequest:
+        types: ["synchronize"]
+        branches: ["main", "develop"]
+    amount: 1
+    duration: "5m"
+```
+
+See ["activity types"](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request) for the list of valid values for `scaleUpTriggers[].githubEvent.pullRequest.types`.
+
+### Runner with DinD
 
 When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
 
@@ -311,7 +547,7 @@ spec:
 
 This also helps with resources, as you don't need to give resources separately to docker and runner.
 
-## Additional tweaks
+### Additional tweaks
 
 You can pass details through the spec selector. Here's an eg. of what you may like to do:
 
@@ -343,12 +579,20 @@ spec:
         requests:
           cpu: "2.0"
           memory: "4Gi"
+      # Timeout after a node crashed or became unreachable to evict your pods somewhere else (default 5mins)
+      tolerations:
+        - key: "node.kubernetes.io/unreachable"
+          operator: "Exists"
+          effect: "NoExecute"
+          tolerationSeconds: 10
+      # true (default) = A privileged docker sidecar container is included in the runner pod.
+      # false = A docker sidecar container is not included in the runner pod and you can't use docker.
       # If set to false, there are no privileged container and you cannot use docker.
       dockerEnabled: false
-      # If set to true, runner pod container only 1 container that's expected to be able to run docker, too.
-      # image summerwind/actions-runner-dind or custom one should be used with true -value
-      dockerdWithinRunnerContainer: false
-      # Valid if dockerdWithinRunnerContainer is not true
+      # false (default) = Docker support is provided by a sidecar container deployed in the runner pod.
+      # true = No docker sidecar container is deployed in the runner pod but docker can be used within teh runner container instead. The image summerwind/actions-runner-dind is used by default.
+      dockerdWithinRunnerContainer: true
+      # Docker sidecar container image tweaks examples below, only applicable if dockerdWithinRunnerContainer = false
       dockerdContainerResources:
         limits:
           cpu: "4.0"
@@ -356,6 +600,7 @@ spec:
         requests:
           cpu: "2.0"
           memory: "4Gi"
+      # Additional N number of sidecar containers
       sidecarContainers:
         - name: mysql
           image: mysql:5.7
@@ -364,13 +609,13 @@ spec:
               value: abcd1234
           securityContext:
             runAsUser: 0
-      # if workDir is not specified, the default working directory is /runner/_work
-      # this setting allows you to customize the working directory location
+      # workDir if not specified (default = /runner/_work)
+      # You can customise this setting allowing you to change the default working directory location
       # for example, the below setting is the same as on the ubuntu-18.04 image
       workDir: /home/runner/work
 ```
 
-## Runner labels
+### Runner labels
 
 To run a workflow job on a self-hosted runner, you can use the following syntax in your workflow:
 
@@ -407,7 +652,7 @@ jobs:
 
 Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
 
-## Runner Groups
+### Runner Groups
 
 Runner groups can be used to limit which repositories are able to use the GitHub Runner at an Organisation level. Runner groups have to be [created in GitHub first](https://docs.github.com/en/actions/hosting-your-own-runners/managing-access-to-self-hosted-runners-using-groups) before they can be referenced.
 
@@ -426,7 +671,7 @@ spec:
       group: NewGroup
 ```
 
-## Using EKS IAM role for service accounts
+### Using EKS IAM role for service accounts
 
 `actions-runner-controller` v0.15.0 or later has support for EKS IAM role for service accounts.
 
@@ -452,7 +697,7 @@ spec:
         fsGroup: 1447
 ```
 
-## Software installed in the runner image
+### Software installed in the runner image
 
 The GitHub hosted runners include a large amount of pre-installed software packages. For Ubuntu 18.04, this list can be found at <https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md>
 
@@ -487,9 +732,9 @@ spec:
   image: YOUR_CUSTOM_DOCKER_IMAGE
 ```
 
-## Common Errors
+### Common Errors
 
-### invalid header field value
+#### invalid header field value
 
 ```json
 2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	{"controller": "runner", "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9", "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"}
@@ -543,9 +788,11 @@ NAME=$DOCKER_USER/actions-runner-controller \
 The following is a list of alternative solutions that may better fit you depending on your use-case:
 
 - <https://github.com/evryfs/github-actions-runner-operator/>
+- <https://github.com/philips-labs/terraform-aws-github-runner/>
 
 Although the situation can change over time, as of writing this sentence, the benefits of using `actions-runner-controller` over the alternatives are:
 
 - `actions-runner-controller` has the ability to autoscale runners based on number of pending/progressing jobs (#99)
 - `actions-runner-controller` is able to gracefully stop runners (#103)
 - `actions-runner-controller` has ARM support
+- `actions-runner-controller` has GitHub Enterprise support (see [GitHub Enterprise support](#github-enterprise-support) section for caveats)

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -41,6 +41,62 @@ type HorizontalRunnerAutoscalerSpec struct {
 	// Metrics is the collection of various metric targets to calculate desired number of runners
 	// +optional
 	Metrics []MetricSpec `json:"metrics,omitempty"`
+
+	// ScaleUpTriggers is an experimental feature to increase the desired replicas by 1
+	// on each webhook requested received by the webhookBasedAutoscaler.
+	//
+	// This feature requires you to also enable and deploy the webhookBasedAutoscaler onto your cluster.
+	//
+	// Note that the added runners remain until the next sync period at least,
+	// and they may or may not be used by GitHub Actions depending on the timing.
+	// They are intended to be used to gain "resource slack" immediately after you
+	// receive a webhook from GitHub, so that you can loosely expect MinReplicas runners to be always available.
+	ScaleUpTriggers []ScaleUpTrigger `json:"scaleUpTriggers,omitempty"`
+
+	CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
+}
+
+type ScaleUpTrigger struct {
+	GitHubEvent *GitHubEventScaleUpTriggerSpec `json:"githubEvent,omitempty"`
+	Amount      int                            `json:"amount,omitempty"`
+	Duration    metav1.Duration                `json:"duration,omitempty"`
+}
+
+type GitHubEventScaleUpTriggerSpec struct {
+	CheckRun    *CheckRunSpec    `json:"checkRun,omitempty"`
+	PullRequest *PullRequestSpec `json:"pullRequest,omitempty"`
+	Push        *PushSpec        `json:"push,omitempty"`
+}
+
+// https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run
+type CheckRunSpec struct {
+	Types  []string `json:"types,omitempty"`
+	Status string   `json:"status,omitempty"`
+
+	// Names is a list of GitHub Actions glob patterns.
+	// Any check_run event whose name matches one of patterns in the list can trigger autoscaling.
+	// Note that check_run name seem to equal to the job name you've defined in your actions workflow yaml file.
+	// So it is very likely that you can utilize this to trigger depending on the job.
+	Names []string `json:"names,omitempty"`
+}
+
+// https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+type PullRequestSpec struct {
+	Types    []string `json:"types,omitempty"`
+	Branches []string `json:"branches,omitempty"`
+}
+
+// PushSpec is the condition for triggering scale-up on push event
+// Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
+type PushSpec struct {
+}
+
+// CapacityReservation specifies the number of replicas temporarily added
+// to the scale target until ExpirationTime.
+type CapacityReservation struct {
+	Name           string      `json:"name,omitempty"`
+	ExpirationTime metav1.Time `json:"expirationTime,omitempty"`
+	Replicas       int         `json:"replicas,omitempty"`
 }
 
 type ScaleTargetRef struct {
@@ -76,6 +132,16 @@ type MetricSpec struct {
 	// to determine how many pods should be removed.
 	// +optional
 	ScaleDownFactor string `json:"scaleDownFactor,omitempty"`
+
+	// ScaleUpAdjustment is the number of runners added on scale-up.
+	// You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+	// +optional
+	ScaleUpAdjustment int `json:"scaleUpAdjustment,omitempty"`
+
+	// ScaleDownAdjustment is the number of runners removed on scale-down.
+	// You can only specify either ScaleDownFactor or ScaleDownAdjustment.
+	// +optional
+	ScaleDownAdjustment int `json:"scaleDownAdjustment,omitempty"`
 }
 
 type HorizontalRunnerAutoscalerStatus struct {
@@ -90,7 +156,19 @@ type HorizontalRunnerAutoscalerStatus struct {
 	DesiredReplicas *int `json:"desiredReplicas,omitempty"`
 
 	// +optional
+	// +nullable
 	LastSuccessfulScaleOutTime *metav1.Time `json:"lastSuccessfulScaleOutTime,omitempty"`
+
+	// +optional
+	CacheEntries []CacheEntry `json:"cacheEntries,omitempty"`
+}
+
+const CacheEntryKeyDesiredReplicas = "desiredReplicas"
+
+type CacheEntry struct {
+	Key            string      `json:"key,omitempty"`
+	Value          int         `json:"value,omitempty"`
+	ExpirationTime metav1.Time `json:"expirationTime,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1alpha1/runner_types.go

@@ -25,6 +25,10 @@ import (
 
 // RunnerSpec defines the desired state of Runner
 type RunnerSpec struct {
+	// +optional
+	// +kubebuilder:validation:Pattern=`^[^/]+$`
+	Enterprise string `json:"enterprise,omitempty"`
+
 	// +optional
 	// +kubebuilder:validation:Pattern=`^[^/]+$`
 	Organization string `json:"organization,omitempty"`
@@ -88,16 +92,28 @@ type RunnerSpec struct {
 	DockerdWithinRunnerContainer *bool `json:"dockerdWithinRunnerContainer,omitempty"`
 	// +optional
 	DockerEnabled *bool `json:"dockerEnabled,omitempty"`
+	// +optional
+	DockerMTU *int64 `json:"dockerMTU,omitempty"`
 }
 
 // ValidateRepository validates repository field.
 func (rs *RunnerSpec) ValidateRepository() error {
-	// Organization and repository are both exclusive.
-	if len(rs.Organization) == 0 && len(rs.Repository) == 0 {
-		return errors.New("Spec needs organization or repository")
+	// Enterprise, Organization and repository are both exclusive.
+	foundCount := 0
+	if len(rs.Organization) > 0 {
+		foundCount += 1
 	}
-	if len(rs.Organization) > 0 && len(rs.Repository) > 0 {
-		return errors.New("Spec cannot have both organization and repository")
+	if len(rs.Repository) > 0 {
+		foundCount += 1
+	}
+	if len(rs.Enterprise) > 0 {
+		foundCount += 1
+	}
+	if foundCount == 0 {
+		return errors.New("Spec needs enterprise, organization or repository")
+	}
+	if foundCount > 1 {
+		return errors.New("Spec cannot have many fields defined enterprise, organization and repository")
 	}
 
 	return nil
@@ -105,14 +121,22 @@ func (rs *RunnerSpec) ValidateRepository() error {
 
 // RunnerStatus defines the observed state of Runner
 type RunnerStatus struct {
+	// +optional
 	Registration RunnerStatusRegistration `json:"registration"`
-	Phase        string                   `json:"phase"`
-	Reason       string                   `json:"reason"`
-	Message      string                   `json:"message"`
+	// +optional
+	Phase string `json:"phase,omitempty"`
+	// +optional
+	Reason string `json:"reason,omitempty"`
+	// +optional
+	Message string `json:"message,omitempty"`
+	// +optional
+	// +nullable
+	LastRegistrationCheckTime *metav1.Time `json:"lastRegistrationCheckTime,omitempty"`
 }
 
 // RunnerStatusRegistration contains runner registration status
 type RunnerStatusRegistration struct {
+	Enterprise   string      `json:"enterprise,omitempty"`
 	Organization string      `json:"organization,omitempty"`
 	Repository   string      `json:"repository,omitempty"`
 	Labels       []string    `json:"labels,omitempty"`
@@ -122,6 +146,7 @@ type RunnerStatusRegistration struct {
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:JSONPath=".spec.enterprise",name=Enterprise,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.organization",name=Organization,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.repository",name=Repository,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.labels",name=Labels,type=string

api/v1alpha1/runnerdeployment_types.go

@@ -25,13 +25,16 @@ const (
 	AutoscalingMetricTypePercentageRunnersBusy                        = "PercentageRunnersBusy"
 )
 
-// RunnerReplicaSetSpec defines the desired state of RunnerDeployment
+// RunnerDeploymentSpec defines the desired state of RunnerDeployment
 type RunnerDeploymentSpec struct {
 	// +optional
 	// +nullable
 	Replicas *int `json:"replicas,omitempty"`
 
-	Template RunnerTemplate `json:"template"`
+	// +optional
+	// +nullable
+	Selector *metav1.LabelSelector `json:"selector"`
+	Template RunnerTemplate        `json:"template"`
 }
 
 type RunnerDeploymentStatus struct {

api/v1alpha1/runnerreplicaset_types.go

@@ -26,7 +26,10 @@ type RunnerReplicaSetSpec struct {
 	// +nullable
 	Replicas *int `json:"replicas,omitempty"`
 
-	Template RunnerTemplate `json:"template"`
+	// +optional
+	// +nullable
+	Selector *metav1.LabelSelector `json:"selector"`
+	Template RunnerTemplate        `json:"template"`
 }
 
 type RunnerReplicaSetStatus struct {

api/v1alpha1/zz_generated.deepcopy.go

@@ -22,9 +22,97 @@ package v1alpha1
 
 import (
 	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CacheEntry) DeepCopyInto(out *CacheEntry) {
+	*out = *in
+	in.ExpirationTime.DeepCopyInto(&out.ExpirationTime)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheEntry.
+func (in *CacheEntry) DeepCopy() *CacheEntry {
+	if in == nil {
+		return nil
+	}
+	out := new(CacheEntry)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CapacityReservation) DeepCopyInto(out *CapacityReservation) {
+	*out = *in
+	in.ExpirationTime.DeepCopyInto(&out.ExpirationTime)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservation.
+func (in *CapacityReservation) DeepCopy() *CapacityReservation {
+	if in == nil {
+		return nil
+	}
+	out := new(CapacityReservation)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CheckRunSpec) DeepCopyInto(out *CheckRunSpec) {
+	*out = *in
+	if in.Types != nil {
+		in, out := &in.Types, &out.Types
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Names != nil {
+		in, out := &in.Names, &out.Names
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckRunSpec.
+func (in *CheckRunSpec) DeepCopy() *CheckRunSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(CheckRunSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTriggerSpec) {
+	*out = *in
+	if in.CheckRun != nil {
+		in, out := &in.CheckRun, &out.CheckRun
+		*out = new(CheckRunSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.PullRequest != nil {
+		in, out := &in.PullRequest, &out.PullRequest
+		*out = new(PullRequestSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Push != nil {
+		in, out := &in.Push, &out.Push
+		*out = new(PushSpec)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubEventScaleUpTriggerSpec.
+func (in *GitHubEventScaleUpTriggerSpec) DeepCopy() *GitHubEventScaleUpTriggerSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(GitHubEventScaleUpTriggerSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HorizontalRunnerAutoscaler) DeepCopyInto(out *HorizontalRunnerAutoscaler) {
 	*out = *in
@@ -110,6 +198,20 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.ScaleUpTriggers != nil {
+		in, out := &in.ScaleUpTriggers, &out.ScaleUpTriggers
+		*out = make([]ScaleUpTrigger, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.CapacityReservations != nil {
+		in, out := &in.CapacityReservations, &out.CapacityReservations
+		*out = make([]CapacityReservation, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
@@ -134,6 +236,13 @@ func (in *HorizontalRunnerAutoscalerStatus) DeepCopyInto(out *HorizontalRunnerAu
 		in, out := &in.LastSuccessfulScaleOutTime, &out.LastSuccessfulScaleOutTime
 		*out = (*in).DeepCopy()
 	}
+	if in.CacheEntries != nil {
+		in, out := &in.CacheEntries, &out.CacheEntries
+		*out = make([]CacheEntry, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerStatus.
@@ -166,6 +275,46 @@ func (in *MetricSpec) DeepCopy() *MetricSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PullRequestSpec) DeepCopyInto(out *PullRequestSpec) {
+	*out = *in
+	if in.Types != nil {
+		in, out := &in.Types, &out.Types
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Branches != nil {
+		in, out := &in.Branches, &out.Branches
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PullRequestSpec.
+func (in *PullRequestSpec) DeepCopy() *PullRequestSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PullRequestSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushSpec) DeepCopyInto(out *PushSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSpec.
+func (in *PushSpec) DeepCopy() *PushSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Runner) DeepCopyInto(out *Runner) {
 	*out = *in
@@ -260,6 +409,11 @@ func (in *RunnerDeploymentSpec) DeepCopyInto(out *RunnerDeploymentSpec) {
 		*out = new(int)
 		**out = **in
 	}
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = new(metav1.LabelSelector)
+		(*in).DeepCopyInto(*out)
+	}
 	in.Template.DeepCopyInto(&out.Template)
 }
 
@@ -392,6 +546,11 @@ func (in *RunnerReplicaSetSpec) DeepCopyInto(out *RunnerReplicaSetSpec) {
 		*out = new(int)
 		**out = **in
 	}
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = new(metav1.LabelSelector)
+		(*in).DeepCopyInto(*out)
+	}
 	in.Template.DeepCopyInto(&out.Template)
 }
 
@@ -535,6 +694,11 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.DockerMTU != nil {
+		in, out := &in.DockerMTU, &out.DockerMTU
+		*out = new(int64)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSpec.
@@ -551,6 +715,10 @@ func (in *RunnerSpec) DeepCopy() *RunnerSpec {
 func (in *RunnerStatus) DeepCopyInto(out *RunnerStatus) {
 	*out = *in
 	in.Registration.DeepCopyInto(&out.Registration)
+	if in.LastRegistrationCheckTime != nil {
+		in, out := &in.LastRegistrationCheckTime, &out.LastRegistrationCheckTime
+		*out = (*in).DeepCopy()
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerStatus.
@@ -615,3 +783,24 @@ func (in *ScaleTargetRef) DeepCopy() *ScaleTargetRef {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ScaleUpTrigger) DeepCopyInto(out *ScaleUpTrigger) {
+	*out = *in
+	if in.GitHubEvent != nil {
+		in, out := &in.GitHubEvent, &out.GitHubEvent
+		*out = new(GitHubEventScaleUpTriggerSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	out.Duration = in.Duration
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleUpTrigger.
+func (in *ScaleUpTrigger) DeepCopy() *ScaleUpTrigger {
+	if in == nil {
+		return nil
+	}
+	out := new(ScaleUpTrigger)
+	in.DeepCopyInto(out)
+	return out
+}

charts/.ci/ct-config.yaml

@@ -0,0 +1,4 @@
+# This file defines the config for "ct" (chart tester) used by the helm linting GitHub workflow
+lint-conf: charts/.ci/lint-config.yaml
+chart-repos:
+  - jetstack=https://charts.jetstack.io

charts/.ci/lint-config.yaml

@@ -0,0 +1,6 @@
+rules:
+  # One blank line is OK
+  empty-lines: 
+    max-start: 1
+    max-end: 1
+    max: 1

charts/.ci/scripts/local-ct-lint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm -it -w /repo -v $(pwd):/repo quay.io/helmpack/chart-testing ct lint --all --config charts/.ci/ct-config.yaml

charts/.ci/scripts/local-kube-score.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+
+for chart in `ls charts`;
+do
+helm template --values charts/$chart/ci/ci-values.yaml charts/$chart | kube-score score - \
+    --ignore-test pod-networkpolicy \
+    --ignore-test deployment-has-poddisruptionbudget \
+    --ignore-test deployment-has-host-podantiaffinity \
+    --ignore-test pod-probes \
+    --ignore-test container-image-tag \
+    --enable-optional-test container-security-context-privileged \
+    --enable-optional-test container-security-context-readonlyrootfilesystem \
+    --ignore-test container-security-context
+done

charts/actions-runner-controller/Chart.yaml

@@ -15,9 +15,17 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.10.4
 
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.11.2
+home: https://github.com/summerwind/actions-runner-controller
+
+sources:
+  - https://github.com/summerwind/actions-runner-controller
+
+maintainers:
+  - name: summerwind
+    email: contact@summerwind.jp
+    url: https://github.com/summerwind
+  - name: funkypenguin
+    email: davidy@funkypenguin.co.nz
+    url: https://www.funkypenguin.co.nz

charts/actions-runner-controller/ci/ci-values.yaml

@@ -0,0 +1,30 @@
+# This file sets some opinionated values for kube-score to use
+# when parsing the chart
+image:
+  pullPolicy: Always
+
+podSecurityContext:
+  fsGroup: 2000
+
+securityContext: 
+  capabilities:
+    drop:
+    - ALL
+  readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 2000
+
+resources: 
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 100m
+    memory: 128Mi  
+
+authSecret:
+  create: false
+
+# Set the following to true to create a dummy secret, allowing the manager pod to start
+# This is only useful in CI
+createDummySecret: true

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -48,6 +48,20 @@ spec:
           description: HorizontalRunnerAutoscalerSpec defines the desired state of
             HorizontalRunnerAutoscaler
           properties:
+            capacityReservations:
+              items:
+                description: CapacityReservation specifies the number of replicas
+                  temporarily added to the scale target until ExpirationTime.
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  name:
+                    type: string
+                  replicas:
+                    type: integer
+                type: object
+              type: array
             maxReplicas:
               description: MinReplicas is the maximum number of replicas the deployment
                 is allowed to scale
@@ -64,6 +78,11 @@ spec:
                     items:
                       type: string
                     type: array
+                  scaleDownAdjustment:
+                    description: ScaleDownAdjustment is the number of runners removed
+                      on scale-down. You can only specify either ScaleDownFactor or
+                      ScaleDownAdjustment.
+                    type: integer
                   scaleDownFactor:
                     description: ScaleDownFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many
@@ -73,6 +92,10 @@ spec:
                     description: ScaleDownThreshold is the percentage of busy runners
                       less than which will trigger the hpa to scale the runners down.
                     type: string
+                  scaleUpAdjustment:
+                    description: ScaleUpAdjustment is the number of runners added
+                      on scale-up. You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+                    type: integer
                   scaleUpFactor:
                     description: ScaleUpFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many
@@ -104,9 +127,79 @@ spec:
                 name:
                   type: string
               type: object
+            scaleUpTriggers:
+              description: "ScaleUpTriggers is an experimental feature to increase
+                the desired replicas by 1 on each webhook requested received by the
+                webhookBasedAutoscaler. \n This feature requires you to also enable
+                and deploy the webhookBasedAutoscaler onto your cluster. \n Note that
+                the added runners remain until the next sync period at least, and
+                they may or may not be used by GitHub Actions depending on the timing.
+                They are intended to be used to gain \"resource slack\" immediately
+                after you receive a webhook from GitHub, so that you can loosely expect
+                MinReplicas runners to be always available."
+              items:
+                properties:
+                  amount:
+                    type: integer
+                  duration:
+                    type: string
+                  githubEvent:
+                    properties:
+                      checkRun:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run
+                        properties:
+                          names:
+                            description: Names is a list of GitHub Actions glob patterns.
+                              Any check_run event whose name matches one of patterns
+                              in the list can trigger autoscaling. Note that check_run
+                              name seem to equal to the job name you've defined in
+                              your actions workflow yaml file. So it is very likely
+                              that you can utilize this to trigger depending on the
+                              job.
+                            items:
+                              type: string
+                            type: array
+                          status:
+                            type: string
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      pullRequest:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+                        properties:
+                          branches:
+                            items:
+                              type: string
+                            type: array
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      push:
+                        description: PushSpec is the condition for triggering scale-up
+                          on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
+                        type: object
+                    type: object
+                type: object
+              type: array
           type: object
         status:
           properties:
+            cacheEntries:
+              items:
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  key:
+                    type: string
+                  value:
+                    type: integer
+                type: object
+              type: array
             desiredReplicas:
               description: DesiredReplicas is the total number of desired, non-terminated
                 and latest pods to be set for the primary RunnerSet This doesn't include
@@ -114,6 +207,7 @@ spec:
               type: integer
             lastSuccessfulScaleOutTime:
               format: date-time
+              nullable: true
               type: string
             observedGeneration:
               description: ObservedGeneration is the most recent generation observed

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerdeployments.yaml

@@ -38,11 +38,42 @@ spec:
         metadata:
           type: object
         spec:
-          description: RunnerReplicaSetSpec defines the desired state of RunnerDeployment
+          description: RunnerDeploymentSpec defines the desired state of RunnerDeployment
           properties:
             replicas:
               nullable: true
               type: integer
+            selector:
+              description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.
+              nullable: true
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                      - key
+                      - operator
+                    type: object
+                  type: array
+                matchLabels:
+                  additionalProperties:
+                    type: string
+                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                  type: object
+              type: object
             template:
               properties:
                 metadata:
@@ -402,6 +433,9 @@ spec:
                       type: array
                     dockerEnabled:
                       type: boolean
+                    dockerMTU:
+                      format: int64
+                      type: integer
                     dockerdContainerResources:
                       description: ResourceRequirements describes the compute resource requirements.
                       properties:
@@ -426,6 +460,9 @@ spec:
                       type: object
                     dockerdWithinRunnerContainer:
                       type: boolean
+                    enterprise:
+                      pattern: ^[^/]+$
+                      type: string
                     env:
                       items:
                         description: EnvVar represents an environment variable present in a Container.

charts/actions-runner-controller/crds/actions.summerwind.dev_runnerreplicasets.yaml

@@ -43,6 +43,37 @@ spec:
             replicas:
               nullable: true
               type: integer
+            selector:
+              description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.
+              nullable: true
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                      - key
+                      - operator
+                    type: object
+                  type: array
+                matchLabels:
+                  additionalProperties:
+                    type: string
+                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                  type: object
+              type: object
             template:
               properties:
                 metadata:
@@ -402,6 +433,9 @@ spec:
                       type: array
                     dockerEnabled:
                       type: boolean
+                    dockerMTU:
+                      format: int64
+                      type: integer
                     dockerdContainerResources:
                       description: ResourceRequirements describes the compute resource requirements.
                       properties:
@@ -426,6 +460,9 @@ spec:
                       type: object
                     dockerdWithinRunnerContainer:
                       type: boolean
+                    enterprise:
+                      pattern: ^[^/]+$
+                      type: string
                     env:
                       items:
                         description: EnvVar represents an environment variable present in a Container.

charts/actions-runner-controller/crds/actions.summerwind.dev_runners.yaml

@@ -7,6 +7,9 @@ metadata:
   name: runners.actions.summerwind.dev
 spec:
   additionalPrinterColumns:
+    - JSONPath: .spec.enterprise
+      name: Enterprise
+      type: string
     - JSONPath: .spec.organization
       name: Organization
       type: string
@@ -395,6 +398,9 @@ spec:
               type: array
             dockerEnabled:
               type: boolean
+            dockerMTU:
+              format: int64
+              type: integer
             dockerdContainerResources:
               description: ResourceRequirements describes the compute resource requirements.
               properties:
@@ -419,6 +425,9 @@ spec:
               type: object
             dockerdWithinRunnerContainer:
               type: boolean
+            enterprise:
+              pattern: ^[^/]+$
+              type: string
             env:
               items:
                 description: EnvVar represents an environment variable present in a Container.
@@ -1532,6 +1541,10 @@ spec:
         status:
           description: RunnerStatus defines the observed state of Runner
           properties:
+            lastRegistrationCheckTime:
+              format: date-time
+              nullable: true
+              type: string
             message:
               type: string
             phase:
@@ -1541,6 +1554,8 @@ spec:
             registration:
               description: RunnerStatusRegistration contains runner registration status
               properties:
+                enterprise:
+                  type: string
                 expiresAt:
                   format: date-time
                   type: string
@@ -1558,11 +1573,6 @@ spec:
                 - expiresAt
                 - token
               type: object
-          required:
-            - message
-            - phase
-            - reason
-            - registration
           type: object
       type: object
   version: v1alpha1

charts/actions-runner-controller/templates/NOTES.txt

@@ -1,8 +1,8 @@
 1. Get the application URL by running these commands:
-{{- if .Values.ingress.enabled }}
-{{- range $host := .Values.ingress.hosts }}
+{{- if .Values.githubWebhookServer.ingress.enabled }}
+{{- range $host := .Values.githubWebhookServer.ingress.hosts }}
   {{- range .paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+  http{{ if $.Values.githubWebhookServer.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
   {{- end }}
 {{- end }}
 {{- else if contains "NodePort" .Values.service.type }}

charts/actions-runner-controller/templates/_github_webhook_server_helpers.tpl

@@ -0,0 +1,56 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.name" -}}
+{{- default .Chart.Name .Values.githubWebhookServer.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "actions-runner-controller-github-webhook-server.instance" -}}
+{{- printf "%s-%s" .Release.Name "github-webhook-server" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.fullname" -}}
+{{- if .Values.githubWebhookServer.fullnameOverride }}
+{{- .Values.githubWebhookServer.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.githubWebhookServer.nameOverride }}
+{{- $instance := include "actions-runner-controller-github-webhook-server.instance" . }}
+{{- if contains $name $instance }}
+{{- $instance | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s-%s" .Release.Name $name "github-webhook-server" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "actions-runner-controller-github-webhook-server.name" . }}
+app.kubernetes.io/instance: {{ include "actions-runner-controller-github-webhook-server.instance" . }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.serviceAccountName" -}}
+{{- if .Values.githubWebhookServer.serviceAccount.create }}
+{{- default (include "actions-runner-controller-github-webhook-server.fullname" .) .Values.githubWebhookServer.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.githubWebhookServer.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{- define "actions-runner-controller-github-webhook-server.secretName" -}}
+{{- default (include "actions-runner-controller-github-webhook-server.fullname" .) .Values.githubWebhookServer.secret.name }}
+{{- end }}
+
+{{- define "actions-runner-controller-github-webhook-server.roleName" -}}
+{{- include "actions-runner-controller-github-webhook-server.fullname" . }}
+{{- end }}

charts/actions-runner-controller/templates/_helpers.tpl

@@ -64,6 +64,10 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{- define "actions-runner-controller.secretName" -}}
+{{- default (include "actions-runner-controller.fullname" .) .Values.authSecret.name -}}
+{{- end }}
+
 {{- define "actions-runner-controller.leaderElectionRoleName" -}}
 {{- include "actions-runner-controller.fullname" . }}-leader-election
 {{- end }}
@@ -85,11 +89,11 @@ Create the name of the service account to use
 {{- end }}
 
 {{- define "actions-runner-controller.webhookServiceName" -}}
-{{- include "actions-runner-controller.fullname" . }}-webhook
+{{- include "actions-runner-controller.fullname" . | trunc 55 }}-webhook
 {{- end }}
 
 {{- define "actions-runner-controller.authProxyServiceName" -}}
-{{- include "actions-runner-controller.fullname" . }}-controller-manager-metrics-service
+{{- include "actions-runner-controller.fullname" . | trunc 47 }}-metrics-service
 {{- end }}
 
 {{- define "actions-runner-controller.selfsignedIssuerName" -}}

charts/actions-runner-controller/templates/certificate.yaml

@@ -5,7 +5,7 @@ apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: {{ include "actions-runner-controller.selfsignedIssuerName" . }}
-  namespace: {{ .Namespace }}
+  namespace: {{ .Release.Namespace }}
 spec:
   selfSigned: {}
 ---
@@ -13,7 +13,7 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: {{ include "actions-runner-controller.servingCertName" . }}
-  namespace: {{ .Namespace }}
+  namespace: {{ .Release.Namespace }}
 spec:
   dnsNames:
   - {{ include "actions-runner-controller.webhookServiceName" . }}.{{ .Release.Namespace }}.svc

charts/actions-runner-controller/templates/ci-secret.yaml

@@ -0,0 +1,10 @@
+# This template only exists to facilitate CI testing of the chart, since
+# a secret is expected to be found in the namespace by the controller manager
+{{ if .Values.createDummySecret -}}
+apiVersion: v1
+data:
+  github_token: dGVzdA==
+kind: Secret
+metadata:
+  name: controller-manager
+{{- end }}

charts/actions-runner-controller/templates/deployment.yaml

@@ -6,6 +6,7 @@ metadata:
   labels:
     {{- include "actions-runner-controller.labels" . | nindent 4 }}
 spec:
+  replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
       {{- include "actions-runner-controller.selectorLabels" . | nindent 6 }}
@@ -34,6 +35,9 @@ spec:
         - "--enable-leader-election"
         - "--sync-period={{ .Values.syncPeriod }}"
         - "--docker-image={{ .Values.image.dindSidecarRepositoryAndTag }}"
+        {{- if .Values.scope.singleNamespace }}
+        - "--watch-namespace={{ default .Release.Namespace .Values.scope.watchNamespace }}"
+        {{- end }}
         command:
         - "/manager"
         env:
@@ -41,23 +45,27 @@ spec:
           valueFrom:
             secretKeyRef:
               key: github_token
-              name: controller-manager
+              name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         - name: GITHUB_APP_ID
           valueFrom:
             secretKeyRef:
               key: github_app_id
-              name: controller-manager
+              name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         - name: GITHUB_APP_INSTALLATION_ID
           valueFrom:
             secretKeyRef:
               key: github_app_installation_id
-              name: controller-manager
+              name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         - name: GITHUB_APP_PRIVATE_KEY
           value: /etc/actions-runner-controller/github_app_private_key
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
+        {{- range $key, $val := .Values.env }}
+        - name: {{ $key }}
+          value: {{ $val | quote }}
+        {{- end }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         name: manager
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         ports:
@@ -66,10 +74,14 @@ spec:
           protocol: TCP
         resources:
           {{- toYaml .Values.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
         volumeMounts:
         - mountPath: "/etc/actions-runner-controller"
-          name: controller-manager
+          name: secret
           readOnly: true
+        - mountPath: /tmp
+          name: tmp
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
@@ -78,20 +90,27 @@ spec:
         - "--upstream=http://127.0.0.1:8080/"
         - "--logtostderr=true"
         - "--v=10"
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
+        image: "{{ .Values.kube_rbac_proxy.image.repository }}:{{ .Values.kube_rbac_proxy.image.tag }}"
         name: kube-rbac-proxy
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
         ports:
         - containerPort: 8443
           name: https
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
       terminationGracePeriodSeconds: 10
       volumes:
-      - name: controller-manager
+      - name: secret
         secret:
-          secretName: controller-manager
+          secretName: {{ include "actions-runner-controller.secretName" . }}
       - name: cert
         secret:
           defaultMode: 420
           secretName: webhook-server-cert
+      - name: tmp
+        emptyDir: {}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -0,0 +1,89 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.githubWebhookServer.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.githubWebhookServer.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.githubWebhookServer.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.githubWebhookServer.podSecurityContext | nindent 8 }}
+      {{- with .Values.githubWebhookServer.priorityClassName }}
+      priorityClassName: "{{ . }}"
+      {{- end }}
+      containers:
+      - args:
+        - "--metrics-addr=127.0.0.1:8080"
+        - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}"
+        command:
+        - "/github-webhook-server"
+        env:
+        - name: GITHUB_WEBHOOK_SECRET_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: github_webhook_secret_token
+              name: {{ include "actions-runner-controller-github-webhook-server.secretName" . }}
+              optional: true
+        {{- range $key, $val := .Values.githubWebhookServer.env }}
+        - name: {{ $key }}
+          value: {{ $val | quote }}
+        {{- end }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        name: github-webhook-server
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: 8000
+          name: http
+          protocol: TCP
+        resources:
+          {{- toYaml .Values.githubWebhookServer.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.githubWebhookServer.securityContext | nindent 12 }}
+      - args:
+        - "--secure-listen-address=0.0.0.0:8443"
+        - "--upstream=http://127.0.0.1:8080/"
+        - "--logtostderr=true"
+        - "--v=10"
+        image: "{{ .Values.kube_rbac_proxy.image.repository }}:{{ .Values.kube_rbac_proxy.image.tag }}"
+        name: kube-rbac-proxy
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: 8443
+          name: https
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
+      terminationGracePeriodSeconds: 10
+      {{- with .Values.githubWebhookServer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.githubWebhookServer.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.githubWebhookServer.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.ingress.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.githubWebhookServer.ingress.enabled -}}
+{{- $fullName := include "actions-runner-controller-github-webhook-server.fullname" . -}}
+{{- $svcPort := .Values.githubWebhookServer.service.port -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  {{- with .Values.githubWebhookServer.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.githubWebhookServer.ingress.tls }}
+  tls:
+    {{- range .Values.githubWebhookServer.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.githubWebhookServer.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+          {{- end }}
+    {{- end }}
+  {{- end }}

charts/actions-runner-controller/templates/githubwebhook.role.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: {{ include "actions-runner-controller-github-webhook-server.roleName" . }}
+rules:
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/status
+  verbs:
+  - get
+  - patch
+  - update
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.role_binding.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.roleName" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "actions-runner-controller-github-webhook-server.roleName" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.secrets.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.githubWebhookServer.enabled }}
+{{- if .Values.githubWebhookServer.secret.create }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.secretName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+type: Opaque
+data:
+{{- if .Values.githubWebhookServer.secret.github_webhook_secret_token }}
+  github_webhook_secret_token: {{ .Values.githubWebhookServer.secret.github_webhook_secret_token | toString | b64enc }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.githubWebhookServer.service.type }}
+  ports:
+    {{ range $_, $port := .Values.githubWebhookServer.service.ports -}}
+    - {{ $port | toYaml | nindent 6 }}
+    {{- end }}
+  selector:
+    {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.serviceaccount.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.githubWebhookServer.enabled -}}
+{{- if .Values.githubWebhookServer.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  {{- with .Values.githubWebhookServer.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
+{{- end }}

charts/actions-runner-controller/templates/manager_secrets.yaml

@@ -1,14 +1,23 @@
-{{- if or .Values.authSecret.enabled }}
+{{- if .Values.authSecret.create }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: controller-manager
+  name: {{ include "actions-runner-controller.secretName" . }}
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "actions-runner-controller.labels" . | nindent 4 }}
 type: Opaque
 data:
-{{- range $k, $v := .Values.authSecret }}
-  {{ $k }}: {{ $v | toString | b64enc }}
-{{- end }} 
-{{- end }}
+{{- if .Values.authSecret.github_app_id }}
+  github_app_id: {{ .Values.authSecret.github_app_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.authSecret.github_app_installation_id }}
+  github_app_installation_id: {{ .Values.authSecret.github_app_installation_id | toString | b64enc }}
+{{- end }}
+{{- if .Values.authSecret.github_app_private_key }}
+  github_app_private_key: {{ .Values.authSecret.github_app_private_key | toString | b64enc }}
+{{- end }}
+{{- if .Values.authSecret.github_token }}
+  github_token: {{ .Values.authSecret.github_token | toString | b64enc }}
+{{- end }}
+{{- end }}

charts/actions-runner-controller/values.yaml

@@ -11,7 +11,8 @@ syncPeriod: 10m
 # Only 1 authentication method can be deployed at a time
 # Uncomment the configuration you are applying and fill in the details
 authSecret:
-  enabled: false
+  create: true
+  name: "controller-manager"
   ### GitHub Apps Configuration
   #github_app_id: ""
   #github_app_installation_id: ""
@@ -21,11 +22,15 @@ authSecret:
 
 image:
   repository: summerwind/actions-runner-controller
-  # Overrides the manager image tag whose default is the chart appVersion if the tag key is commented out
-  tag: "latest"
+  tag: "v0.17.0"
   dindSidecarRepositoryAndTag: "docker:dind"
   pullPolicy: IfNotPresent
 
+kube_rbac_proxy:
+  image:
+    repository: quay.io/brancz/kube-rbac-proxy
+    tag: v0.8.0
+
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
@@ -41,10 +46,12 @@ serviceAccount:
 
 podAnnotations: {}
 
-podSecurityContext: {}
+podSecurityContext:
+  {}
   # fsGroup: 2000
 
-securityContext: {}
+securityContext:
+  {}
   # capabilities:
   #   drop:
   #   - ALL
@@ -56,20 +63,8 @@ service:
   type: ClusterIP
   port: 443
 
-ingress:
-  enabled: false
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-    # kubernetes.io/tls-acme: "true"
-  hosts:
-    - host: chart-example.local
-      paths: []
-  tls: []
-  #  - secretName: chart-example-tls
-  #    hosts:
-  #      - chart-example.local
-
-resources: {}
+resources:
+  {}
   # We usually recommend not to specify default resources and to leave this as a conscious
   # choice for the user. This also increases chances charts run on environments with little
   # resources, such as Minikube. If you do want to specify resources, uncomment the following
@@ -98,3 +93,68 @@ affinity: {}
 # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
 # PriorityClass: system-cluster-critical
 priorityClassName: ""
+
+env:
+  {}
+  # http_proxy: "proxy.com:8080"
+  # https_proxy: "proxy.com:8080"
+  # no_proxy: ""
+
+scope:
+  # If true, the controller will only watch custom resources in a single namespace
+  singleNamespace: false
+  # If `scope.singleNamespace=true`, the controller will only watch custom resources in this namespace
+  # The default value is "", which means the namespace of the controller
+  watchNamespace: ""
+
+githubWebhookServer:
+  enabled: false
+  labels: {}
+  replicaCount: 1
+  syncPeriod: 10m
+  secret:
+    create: true
+    name: "github-webhook-server"
+    ### GitHub Webhook Configuration
+    #github_webhook_secret_token: ""
+  imagePullSecrets: []
+  nameOverride: ""
+  fullnameOverride: ""
+  serviceAccount:
+    # Specifies whether a service account should be created
+    create: true
+    # Annotations to add to the service account
+    annotations: {}
+    # The name of the service account to use.
+    # If not set and create is true, a name is generated using the fullname template
+    name: ""
+  podAnnotations: {}
+  podSecurityContext: {}
+  # fsGroup: 2000
+  securityContext: {}
+  resources: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  priorityClassName: ""
+  service:
+    type: ClusterIP
+    ports:
+      - port: 80
+        targetPort: http
+        protocol: TCP
+        name: http
+        #nodePort: someFixedPortForUseWithTerraformCdkCfnEtc
+  ingress:
+    enabled: false
+    annotations:
+      {}
+      # kubernetes.io/ingress.class: nginx
+      # kubernetes.io/tls-acme: "true"
+    hosts:
+      - host: chart-example.local
+        paths: []
+    tls: []
+    #  - secretName: chart-example-tls
+    #    hosts:
+    #      - chart-example.local

cmd/githubwebhookserver/main.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2021 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/controllers"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/exec"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	// +kubebuilder:scaffold:imports
+)
+
+var (
+	scheme   = runtime.NewScheme()
+	setupLog = ctrl.Log.WithName("setup")
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(scheme)
+
+	_ = actionsv1alpha1.AddToScheme(scheme)
+	// +kubebuilder:scaffold:scheme
+}
+
+func main() {
+	var (
+		err error
+
+		webhookAddr string
+		metricsAddr string
+
+		// The secret token of the GitHub Webhook. See https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks
+		webhookSecretToken string
+
+		watchNamespace string
+
+		enableLeaderElection bool
+		syncPeriod           time.Duration
+	)
+
+	webhookSecretToken = os.Getenv("GITHUB_WEBHOOK_SECRET_TOKEN")
+
+	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&watchNamespace, "watch-namespace", "", "The namespace to watch for HorizontalRunnerAutoscaler's to scale on Webhook. Set to empty for letting it watch for all namespaces.")
+	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
+		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
+	flag.Parse()
+
+	if webhookSecretToken == "" {
+		setupLog.Info("-webhook-secret-token is missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks")
+	}
+
+	if watchNamespace == "" {
+		setupLog.Info("-watch-namespace is empty. HorizontalRunnerAutoscalers in all the namespaces are watched, cached, and considered as scale targets.")
+	} else {
+		setupLog.Info("-watch-namespace is %q. Only HorizontalRunnerAutoscalers in %q are watched, cached, and considered as scale targets.")
+	}
+
+	logger := zap.New(func(o *zap.Options) {
+		o.Development = true
+	})
+
+	ctrl.SetLogger(logger)
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme:             scheme,
+		SyncPeriod:         &syncPeriod,
+		LeaderElection:     enableLeaderElection,
+		Namespace:          watchNamespace,
+		MetricsBindAddress: metricsAddr,
+		Port:               9443,
+	})
+	if err != nil {
+		setupLog.Error(err, "unable to start manager")
+		os.Exit(1)
+	}
+
+	hraGitHubWebhook := &controllers.HorizontalRunnerAutoscalerGitHubWebhook{
+		Client:         mgr.GetClient(),
+		Log:            ctrl.Log.WithName("controllers").WithName("Runner"),
+		Recorder:       nil,
+		Scheme:         mgr.GetScheme(),
+		SecretKeyBytes: []byte(webhookSecretToken),
+		Namespace:      watchNamespace,
+	}
+
+	if err = hraGitHubWebhook.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Runner")
+		os.Exit(1)
+	}
+
+	var wg sync.WaitGroup
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+
+		setupLog.Info("starting webhook server")
+		if err := mgr.Start(ctx.Done()); err != nil {
+			setupLog.Error(err, "problem running manager")
+			os.Exit(1)
+		}
+	}()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", hraGitHubWebhook.Handle)
+
+	srv := http.Server{
+		Addr:    webhookAddr,
+		Handler: mux,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+
+		go func() {
+			<-ctx.Done()
+
+			srv.Shutdown(context.Background())
+		}()
+
+		if err := srv.ListenAndServe(); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				setupLog.Error(err, "problem running http server")
+			}
+		}
+	}()
+
+	go func() {
+		<-ctrl.SetupSignalHandler()
+		cancel()
+	}()
+
+	wg.Wait()
+}

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -48,6 +48,20 @@ spec:
           description: HorizontalRunnerAutoscalerSpec defines the desired state of
             HorizontalRunnerAutoscaler
           properties:
+            capacityReservations:
+              items:
+                description: CapacityReservation specifies the number of replicas
+                  temporarily added to the scale target until ExpirationTime.
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  name:
+                    type: string
+                  replicas:
+                    type: integer
+                type: object
+              type: array
             maxReplicas:
               description: MinReplicas is the maximum number of replicas the deployment
                 is allowed to scale
@@ -64,6 +78,11 @@ spec:
                     items:
                       type: string
                     type: array
+                  scaleDownAdjustment:
+                    description: ScaleDownAdjustment is the number of runners removed
+                      on scale-down. You can only specify either ScaleDownFactor or
+                      ScaleDownAdjustment.
+                    type: integer
                   scaleDownFactor:
                     description: ScaleDownFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many
@@ -73,6 +92,10 @@ spec:
                     description: ScaleDownThreshold is the percentage of busy runners
                       less than which will trigger the hpa to scale the runners down.
                     type: string
+                  scaleUpAdjustment:
+                    description: ScaleUpAdjustment is the number of runners added
+                      on scale-up. You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+                    type: integer
                   scaleUpFactor:
                     description: ScaleUpFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many
@@ -104,9 +127,79 @@ spec:
                 name:
                   type: string
               type: object
+            scaleUpTriggers:
+              description: "ScaleUpTriggers is an experimental feature to increase
+                the desired replicas by 1 on each webhook requested received by the
+                webhookBasedAutoscaler. \n This feature requires you to also enable
+                and deploy the webhookBasedAutoscaler onto your cluster. \n Note that
+                the added runners remain until the next sync period at least, and
+                they may or may not be used by GitHub Actions depending on the timing.
+                They are intended to be used to gain \"resource slack\" immediately
+                after you receive a webhook from GitHub, so that you can loosely expect
+                MinReplicas runners to be always available."
+              items:
+                properties:
+                  amount:
+                    type: integer
+                  duration:
+                    type: string
+                  githubEvent:
+                    properties:
+                      checkRun:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run
+                        properties:
+                          names:
+                            description: Names is a list of GitHub Actions glob patterns.
+                              Any check_run event whose name matches one of patterns
+                              in the list can trigger autoscaling. Note that check_run
+                              name seem to equal to the job name you've defined in
+                              your actions workflow yaml file. So it is very likely
+                              that you can utilize this to trigger depending on the
+                              job.
+                            items:
+                              type: string
+                            type: array
+                          status:
+                            type: string
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      pullRequest:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+                        properties:
+                          branches:
+                            items:
+                              type: string
+                            type: array
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      push:
+                        description: PushSpec is the condition for triggering scale-up
+                          on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
+                        type: object
+                    type: object
+                type: object
+              type: array
           type: object
         status:
           properties:
+            cacheEntries:
+              items:
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  key:
+                    type: string
+                  value:
+                    type: integer
+                type: object
+              type: array
             desiredReplicas:
               description: DesiredReplicas is the total number of desired, non-terminated
                 and latest pods to be set for the primary RunnerSet This doesn't include
@@ -114,6 +207,7 @@ spec:
               type: integer
             lastSuccessfulScaleOutTime:
               format: date-time
+              nullable: true
               type: string
             observedGeneration:
               description: ObservedGeneration is the most recent generation observed

config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml

@@ -38,11 +38,42 @@ spec:
         metadata:
           type: object
         spec:
-          description: RunnerReplicaSetSpec defines the desired state of RunnerDeployment
+          description: RunnerDeploymentSpec defines the desired state of RunnerDeployment
           properties:
             replicas:
               nullable: true
               type: integer
+            selector:
+              description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.
+              nullable: true
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                      - key
+                      - operator
+                    type: object
+                  type: array
+                matchLabels:
+                  additionalProperties:
+                    type: string
+                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                  type: object
+              type: object
             template:
               properties:
                 metadata:
@@ -402,6 +433,9 @@ spec:
                       type: array
                     dockerEnabled:
                       type: boolean
+                    dockerMTU:
+                      format: int64
+                      type: integer
                     dockerdContainerResources:
                       description: ResourceRequirements describes the compute resource requirements.
                       properties:
@@ -426,6 +460,9 @@ spec:
                       type: object
                     dockerdWithinRunnerContainer:
                       type: boolean
+                    enterprise:
+                      pattern: ^[^/]+$
+                      type: string
                     env:
                       items:
                         description: EnvVar represents an environment variable present in a Container.

config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml

@@ -43,6 +43,37 @@ spec:
             replicas:
               nullable: true
               type: integer
+            selector:
+              description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.
+              nullable: true
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                      - key
+                      - operator
+                    type: object
+                  type: array
+                matchLabels:
+                  additionalProperties:
+                    type: string
+                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                  type: object
+              type: object
             template:
               properties:
                 metadata:
@@ -402,6 +433,9 @@ spec:
                       type: array
                     dockerEnabled:
                       type: boolean
+                    dockerMTU:
+                      format: int64
+                      type: integer
                     dockerdContainerResources:
                       description: ResourceRequirements describes the compute resource requirements.
                       properties:
@@ -426,6 +460,9 @@ spec:
                       type: object
                     dockerdWithinRunnerContainer:
                       type: boolean
+                    enterprise:
+                      pattern: ^[^/]+$
+                      type: string
                     env:
                       items:
                         description: EnvVar represents an environment variable present in a Container.

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -7,6 +7,9 @@ metadata:
   name: runners.actions.summerwind.dev
 spec:
   additionalPrinterColumns:
+    - JSONPath: .spec.enterprise
+      name: Enterprise
+      type: string
     - JSONPath: .spec.organization
       name: Organization
       type: string
@@ -395,6 +398,9 @@ spec:
               type: array
             dockerEnabled:
               type: boolean
+            dockerMTU:
+              format: int64
+              type: integer
             dockerdContainerResources:
               description: ResourceRequirements describes the compute resource requirements.
               properties:
@@ -419,6 +425,9 @@ spec:
               type: object
             dockerdWithinRunnerContainer:
               type: boolean
+            enterprise:
+              pattern: ^[^/]+$
+              type: string
             env:
               items:
                 description: EnvVar represents an environment variable present in a Container.
@@ -1532,6 +1541,10 @@ spec:
         status:
           description: RunnerStatus defines the observed state of Runner
           properties:
+            lastRegistrationCheckTime:
+              format: date-time
+              nullable: true
+              type: string
             message:
               type: string
             phase:
@@ -1541,6 +1554,8 @@ spec:
             registration:
               description: RunnerStatusRegistration contains runner registration status
               properties:
+                enterprise:
+                  type: string
                 expiresAt:
                   format: date-time
                   type: string
@@ -1558,11 +1573,6 @@ spec:
                 - expiresAt
                 - token
               type: object
-          required:
-            - message
-            - phase
-            - reason
-            - registration
           type: object
       type: object
   version: v1alpha1

config/default/manager_auth_proxy_patch.yaml

@@ -10,7 +10,7 @@ spec:
     spec:
       containers:
       - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
+        image: quay.io/brancz/kube-rbac-proxy:v0.8.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8080/"

controllers/autoscaling.go

@@ -7,8 +7,11 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -19,7 +22,48 @@ const (
 	defaultScaleDownFactor    = 0.7
 )
 
-func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+func getValueAvailableAt(now time.Time, from, to *time.Time, reservedValue int) *int {
+	if to != nil && now.After(*to) {
+		return nil
+	}
+
+	if from != nil && now.Before(*from) {
+		return nil
+	}
+
+	return &reservedValue
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) fetchSuggestedReplicasFromCache(hra v1alpha1.HorizontalRunnerAutoscaler) *int {
+	var entry *v1alpha1.CacheEntry
+
+	for i := range hra.Status.CacheEntries {
+		ent := hra.Status.CacheEntries[i]
+
+		if ent.Key != v1alpha1.CacheEntryKeyDesiredReplicas {
+			continue
+		}
+
+		if !time.Now().Before(ent.ExpirationTime.Time) {
+			continue
+		}
+
+		entry = &ent
+
+		break
+	}
+
+	if entry != nil {
+		v := getValueAvailableAt(time.Now(), nil, &entry.ExpirationTime.Time, entry.Value)
+		if v != nil {
+			return v
+		}
+	}
+
+	return nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	if hra.Spec.MinReplicas == nil {
 		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
 	} else if hra.Spec.MaxReplicas == nil {
@@ -27,16 +71,22 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp
 	}
 
 	metrics := hra.Spec.Metrics
-	if len(metrics) == 0 || metrics[0].Type == v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns {
-		return r.calculateReplicasByQueuedAndInProgressWorkflowRuns(rd, hra)
+	if len(metrics) == 0 {
+		if len(hra.Spec.ScaleUpTriggers) == 0 {
+			return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(rd, hra)
+		}
+
+		return nil, nil
+	} else if metrics[0].Type == v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns {
+		return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(rd, hra)
 	} else if metrics[0].Type == v1alpha1.AutoscalingMetricTypePercentageRunnersBusy {
-		return r.calculateReplicasByPercentageRunnersBusy(rd, hra)
+		return r.suggestReplicasByPercentageRunnersBusy(rd, hra)
 	} else {
 		return nil, fmt.Errorf("validting autoscaling metrics: unsupported metric type %q", metrics[0].Type)
 	}
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByQueuedAndInProgressWorkflowRuns(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 
 	var repos [][]string
 	metrics := hra.Spec.Metrics
@@ -47,6 +97,13 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByQueuedAndInPro
 			return nil, fmt.Errorf("asserting runner deployment spec to detect bug: spec.template.organization should not be empty on this code path")
 		}
 
+		// In case it's an organizational runners deployment without any scaling metrics defined,
+		// we assume that the desired replicas should always be `minReplicas + capacityReservedThroughWebhook`.
+		// See https://github.com/summerwind/actions-runner-controller/issues/377#issuecomment-793372693
+		if len(metrics) == 0 {
+			return nil, nil
+		}
+
 		if len(metrics[0].RepositoryNames) == 0 {
 			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment")
 		}
@@ -96,12 +153,12 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByQueuedAndInPro
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
-		list, _, err := r.GitHubClient.Actions.ListRepositoryWorkflowRuns(context.TODO(), user, repoName, nil)
+		workflowRuns, err := r.GitHubClient.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
 		if err != nil {
 			return nil, err
 		}
 
-		for _, run := range list.WorkflowRuns {
+		for _, run := range workflowRuns {
 			total++
 
 			// In May 2020, there are only 3 statuses.
@@ -121,42 +178,24 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByQueuedAndInPro
 		}
 	}
 
-	minReplicas := *hra.Spec.MinReplicas
-	maxReplicas := *hra.Spec.MaxReplicas
 	necessaryReplicas := queued + inProgress
 
-	var desiredReplicas int
-
-	if necessaryReplicas < minReplicas {
-		desiredReplicas = minReplicas
-	} else if necessaryReplicas > maxReplicas {
-		desiredReplicas = maxReplicas
-	} else {
-		desiredReplicas = necessaryReplicas
-	}
-
-	rd.Status.Replicas = &desiredReplicas
-	replicas := desiredReplicas
-
 	r.Log.V(1).Info(
-		"Calculated desired replicas",
-		"computed_replicas_desired", desiredReplicas,
-		"spec_replicas_min", minReplicas,
-		"spec_replicas_max", maxReplicas,
+		fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas),
 		"workflow_runs_completed", completed,
 		"workflow_runs_in_progress", inProgress,
 		"workflow_runs_queued", queued,
 		"workflow_runs_unknown", unknown,
+		"namespace", hra.Namespace,
+		"runner_deployment", rd.Name,
+		"horizontal_runner_autoscaler", hra.Name,
 	)
 
-	return &replicas, nil
+	return &necessaryReplicas, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunnersBusy(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	ctx := context.Background()
-	orgName := rd.Spec.Template.Spec.Organization
-	minReplicas := *hra.Spec.MinReplicas
-	maxReplicas := *hra.Spec.MaxReplicas
 	metrics := hra.Spec.Metrics[0]
 	scaleUpThreshold := defaultScaleUpThreshold
 	scaleDownThreshold := defaultScaleDownThreshold
@@ -178,14 +217,34 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunn
 
 		scaleDownThreshold = sdt
 	}
-	if metrics.ScaleUpFactor != "" {
+
+	scaleUpAdjustment := metrics.ScaleUpAdjustment
+	if scaleUpAdjustment != 0 {
+		if metrics.ScaleUpAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpAdjustment cannot be lower than 0")
+		}
+
+		if metrics.ScaleUpFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleUpAdjustment and scaleUpFactor cannot be specified together")
+		}
+	} else if metrics.ScaleUpFactor != "" {
 		suf, err := strconv.ParseFloat(metrics.ScaleUpFactor, 64)
 		if err != nil {
 			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpFactor cannot be parsed into a float64")
 		}
 		scaleUpFactor = suf
 	}
-	if metrics.ScaleDownFactor != "" {
+
+	scaleDownAdjustment := metrics.ScaleDownAdjustment
+	if scaleDownAdjustment != 0 {
+		if metrics.ScaleDownAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownAdjustment cannot be lower than 0")
+		}
+
+		if metrics.ScaleDownFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleDownAdjustment and scaleDownFactor cannot be specified together")
+		}
+	} else if metrics.ScaleDownFactor != "" {
 		sdf, err := strconv.ParseFloat(metrics.ScaleDownFactor, 64)
 		if err != nil {
 			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownFactor cannot be parsed into a float64")
@@ -195,55 +254,114 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunn
 
 	// return the list of runners in namespace. Horizontal Runner Autoscaler should only be responsible for scaling resources in its own ns.
 	var runnerList v1alpha1.RunnerList
-	if err := r.List(ctx, &runnerList, client.InNamespace(rd.Namespace)); err != nil {
+
+	var opts []client.ListOption
+
+	opts = append(opts, client.InNamespace(rd.Namespace))
+
+	selector, err := metav1.LabelSelectorAsSelector(getSelector(&rd))
+	if err != nil {
 		return nil, err
 	}
+
+	opts = append(opts, client.MatchingLabelsSelector{Selector: selector})
+
+	r.Log.V(2).Info("Finding runners with selector", "ns", rd.Namespace)
+
+	if err := r.List(
+		ctx,
+		&runnerList,
+		opts...,
+	); err != nil {
+		if !kerrors.IsNotFound(err) {
+			return nil, err
+		}
+	}
+
 	runnerMap := make(map[string]struct{})
 	for _, items := range runnerList.Items {
 		runnerMap[items.Name] = struct{}{}
 	}
 
+	var (
+		enterprise   = rd.Spec.Template.Spec.Enterprise
+		organization = rd.Spec.Template.Spec.Organization
+		repository   = rd.Spec.Template.Spec.Repository
+	)
+
 	// ListRunners will return all runners managed by GitHub - not restricted to ns
-	runners, err := r.GitHubClient.ListRunners(ctx, orgName, "")
+	runners, err := r.GitHubClient.ListRunners(
+		ctx,
+		enterprise,
+		organization,
+		repository)
 	if err != nil {
 		return nil, err
 	}
-	numRunners := len(runnerList.Items)
-	numRunnersBusy := 0
+
+	var desiredReplicasBefore int
+
+	if v := rd.Spec.Replicas; v == nil {
+		desiredReplicasBefore = 1
+	} else {
+		desiredReplicasBefore = *v
+	}
+
+	var (
+		numRunners           int
+		numRunnersRegistered int
+		numRunnersBusy       int
+	)
+
+	numRunners = len(runnerList.Items)
+
 	for _, runner := range runners {
-		if _, ok := runnerMap[*runner.Name]; ok && runner.GetBusy() {
-			numRunnersBusy++
+		if _, ok := runnerMap[*runner.Name]; ok {
+			numRunnersRegistered++
+
+			if runner.GetBusy() {
+				numRunnersBusy++
+			}
 		}
 	}
 
 	var desiredReplicas int
-	fractionBusy := float64(numRunnersBusy) / float64(numRunners)
+	fractionBusy := float64(numRunnersBusy) / float64(desiredReplicasBefore)
 	if fractionBusy >= scaleUpThreshold {
-		desiredReplicas = int(math.Ceil(float64(numRunners) * scaleUpFactor))
+		if scaleUpAdjustment > 0 {
+			desiredReplicas = desiredReplicasBefore + scaleUpAdjustment
+		} else {
+			desiredReplicas = int(math.Ceil(float64(desiredReplicasBefore) * scaleUpFactor))
+		}
 	} else if fractionBusy < scaleDownThreshold {
-		desiredReplicas = int(float64(numRunners) * scaleDownFactor)
+		if scaleDownAdjustment > 0 {
+			desiredReplicas = desiredReplicasBefore - scaleDownAdjustment
+		} else {
+			desiredReplicas = int(float64(desiredReplicasBefore) * scaleDownFactor)
+		}
 	} else {
 		desiredReplicas = *rd.Spec.Replicas
 	}
 
-	if desiredReplicas < minReplicas {
-		desiredReplicas = minReplicas
-	} else if desiredReplicas > maxReplicas {
-		desiredReplicas = maxReplicas
-	}
+	// NOTES for operators:
+	//
+	// - num_runners can be as twice as large as replicas_desired_before while
+	//   the runnerdeployment controller is replacing RunnerReplicaSet for runner update.
 
 	r.Log.V(1).Info(
-		"Calculated desired replicas",
-		"computed_replicas_desired", desiredReplicas,
-		"spec_replicas_min", minReplicas,
-		"spec_replicas_max", maxReplicas,
-		"current_replicas", rd.Spec.Replicas,
+		fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas),
+		"replicas_desired_before", desiredReplicasBefore,
+		"replicas_desired", desiredReplicas,
 		"num_runners", numRunners,
+		"num_runners_registered", numRunnersRegistered,
 		"num_runners_busy", numRunnersBusy,
+		"namespace", hra.Namespace,
+		"runner_deployment", rd.Name,
+		"horizontal_runner_autoscaler", hra.Name,
+		"enterprise", enterprise,
+		"organization", organization,
+		"repository", repository,
 	)
 
-	rd.Status.Replicas = &desiredReplicas
-	replicas := desiredReplicas
-
-	return &replicas, nil
+	return &desiredReplicas, nil
 }

controllers/autoscaling_test.go

@@ -40,14 +40,18 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 	metav1Now := metav1.Now()
 	testcases := []struct {
-		repo         string
-		org          string
-		fixed        *int
-		max          *int
-		min          *int
-		sReplicas    *int
-		sTime        *metav1.Time
-		workflowRuns string
+		repo      string
+		org       string
+		fixed     *int
+		max       *int
+		min       *int
+		sReplicas *int
+		sTime     *metav1.Time
+
+		workflowRuns             string
+		workflowRuns_queued      string
+		workflowRuns_in_progress string
+
 		workflowJobs map[int]string
 		want         int
 		err          string
@@ -55,87 +59,107 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		// Legacy functionality
 		// 3 demanded, max at 3
 		{
-			repo:         "test/valid",
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         3,
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
 		},
 		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
 		{
-			repo:         "test/valid",
-			min:          intPtr(2),
-			max:          intPtr(3),
-			sReplicas:    intPtr(3),
-			sTime:        &metav1Now,
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         3,
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			sReplicas:                intPtr(3),
+			sTime:                    &metav1Now,
+			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     3,
 		},
 		// 3 demanded, max at 2
 		{
-			repo:         "test/valid",
-			min:          intPtr(2),
-			max:          intPtr(2),
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         2,
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(2),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     2,
 		},
 		// 2 demanded, min at 2
 		{
-			repo:         "test/valid",
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         2,
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
 		},
 		// 1 demanded, min at 2
 		{
-			repo:         "test/valid",
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
-			want:         2,
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     2,
 		},
 		// 1 demanded, min at 2
 		{
-			repo:         "test/valid",
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         2,
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
 		},
 		// 1 demanded, min at 1
 		{
-			repo:         "test/valid",
-			min:          intPtr(1),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
-			want:         1,
+			repo:                     "test/valid",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     1,
 		},
 		// 1 demanded, min at 1
 		{
-			repo:         "test/valid",
-			min:          intPtr(1),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         1,
+			repo:                     "test/valid",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     1,
 		},
 		// fixed at 3
 		{
-			repo:         "test/valid",
-			min:          intPtr(1),
-			max:          intPtr(3),
-			fixed:        intPtr(3),
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         3,
+			repo:                     "test/valid",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			fixed:                    intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
 		},
 
 		// Job-level autoscaling
 		// 5 requested from 3 workflows
 		{
-			repo:         "test/valid",
-			min:          intPtr(2),
-			max:          intPtr(10),
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
 			workflowJobs: map[int]string{
 				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
 				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
@@ -157,7 +181,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
+			server := fake.NewServer(
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
+				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
+				fake.WithListRunnersResponse(200, fake.RunnersListBody),
+			)
 			defer server.Close()
 			client := newGithubClient(server)
 
@@ -196,7 +224,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 				},
 			}
 
-			got, err := h.computeReplicas(rd, hra)
+			got, _, _, err := h.computeReplicasWithCache(log, metav1Now.Time, rd, hra)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)
@@ -206,12 +234,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 				return
 			}
 
-			if got == nil {
-				t.Fatalf("unexpected value of rs.Spec.Replicas: nil")
-			}
-
-			if *got != tc.want {
-				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, *got)
+			if got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, got)
 			}
 		})
 	}
@@ -224,129 +248,157 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 	metav1Now := metav1.Now()
 	testcases := []struct {
-		repos        []string
-		org          string
-		fixed        *int
-		max          *int
-		min          *int
-		sReplicas    *int
-		sTime        *metav1.Time
-		workflowRuns string
+		repos     []string
+		org       string
+		fixed     *int
+		max       *int
+		min       *int
+		sReplicas *int
+		sTime     *metav1.Time
+
+		workflowRuns             string
+		workflowRuns_queued      string
+		workflowRuns_in_progress string
+
 		workflowJobs map[int]string
 		want         int
 		err          string
 	}{
 		// 3 demanded, max at 3
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         3,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
 		},
 		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(2),
-			max:          intPtr(3),
-			sReplicas:    intPtr(3),
-			sTime:        &metav1Now,
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         3,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			sReplicas:                intPtr(3),
+			sTime:                    &metav1Now,
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     3,
 		},
 		// 3 demanded, max at 2
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(2),
-			max:          intPtr(2),
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         2,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(2),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     2,
 		},
 		// 2 demanded, min at 2
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         2,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
 		},
 		// 1 demanded, min at 2
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
-			want:         2,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     2,
 		},
 		// 1 demanded, min at 2
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(2),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         2,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
 		},
 		// 1 demanded, min at 1
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(1),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
-			want:         1,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     1,
 		},
 		// 1 demanded, min at 1
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(1),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         1,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     1,
 		},
 		// fixed at 3
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			fixed:        intPtr(1),
-			min:          intPtr(1),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         3,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			fixed:                    intPtr(1),
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
+			want:                     3,
 		},
 		// org runner, fixed at 3
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			fixed:        intPtr(1),
-			min:          intPtr(1),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
-			want:         3,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			fixed:                    intPtr(1),
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
+			want:                     3,
 		},
 		// org runner, 1 demanded, min at 1, no repos
 		{
-			org:          "test",
-			min:          intPtr(1),
-			max:          intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
-			err:          "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
+			org:                      "test",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			err:                      "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
 		},
 
 		// Job-level autoscaling
 		// 5 requested from 3 workflows
 		{
-			org:          "test",
-			repos:        []string{"valid"},
-			min:          intPtr(2),
-			max:          intPtr(10),
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
 			workflowJobs: map[int]string{
 				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
 				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
@@ -368,7 +420,13 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
+			t.Helper()
+
+			server := fake.NewServer(
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
+				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
+				fake.WithListRunnersResponse(200, fake.RunnersListBody),
+			)
 			defer server.Close()
 			client := newGithubClient(server)
 
@@ -383,7 +441,17 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 					Name: "testrd",
 				},
 				Spec: v1alpha1.RunnerDeploymentSpec{
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							"foo": "bar",
+						},
+					},
 					Template: v1alpha1.RunnerTemplate{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels: map[string]string{
+								"foo": "bar",
+							},
+						},
 						Spec: v1alpha1.RunnerSpec{
 							Organization: tc.org,
 						},
@@ -415,7 +483,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 				},
 			}
 
-			got, err := h.computeReplicas(rd, hra)
+			got, _, _, err := h.computeReplicasWithCache(log, metav1Now.Time, rd, hra)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)
@@ -425,12 +493,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 				return
 			}
 
-			if got == nil {
-				t.Fatalf("unexpected value of rs.Spec.Replicas: nil, wanted %v", tc.want)
-			}
-
-			if *got != tc.want {
-				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, *got)
+			if got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, got)
 			}
 		})
 	}

controllers/horizontal_runner_autoscaler_webhook.go

@@ -0,0 +1,458 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"net/http"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"strings"
+	"time"
+
+	"github.com/go-logr/logr"
+	gogithub "github.com/google/go-github/v33/github"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+const (
+	scaleTargetKey = "scaleTarget"
+)
+
+// HorizontalRunnerAutoscalerGitHubWebhook autoscales a HorizontalRunnerAutoscaler and the RunnerDeployment on each
+// GitHub Webhook received
+type HorizontalRunnerAutoscalerGitHubWebhook struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+
+	// SecretKeyBytes is the byte representation of the Webhook secret token
+	// the administrator is generated and specified in GitHub Web UI.
+	SecretKeyBytes []byte
+
+	// Namespace is the namespace to watch for HorizontalRunnerAutoscaler's to be
+	// scaled on Webhook.
+	// Set to empty for letting it watch for all namespaces.
+	Namespace string
+	Name      string
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+	return ctrl.Result{}, nil
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.ResponseWriter, r *http.Request) {
+	var (
+		ok bool
+
+		err error
+	)
+
+	defer func() {
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+
+			if err != nil {
+				msg := err.Error()
+				if written, err := w.Write([]byte(msg)); err != nil {
+					autoscaler.Log.Error(err, "failed writing http error response", "msg", msg, "written", written)
+				}
+			}
+		}
+	}()
+
+	defer func() {
+		if r.Body != nil {
+			r.Body.Close()
+		}
+	}()
+
+	// respond ok to GET / e.g. for health check
+	if r.Method == http.MethodGet {
+		fmt.Fprintln(w, "webhook server is running")
+		return
+	}
+
+	var payload []byte
+
+	if len(autoscaler.SecretKeyBytes) > 0 {
+		payload, err = gogithub.ValidatePayload(r, autoscaler.SecretKeyBytes)
+		if err != nil {
+			autoscaler.Log.Error(err, "error validating request body")
+
+			return
+		}
+	} else {
+		payload, err = ioutil.ReadAll(r.Body)
+		if err != nil {
+			autoscaler.Log.Error(err, "error reading request body")
+
+			return
+		}
+	}
+
+	webhookType := gogithub.WebHookType(r)
+	event, err := gogithub.ParseWebHook(webhookType, payload)
+	if err != nil {
+		var s string
+		if payload != nil {
+			s = string(payload)
+		}
+
+		autoscaler.Log.Error(err, "could not parse webhook", "webhookType", webhookType, "payload", s)
+
+		return
+	}
+
+	var target *ScaleTarget
+
+	log := autoscaler.Log.WithValues(
+		"event", webhookType,
+		"hookID", r.Header.Get("X-GitHub-Hook-ID"),
+		"delivery", r.Header.Get("X-GitHub-Delivery"),
+	)
+
+	switch e := event.(type) {
+	case *gogithub.PushEvent:
+		target, err = autoscaler.getScaleUpTarget(
+			context.TODO(),
+			log,
+			e.Repo.GetName(),
+			e.Repo.Owner.GetLogin(),
+			e.Repo.Owner.GetType(),
+			autoscaler.MatchPushEvent(e),
+		)
+	case *gogithub.PullRequestEvent:
+		target, err = autoscaler.getScaleUpTarget(
+			context.TODO(),
+			log,
+			e.Repo.GetName(),
+			e.Repo.Owner.GetLogin(),
+			e.Repo.Owner.GetType(),
+			autoscaler.MatchPullRequestEvent(e),
+		)
+
+		if pullRequest := e.PullRequest; pullRequest != nil {
+			log = log.WithValues(
+				"pullRequest.base.ref", e.PullRequest.Base.GetRef(),
+				"action", e.GetAction(),
+			)
+		}
+	case *gogithub.CheckRunEvent:
+		target, err = autoscaler.getScaleUpTarget(
+			context.TODO(),
+			log,
+			e.Repo.GetName(),
+			e.Repo.Owner.GetLogin(),
+			e.Repo.Owner.GetType(),
+			autoscaler.MatchCheckRunEvent(e),
+		)
+
+		if checkRun := e.GetCheckRun(); checkRun != nil {
+			log = log.WithValues(
+				"checkRun.status", checkRun.GetStatus(),
+				"action", e.GetAction(),
+			)
+		}
+	case *gogithub.PingEvent:
+		ok = true
+
+		w.WriteHeader(http.StatusOK)
+
+		msg := "pong"
+
+		if written, err := w.Write([]byte(msg)); err != nil {
+			log.Error(err, "failed writing http response", "msg", msg, "written", written)
+		}
+
+		log.Info("received ping event")
+
+		return
+	default:
+		log.Info("unknown event type", "eventType", webhookType)
+
+		return
+	}
+
+	if err != nil {
+		log.Error(err, "handling check_run event")
+
+		return
+	}
+
+	if target == nil {
+		log.Info(
+			"Scale target not found. If this is unexpected, ensure that there is exactly one repository-wide or organizational runner deployment that matches this webhook event",
+		)
+
+		msg := "no horizontalrunnerautoscaler to scale for this github event"
+
+		ok = true
+
+		w.WriteHeader(http.StatusOK)
+
+		if written, err := w.Write([]byte(msg)); err != nil {
+			log.Error(err, "failed writing http response", "msg", msg, "written", written)
+		}
+
+		return
+	}
+
+	if err := autoscaler.tryScaleUp(context.TODO(), target); err != nil {
+		log.Error(err, "could not scale up")
+
+		return
+	}
+
+	ok = true
+
+	w.WriteHeader(http.StatusOK)
+
+	msg := fmt.Sprintf("scaled %s by 1", target.Name)
+
+	autoscaler.Log.Info(msg)
+
+	if written, err := w.Write([]byte(msg)); err != nil {
+		log.Error(err, "failed writing http response", "msg", msg, "written", written)
+	}
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) findHRAsByKey(ctx context.Context, value string) ([]v1alpha1.HorizontalRunnerAutoscaler, error) {
+	ns := autoscaler.Namespace
+
+	var defaultListOpts []client.ListOption
+
+	if ns != "" {
+		defaultListOpts = append(defaultListOpts, client.InNamespace(ns))
+	}
+
+	var hras []v1alpha1.HorizontalRunnerAutoscaler
+
+	if value != "" {
+		opts := append([]client.ListOption{}, defaultListOpts...)
+		opts = append(opts, client.MatchingFields{scaleTargetKey: value})
+
+		if autoscaler.Namespace != "" {
+			opts = append(opts, client.InNamespace(autoscaler.Namespace))
+		}
+
+		var hraList v1alpha1.HorizontalRunnerAutoscalerList
+
+		if err := autoscaler.List(ctx, &hraList, opts...); err != nil {
+			return nil, err
+		}
+
+		for _, d := range hraList.Items {
+			hras = append(hras, d)
+		}
+	}
+
+	return hras, nil
+}
+
+func matchTriggerConditionAgainstEvent(types []string, eventAction *string) bool {
+	if len(types) == 0 {
+		return true
+	}
+
+	if eventAction == nil {
+		return false
+	}
+
+	for _, tpe := range types {
+		if tpe == *eventAction {
+			return true
+		}
+	}
+
+	return false
+}
+
+type ScaleTarget struct {
+	v1alpha1.HorizontalRunnerAutoscaler
+	v1alpha1.ScaleUpTrigger
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) searchScaleTargets(hras []v1alpha1.HorizontalRunnerAutoscaler, f func(v1alpha1.ScaleUpTrigger) bool) []ScaleTarget {
+	var matched []ScaleTarget
+
+	for _, hra := range hras {
+		if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		for _, scaleUpTrigger := range hra.Spec.ScaleUpTriggers {
+			if !f(scaleUpTrigger) {
+				continue
+			}
+
+			matched = append(matched, ScaleTarget{
+				HorizontalRunnerAutoscaler: hra,
+				ScaleUpTrigger:             scaleUpTrigger,
+			})
+		}
+	}
+
+	return matched
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleTarget(ctx context.Context, name string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) {
+	hras, err := autoscaler.findHRAsByKey(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	targets := autoscaler.searchScaleTargets(hras, f)
+
+	n := len(targets)
+
+	if n == 0 {
+		return nil, nil
+	}
+
+	if n > 1 {
+		var scaleTargetIDs []string
+
+		for _, t := range targets {
+			scaleTargetIDs = append(scaleTargetIDs, t.HorizontalRunnerAutoscaler.Name)
+		}
+
+		autoscaler.Log.Info(
+			"Found too many scale targets: "+
+				"It must be exactly one to avoid ambiguity. "+
+				"Either set Namespace for the webhook-based autoscaler to let it only find HRAs in the namespace, "+
+				"or update Repository or Organization fields in your RunnerDeployment resources to fix the ambiguity.",
+			"scaleTargets", strings.Join(scaleTargetIDs, ","))
+
+		return nil, nil
+	}
+
+	return &targets[0], nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTarget(ctx context.Context, log logr.Logger, repo, owner, ownerType string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) {
+	repositoryRunnerKey := owner + "/" + repo
+
+	if target, err := autoscaler.getScaleTarget(ctx, repositoryRunnerKey, f); err != nil {
+		autoscaler.Log.Info("finding repository-wide runner", "repository", repositoryRunnerKey)
+		return nil, err
+	} else if target != nil {
+		autoscaler.Log.Info("scale up target is repository-wide runners", "repository", repo)
+		return target, nil
+	}
+
+	if ownerType == "User" {
+		return nil, nil
+	}
+
+	if target, err := autoscaler.getScaleTarget(ctx, owner, f); err != nil {
+		log.Info("finding organizational runner", "organization", owner)
+		return nil, err
+	} else if target != nil {
+		log.Info("scale up target is organizational runners", "organization", owner)
+		return target, nil
+	}
+
+	return nil, nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScaleUp(ctx context.Context, target *ScaleTarget) error {
+	if target == nil {
+		return nil
+	}
+
+	copy := target.HorizontalRunnerAutoscaler.DeepCopy()
+
+	amount := 1
+
+	if target.ScaleUpTrigger.Amount > 0 {
+		amount = target.ScaleUpTrigger.Amount
+	}
+
+	capacityReservations := getValidCapacityReservations(copy)
+
+	copy.Spec.CapacityReservations = append(capacityReservations, v1alpha1.CapacityReservation{
+		ExpirationTime: metav1.Time{Time: time.Now().Add(target.ScaleUpTrigger.Duration.Duration)},
+		Replicas:       amount,
+	})
+
+	if err := autoscaler.Client.Patch(ctx, copy, client.MergeFrom(&target.HorizontalRunnerAutoscaler)); err != nil {
+		return fmt.Errorf("patching horizontalrunnerautoscaler to add capacity reservation: %w", err)
+	}
+
+	return nil
+}
+
+func getValidCapacityReservations(autoscaler *v1alpha1.HorizontalRunnerAutoscaler) []v1alpha1.CapacityReservation {
+	var capacityReservations []v1alpha1.CapacityReservation
+
+	now := time.Now()
+
+	for _, reservation := range autoscaler.Spec.CapacityReservations {
+		if reservation.ExpirationTime.Time.After(now) {
+			capacityReservations = append(capacityReservations, reservation)
+		}
+	}
+
+	return capacityReservations
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr ctrl.Manager) error {
+	name := "webhookbasedautoscaler"
+	if autoscaler.Name != "" {
+		name = autoscaler.Name
+	}
+
+	autoscaler.Recorder = mgr.GetEventRecorderFor(name)
+
+	if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.HorizontalRunnerAutoscaler{}, scaleTargetKey, func(rawObj runtime.Object) []string {
+		hra := rawObj.(*v1alpha1.HorizontalRunnerAutoscaler)
+
+		if hra.Spec.ScaleTargetRef.Name == "" {
+			return nil
+		}
+
+		var rd v1alpha1.RunnerDeployment
+
+		if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil {
+			return nil
+		}
+
+		return []string{rd.Spec.Template.Spec.Repository, rd.Spec.Template.Spec.Organization}
+	}); err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Named(name).
+		Complete(autoscaler)
+}

controllers/horizontal_runner_autoscaler_webhook_on_check_run.go

@@ -0,0 +1,43 @@
+package controllers
+
+import (
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/pkg/actionsglob"
+)
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchCheckRunEvent(event *github.CheckRunEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+	return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+		g := scaleUpTrigger.GitHubEvent
+
+		if g == nil {
+			return false
+		}
+
+		cr := g.CheckRun
+
+		if cr == nil {
+			return false
+		}
+
+		if !matchTriggerConditionAgainstEvent(cr.Types, event.Action) {
+			return false
+		}
+
+		if cr.Status != "" && (event.CheckRun == nil || event.CheckRun.Status == nil || *event.CheckRun.Status != cr.Status) {
+			return false
+		}
+
+		if checkRun := event.CheckRun; checkRun != nil && len(cr.Names) > 0 {
+			for _, pat := range cr.Names {
+				if r := actionsglob.Match(pat, checkRun.GetName()); r {
+					return true
+				}
+			}
+
+			return false
+		}
+
+		return true
+	}
+}

controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go

@@ -0,0 +1,32 @@
+package controllers
+
+import (
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPullRequestEvent(event *github.PullRequestEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+	return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+		g := scaleUpTrigger.GitHubEvent
+
+		if g == nil {
+			return false
+		}
+
+		pr := g.PullRequest
+
+		if pr == nil {
+			return false
+		}
+
+		if !matchTriggerConditionAgainstEvent(pr.Types, event.Action) {
+			return false
+		}
+
+		if !matchTriggerConditionAgainstEvent(pr.Branches, event.PullRequest.Base.Ref) {
+			return false
+		}
+
+		return true
+	}
+}

controllers/horizontal_runner_autoscaler_webhook_on_push.go

@@ -0,0 +1,24 @@
+package controllers
+
+import (
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event *github.PushEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+	return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+		g := scaleUpTrigger.GitHubEvent
+
+		if g == nil {
+			return false
+		}
+
+		push := g.Push
+
+		if push == nil {
+			return false
+		}
+
+		return true
+	}
+}

controllers/horizontal_runner_autoscaler_webhook_test.go

@@ -0,0 +1,314 @@
+package controllers
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/go-logr/logr"
+	"github.com/google/go-github/v33/github"
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"io"
+	"io/ioutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"testing"
+	"time"
+)
+
+var (
+	sc = runtime.NewScheme()
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(sc)
+	_ = actionsv1alpha1.AddToScheme(sc)
+}
+
+func TestOrgWebhookCheckRun(t *testing.T) {
+	f, err := os.Open("testdata/org_webhook_check_run_payload.json")
+	if err != nil {
+		t.Fatalf("could not open the fixture: %s", err)
+	}
+	defer f.Close()
+	var e github.CheckRunEvent
+	if err := json.NewDecoder(f).Decode(&e); err != nil {
+		t.Fatalf("invalid json: %s", err)
+	}
+	testServer(t,
+		"check_run",
+		&e,
+		200,
+		"no horizontalrunnerautoscaler to scale for this github event",
+	)
+}
+
+func TestRepoWebhookCheckRun(t *testing.T) {
+	f, err := os.Open("testdata/repo_webhook_check_run_payload.json")
+	if err != nil {
+		t.Fatalf("could not open the fixture: %s", err)
+	}
+	defer f.Close()
+	var e github.CheckRunEvent
+	if err := json.NewDecoder(f).Decode(&e); err != nil {
+		t.Fatalf("invalid json: %s", err)
+	}
+	testServer(t,
+		"check_run",
+		&e,
+		200,
+		"no horizontalrunnerautoscaler to scale for this github event",
+	)
+}
+
+func TestWebhookPullRequest(t *testing.T) {
+	testServer(t,
+		"pull_request",
+		&github.PullRequestEvent{
+			PullRequest: &github.PullRequest{
+				Base: &github.PullRequestBranch{
+					Ref: github.String("main"),
+				},
+			},
+			Repo: &github.Repository{
+				Name: github.String("myorg/myrepo"),
+				Organization: &github.Organization{
+					Name: github.String("myorg"),
+				},
+			},
+			Action: github.String("created"),
+		},
+		200,
+		"no horizontalrunnerautoscaler to scale for this github event",
+	)
+}
+
+func TestWebhookPush(t *testing.T) {
+	testServer(t,
+		"push",
+		&github.PushEvent{
+			Repo: &github.PushEventRepository{
+				Name:         github.String("myrepo"),
+				Organization: github.String("myorg"),
+			},
+		},
+		200,
+		"no horizontalrunnerautoscaler to scale for this github event",
+	)
+}
+
+func TestWebhookPing(t *testing.T) {
+	testServer(t,
+		"ping",
+		&github.PingEvent{
+			Zen: github.String("zen"),
+		},
+		200,
+		"pong",
+	)
+}
+
+func TestGetRequest(t *testing.T) {
+	hra := HorizontalRunnerAutoscalerGitHubWebhook{}
+	request, _ := http.NewRequest(http.MethodGet, "/", nil)
+	recorder := httptest.ResponseRecorder{}
+
+	hra.Handle(&recorder, request)
+	response := recorder.Result()
+
+	if response.StatusCode != http.StatusOK {
+		t.Errorf("want %d, got %d", http.StatusOK, response.StatusCode)
+	}
+}
+
+func TestGetValidCapacityReservations(t *testing.T) {
+	now := time.Now()
+
+	hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+		Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+			CapacityReservations: []actionsv1alpha1.CapacityReservation{
+				{
+					ExpirationTime: metav1.Time{Time: now.Add(-time.Second)},
+					Replicas:       1,
+				},
+				{
+					ExpirationTime: metav1.Time{Time: now},
+					Replicas:       2,
+				},
+				{
+					ExpirationTime: metav1.Time{Time: now.Add(time.Second)},
+					Replicas:       3,
+				},
+			},
+		},
+	}
+
+	revs := getValidCapacityReservations(hra)
+
+	var count int
+
+	for _, r := range revs {
+		count += r.Replicas
+	}
+
+	want := 3
+
+	if count != want {
+		t.Errorf("want %d, got %d", want, count)
+	}
+}
+
+func installTestLogger(webhook *HorizontalRunnerAutoscalerGitHubWebhook) *bytes.Buffer {
+	logs := &bytes.Buffer{}
+
+	log := testLogger{
+		name:   "testlog",
+		writer: logs,
+	}
+
+	webhook.Log = &log
+
+	return logs
+}
+
+func testServer(t *testing.T, eventType string, event interface{}, wantCode int, wantBody string) {
+	t.Helper()
+
+	hraWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{}
+
+	var initObjs []runtime.Object
+
+	client := fake.NewFakeClientWithScheme(sc, initObjs...)
+
+	logs := installTestLogger(hraWebhook)
+
+	defer func() {
+		if t.Failed() {
+			t.Logf("diagnostics: %s", logs.String())
+		}
+	}()
+
+	hraWebhook.Client = client
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", hraWebhook.Handle)
+
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	resp, err := sendWebhook(server, eventType, event)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if resp != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	if resp.StatusCode != wantCode {
+		t.Error("status:", resp.StatusCode)
+	}
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(respBody) != wantBody {
+		t.Fatal("body:", string(respBody))
+	}
+}
+
+func sendWebhook(server *httptest.Server, eventType string, event interface{}) (*http.Response, error) {
+	jsonBuf := &bytes.Buffer{}
+	enc := json.NewEncoder(jsonBuf)
+	enc.SetIndent("  ", "")
+	err := enc.Encode(event)
+	if err != nil {
+		return nil, fmt.Errorf("[bug in test] encoding event to json: %+v", err)
+	}
+
+	reqBody := jsonBuf.Bytes()
+
+	u, err := url.Parse(server.URL)
+	if err != nil {
+		return nil, fmt.Errorf("parsing server url: %v", err)
+	}
+
+	req := &http.Request{
+		Method: http.MethodPost,
+		URL:    u,
+		Header: map[string][]string{
+			"X-GitHub-Event": {eventType},
+			"Content-Type":   {"application/json"},
+		},
+		Body: ioutil.NopCloser(bytes.NewBuffer(reqBody)),
+	}
+
+	return http.DefaultClient.Do(req)
+}
+
+// testLogger is a sample logr.Logger that logs in-memory.
+// It's only for testing log outputs.
+type testLogger struct {
+	name      string
+	keyValues map[string]interface{}
+
+	writer io.Writer
+}
+
+var _ logr.Logger = &testLogger{}
+
+func (l *testLogger) Info(msg string, kvs ...interface{}) {
+	fmt.Fprintf(l.writer, "%s] %s\t", l.name, msg)
+	for k, v := range l.keyValues {
+		fmt.Fprintf(l.writer, "%s=%+v ", k, v)
+	}
+	for i := 0; i < len(kvs); i += 2 {
+		fmt.Fprintf(l.writer, "%s=%+v ", kvs[i], kvs[i+1])
+	}
+	fmt.Fprintf(l.writer, "\n")
+}
+
+func (_ *testLogger) Enabled() bool {
+	return true
+}
+
+func (l *testLogger) Error(err error, msg string, kvs ...interface{}) {
+	kvs = append(kvs, "error", err)
+	l.Info(msg, kvs...)
+}
+
+func (l *testLogger) V(_ int) logr.InfoLogger {
+	return l
+}
+
+func (l *testLogger) WithName(name string) logr.Logger {
+	return &testLogger{
+		name:      l.name + "." + name,
+		keyValues: l.keyValues,
+		writer:    l.writer,
+	}
+}
+
+func (l *testLogger) WithValues(kvs ...interface{}) logr.Logger {
+	newMap := make(map[string]interface{}, len(l.keyValues)+len(kvs)/2)
+	for k, v := range l.keyValues {
+		newMap[k] = v
+	}
+	for i := 0; i < len(kvs); i += 2 {
+		newMap[kvs[i].(string)] = kvs[i+1]
+	}
+	return &testLogger{
+		name:      l.name,
+		keyValues: newMap,
+		writer:    l.writer,
+	}
+}

controllers/horizontalrunnerautoscaler_controller.go

@@ -18,6 +18,8 @@ package controllers
 
 import (
 	"context"
+	"fmt"
+	corev1 "k8s.io/api/core/v1"
 	"time"
 
 	"github.com/summerwind/actions-runner-controller/github"
@@ -29,10 +31,10 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/controllers/metrics"
 )
 
 const (
@@ -46,8 +48,13 @@ type HorizontalRunnerAutoscalerReconciler struct {
 	Log          logr.Logger
 	Recorder     record.EventRecorder
 	Scheme       *runtime.Scheme
+
+	CacheDuration time.Duration
+	Name          string
 }
 
+const defaultReplicas = 1
+
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
@@ -67,6 +74,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl
 		return ctrl.Result{}, nil
 	}
 
+	metrics.SetHorizontalRunnerAutoscalerSpec(hra.ObjectMeta, hra.Spec)
+
 	var rd v1alpha1.RunnerDeployment
 	if err := r.Get(ctx, types.NamespacedName{
 		Namespace: req.Namespace,
@@ -79,7 +88,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl
 		return ctrl.Result{}, nil
 	}
 
-	replicas, err := r.computeReplicas(rd, hra)
+	now := time.Now()
+
+	newDesiredReplicas, computedReplicas, computedReplicasFromCache, err := r.computeReplicasWithCache(log, now, rd, hra)
 	if err != nil {
 		r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -88,62 +99,144 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl
 		return ctrl.Result{}, err
 	}
 
-	const defaultReplicas = 1
-
 	currentDesiredReplicas := getIntOrDefault(rd.Spec.Replicas, defaultReplicas)
-	newDesiredReplicas := getIntOrDefault(replicas, defaultReplicas)
 
 	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
 	if currentDesiredReplicas != newDesiredReplicas {
 		copy := rd.DeepCopy()
 		copy.Spec.Replicas = &newDesiredReplicas
 
-		if err := r.Client.Update(ctx, copy); err != nil {
-			log.Error(err, "Failed to update runnerderployment resource")
-
-			return ctrl.Result{}, err
+		if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rd)); err != nil {
+			return ctrl.Result{}, fmt.Errorf("patching runnerdeployment to have %d replicas: %w", newDesiredReplicas, err)
 		}
-
-		return ctrl.Result{}, err
 	}
 
-	if hra.Status.DesiredReplicas == nil || *hra.Status.DesiredReplicas != *replicas {
-		updated := hra.DeepCopy()
+	var updated *v1alpha1.HorizontalRunnerAutoscaler
 
-		if (hra.Status.DesiredReplicas == nil && *replicas > 1) ||
-			(hra.Status.DesiredReplicas != nil && *replicas > *hra.Status.DesiredReplicas) {
+	if hra.Status.DesiredReplicas == nil || *hra.Status.DesiredReplicas != newDesiredReplicas {
+		updated = hra.DeepCopy()
+
+		if (hra.Status.DesiredReplicas == nil && newDesiredReplicas > 1) ||
+			(hra.Status.DesiredReplicas != nil && newDesiredReplicas > *hra.Status.DesiredReplicas) {
 
 			updated.Status.LastSuccessfulScaleOutTime = &metav1.Time{Time: time.Now()}
 		}
 
-		updated.Status.DesiredReplicas = replicas
+		updated.Status.DesiredReplicas = &newDesiredReplicas
+	}
 
-		if err := r.Status().Update(ctx, updated); err != nil {
-			log.Error(err, "Failed to update horizontalrunnerautoscaler status")
+	if computedReplicasFromCache == nil {
+		if updated == nil {
+			updated = hra.DeepCopy()
+		}
 
-			return ctrl.Result{}, err
+		cacheEntries := getValidCacheEntries(updated, now)
+
+		var cacheDuration time.Duration
+
+		if r.CacheDuration > 0 {
+			cacheDuration = r.CacheDuration
+		} else {
+			cacheDuration = 10 * time.Minute
+		}
+
+		updated.Status.CacheEntries = append(cacheEntries, v1alpha1.CacheEntry{
+			Key:            v1alpha1.CacheEntryKeyDesiredReplicas,
+			Value:          computedReplicas,
+			ExpirationTime: metav1.Time{Time: time.Now().Add(cacheDuration)},
+		})
+	}
+
+	if updated != nil {
+		metrics.SetHorizontalRunnerAutoscalerStatus(updated.ObjectMeta, updated.Status)
+
+		if err := r.Status().Patch(ctx, updated, client.MergeFrom(&hra)); err != nil {
+			return ctrl.Result{}, fmt.Errorf("patching horizontalrunnerautoscaler status to add cache entry: %w", err)
 		}
 	}
 
 	return ctrl.Result{}, nil
 }
 
+func getValidCacheEntries(hra *v1alpha1.HorizontalRunnerAutoscaler, now time.Time) []v1alpha1.CacheEntry {
+	var cacheEntries []v1alpha1.CacheEntry
+
+	for _, ent := range hra.Status.CacheEntries {
+		if ent.ExpirationTime.After(now) {
+			cacheEntries = append(cacheEntries, ent)
+		}
+	}
+
+	return cacheEntries
+}
+
 func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller")
+	name := "horizontalrunnerautoscaler-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Named(name).
 		Complete(r)
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) computeReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
-	var computedReplicas *int
-
-	replicas, err := r.determineDesiredReplicas(rd, hra)
-	if err != nil {
-		return nil, err
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (int, int, *int, error) {
+	minReplicas := defaultReplicas
+	if hra.Spec.MinReplicas != nil && *hra.Spec.MinReplicas > 0 {
+		minReplicas = *hra.Spec.MinReplicas
 	}
 
+	var suggestedReplicas int
+
+	suggestedReplicasFromCache := r.fetchSuggestedReplicasFromCache(hra)
+
+	var cached *int
+
+	if suggestedReplicasFromCache != nil {
+		cached = suggestedReplicasFromCache
+
+		if cached == nil {
+			suggestedReplicas = minReplicas
+		} else {
+			suggestedReplicas = *cached
+		}
+	} else {
+		v, err := r.suggestDesiredReplicas(rd, hra)
+		if err != nil {
+			return 0, 0, nil, err
+		}
+
+		if v == nil {
+			suggestedReplicas = minReplicas
+		} else {
+			suggestedReplicas = *v
+		}
+	}
+
+	var reserved int
+
+	for _, reservation := range hra.Spec.CapacityReservations {
+		if reservation.ExpirationTime.Time.After(now) {
+			reserved += reservation.Replicas
+		}
+	}
+
+	newDesiredReplicas := suggestedReplicas + reserved
+
+	if newDesiredReplicas < minReplicas {
+		newDesiredReplicas = minReplicas
+	} else if hra.Spec.MaxReplicas != nil && newDesiredReplicas > *hra.Spec.MaxReplicas {
+		newDesiredReplicas = *hra.Spec.MaxReplicas
+	}
+
+	//
+	// Delay scaling-down for ScaleDownDelaySecondsAfterScaleUp or DefaultScaleDownDelay
+	//
+
 	var scaleDownDelay time.Duration
 
 	if hra.Spec.ScaleDownDelaySecondsAfterScaleUp != nil {
@@ -152,17 +245,50 @@ func (r *HorizontalRunnerAutoscalerReconciler) computeReplicas(rd v1alpha1.Runne
 		scaleDownDelay = DefaultScaleDownDelay
 	}
 
-	now := time.Now()
+	var scaleDownDelayUntil *time.Time
 
 	if hra.Status.DesiredReplicas == nil ||
-		*hra.Status.DesiredReplicas < *replicas ||
-		hra.Status.LastSuccessfulScaleOutTime == nil ||
-		hra.Status.LastSuccessfulScaleOutTime.Add(scaleDownDelay).Before(now) {
+		*hra.Status.DesiredReplicas < newDesiredReplicas ||
+		hra.Status.LastSuccessfulScaleOutTime == nil {
 
-		computedReplicas = replicas
+	} else if hra.Status.LastSuccessfulScaleOutTime != nil {
+		t := hra.Status.LastSuccessfulScaleOutTime.Add(scaleDownDelay)
+
+		// ScaleDownDelay is not passed
+		if t.After(now) {
+			scaleDownDelayUntil = &t
+			newDesiredReplicas = *hra.Status.DesiredReplicas
+		}
 	} else {
-		computedReplicas = hra.Status.DesiredReplicas
+		newDesiredReplicas = *hra.Status.DesiredReplicas
 	}
 
-	return computedReplicas, nil
+	//
+	// Logs various numbers for monitoring and debugging purpose
+	//
+
+	kvs := []interface{}{
+		"suggested", suggestedReplicas,
+		"reserved", reserved,
+		"min", minReplicas,
+	}
+
+	if cached != nil {
+		kvs = append(kvs, "cached", *cached)
+	}
+
+	if scaleDownDelayUntil != nil {
+		kvs = append(kvs, "last_scale_up_time", *hra.Status.LastSuccessfulScaleOutTime)
+		kvs = append(kvs, "scale_down_delay_until", scaleDownDelayUntil)
+	}
+
+	if maxReplicas := hra.Spec.MaxReplicas; maxReplicas != nil {
+		kvs = append(kvs, "max", *maxReplicas)
+	}
+
+	log.V(1).Info(fmt.Sprintf("Calculated desired replicas of %d", newDesiredReplicas),
+		kvs...,
+	)
+
+	return newDesiredReplicas, suggestedReplicas, suggestedReplicasFromCache, nil
 }

controllers/horizontalrunnerautoscaler_controller_test.go

@@ -0,0 +1,49 @@
+package controllers
+
+import (
+	"github.com/google/go-cmp/cmp"
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"testing"
+	"time"
+)
+
+func TestGetValidCacheEntries(t *testing.T) {
+	now := time.Now()
+
+	hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+		Status: actionsv1alpha1.HorizontalRunnerAutoscalerStatus{
+			CacheEntries: []actionsv1alpha1.CacheEntry{
+				{
+					Key:            "foo",
+					Value:          1,
+					ExpirationTime: metav1.Time{Time: now.Add(-time.Second)},
+				},
+				{
+					Key:            "foo",
+					Value:          2,
+					ExpirationTime: metav1.Time{Time: now},
+				},
+				{
+					Key:            "foo",
+					Value:          3,
+					ExpirationTime: metav1.Time{Time: now.Add(time.Second)},
+				},
+			},
+		},
+	}
+
+	revs := getValidCacheEntries(hra, now)
+
+	counts := map[string]int{}
+
+	for _, r := range revs {
+		counts[r.Key] += r.Value
+	}
+
+	want := map[string]int{"foo": 3}
+
+	if d := cmp.Diff(want, counts); d != "" {
+		t.Errorf("%s", d)
+	}
+}

controllers/metrics/horizontalrunnerautoscaler.go

@@ -0,0 +1,67 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	hraName      = "horizontalrunnerautoscaler"
+	hraNamespace = "namespace"
+)
+
+var (
+	horizontalRunnerAutoscalerMetrics = []prometheus.Collector{
+		horizontalRunnerAutoscalerMinReplicas,
+		horizontalRunnerAutoscalerMaxReplicas,
+		horizontalRunnerAutoscalerDesiredReplicas,
+	}
+)
+
+var (
+	horizontalRunnerAutoscalerMinReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_spec_min_replicas",
+			Help: "minReplicas of HorizontalRunnerAutoscaler",
+		},
+		[]string{hraName, hraNamespace},
+	)
+	horizontalRunnerAutoscalerMaxReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_spec_max_replicas",
+			Help: "maxReplicas of HorizontalRunnerAutoscaler",
+		},
+		[]string{hraName, hraNamespace},
+	)
+	horizontalRunnerAutoscalerDesiredReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_status_desired_replicas",
+			Help: "desiredReplicas of HorizontalRunnerAutoscaler",
+		},
+		[]string{hraName, hraNamespace},
+	)
+)
+
+func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) {
+	labels := prometheus.Labels{
+		hraName:      o.Name,
+		hraNamespace: o.Namespace,
+	}
+	if spec.MaxReplicas != nil {
+		horizontalRunnerAutoscalerMaxReplicas.With(labels).Set(float64(*spec.MaxReplicas))
+	}
+	if spec.MinReplicas != nil {
+		horizontalRunnerAutoscalerMinReplicas.With(labels).Set(float64(*spec.MinReplicas))
+	}
+}
+
+func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.HorizontalRunnerAutoscalerStatus) {
+	labels := prometheus.Labels{
+		hraName:      o.Name,
+		hraNamespace: o.Namespace,
+	}
+	if status.DesiredReplicas != nil {
+		horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas))
+	}
+}

controllers/metrics/metrics.go

@@ -0,0 +1,14 @@
+// Package metrics provides the metrics of custom resources such as HRA.
+//
+// This depends on the metrics exporter of kubebuilder.
+// See https://book.kubebuilder.io/reference/metrics.html for details.
+package metrics
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+func init() {
+	metrics.Registry.MustRegister(runnerDeploymentMetrics...)
+	metrics.Registry.MustRegister(horizontalRunnerAutoscalerMetrics...)
+}

controllers/metrics/runnerdeployment.go

@@ -0,0 +1,37 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+const (
+	rdName      = "runnerdeployment"
+	rdNamespace = "namespace"
+)
+
+var (
+	runnerDeploymentMetrics = []prometheus.Collector{
+		runnerDeploymentReplicas,
+	}
+)
+
+var (
+	runnerDeploymentReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "runnerdeployment_spec_replicas",
+			Help: "replicas of RunnerDeployment",
+		},
+		[]string{rdName, rdNamespace},
+	)
+)
+
+func SetRunnerDeployment(rd v1alpha1.RunnerDeployment) {
+	labels := prometheus.Labels{
+		rdName:      rd.Name,
+		rdNamespace: rd.Namespace,
+	}
+	if rd.Spec.Replicas != nil {
+		runnerDeploymentReplicas.With(labels).Set(float64(*rd.Spec.Replicas))
+	}
+}

controllers/runner_controller.go

@@ -18,12 +18,16 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	gogithub "github.com/google/go-github/v33/github"
 	"github.com/summerwind/actions-runner-controller/hash"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"strings"
+	"time"
 
 	"github.com/go-logr/logr"
-	"k8s.io/apimachinery/pkg/api/errors"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -41,17 +45,22 @@ const (
 	finalizerName = "runner.actions.summerwind.dev"
 
 	LabelKeyPodTemplateHash = "pod-template-hash"
+
+	retryDelayOnGitHubAPIRateLimitError = 30 * time.Second
 )
 
 // RunnerReconciler reconciles a Runner object
 type RunnerReconciler struct {
 	client.Client
-	Log          logr.Logger
-	Recorder     record.EventRecorder
-	Scheme       *runtime.Scheme
-	GitHubClient *github.Client
-	RunnerImage  string
-	DockerImage  string
+	Log                         logr.Logger
+	Recorder                    record.EventRecorder
+	Scheme                      *runtime.Scheme
+	GitHubClient                *github.Client
+	RunnerImage                 string
+	DockerImage                 string
+	Name                        string
+	RegistrationRecheckInterval time.Duration
+	RegistrationRecheckJitter   time.Duration
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
@@ -95,9 +104,22 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 
 		if removed {
 			if len(runner.Status.Registration.Token) > 0 {
-				ok, err := r.unregisterRunner(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+				ok, err := r.unregisterRunner(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 				if err != nil {
-					log.Error(err, "Failed to unregister runner")
+					if errors.Is(err, &gogithub.RateLimitError{}) {
+						// We log the underlying error when we failed calling GitHub API to list or unregisters,
+						// or the runner is still busy.
+						log.Error(
+							err,
+							fmt.Sprintf(
+								"Failed to unregister runner due to GitHub API rate limits. Delaying retry for %s to avoid excessive GitHub API calls",
+								retryDelayOnGitHubAPIRateLimitError,
+							),
+						)
+
+						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+					}
+
 					return ctrl.Result{}, err
 				}
 
@@ -111,8 +133,8 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			newRunner := runner.DeepCopy()
 			newRunner.ObjectMeta.Finalizers = finalizers
 
-			if err := r.Update(ctx, newRunner); err != nil {
-				log.Error(err, "Failed to update runner")
+			if err := r.Patch(ctx, newRunner, client.MergeFrom(&runner)); err != nil {
+				log.Error(err, "Failed to update runner for finalizer removal")
 				return ctrl.Result{}, err
 			}
 
@@ -124,7 +146,7 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 
 	var pod corev1.Pod
 	if err := r.Get(ctx, req.NamespacedName, &pod); err != nil {
-		if !errors.IsNotFound(err) {
+		if !kerrors.IsNotFound(err) {
 			return ctrl.Result{}, err
 		}
 
@@ -141,35 +163,65 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		}
 
 		if err := r.Create(ctx, &newPod); err != nil {
+			if kerrors.IsAlreadyExists(err) {
+				// Gracefully handle pod-already-exists errors due to informer cache delay.
+				// Without this we got a few errors like the below on new runner pod:
+				// 2021-03-16T00:23:10.116Z        ERROR   controller-runtime.controller   Reconciler error      {"controller": "runner-controller", "request": "default/example-runnerdeploy-b2g2g-j4mcp", "error": "pods \"example-runnerdeploy-b2g2g-j4mcp\" already exists"}
+				log.Info(
+					"Failed to create pod due to AlreadyExists error. Probably this pod has been already created in previous reconcilation but is still not in the informer cache. Will retry on pod created. If it doesn't repeat, there's no problem",
+				)
+
+				return ctrl.Result{}, nil
+			}
+
 			log.Error(err, "Failed to create pod resource")
+
 			return ctrl.Result{}, err
 		}
 
 		r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodCreated", fmt.Sprintf("Created pod '%s'", newPod.Name))
 		log.Info("Created runner pod", "repository", runner.Spec.Repository)
 	} else {
+		if !pod.ObjectMeta.DeletionTimestamp.IsZero() {
+			deletionTimeout := 1 * time.Minute
+			currentTime := time.Now()
+			deletionDidTimeout := currentTime.Sub(pod.DeletionTimestamp.Add(deletionTimeout)) > 0
+
+			if deletionDidTimeout {
+				log.Info(
+					fmt.Sprintf("Failed to delete pod within %s. ", deletionTimeout)+
+						"This is typically the case when a Kubernetes node became unreachable "+
+						"and the kube controller started evicting nodes. Forcefully deleting the pod to not get stuck.",
+					"podDeletionTimestamp", pod.DeletionTimestamp,
+					"currentTime", currentTime,
+					"configuredDeletionTimeout", deletionTimeout,
+				)
+
+				var force int64 = 0
+				// forcefully delete runner as we would otherwise get stuck if the node stays unreachable
+				if err := r.Delete(ctx, &pod, &client.DeleteOptions{GracePeriodSeconds: &force}); err != nil {
+					// probably
+					if !kerrors.IsNotFound(err) {
+						log.Error(err, "Failed to forcefully delete pod resource ...")
+						return ctrl.Result{}, err
+					}
+					// forceful deletion finally succeeded
+					return ctrl.Result{Requeue: true}, nil
+				}
+
+				r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodDeleted", fmt.Sprintf("Forcefully deleted pod '%s'", pod.Name))
+				log.Info("Forcefully deleted runner pod", "repository", runner.Spec.Repository)
+				// give kube manager a little time to forcefully delete the stuck pod
+				return ctrl.Result{RequeueAfter: 3 * time.Second}, err
+			} else {
+				return ctrl.Result{}, err
+			}
+		}
+
 		// If pod has ended up succeeded we need to restart it
 		// Happens e.g. when dind is in runner and run completes
 		restart := pod.Status.Phase == corev1.PodSucceeded
 
-		if !restart && runner.Status.Phase != string(pod.Status.Phase) {
-			updated := runner.DeepCopy()
-			updated.Status.Phase = string(pod.Status.Phase)
-			updated.Status.Reason = pod.Status.Reason
-			updated.Status.Message = pod.Status.Message
-
-			if err := r.Status().Update(ctx, updated); err != nil {
-				log.Error(err, "Failed to update runner status")
-				return ctrl.Result{}, err
-			}
-
-			return ctrl.Result{}, nil
-		}
-
-		if !pod.ObjectMeta.DeletionTimestamp.IsZero() {
-			return ctrl.Result{}, err
-		}
-
 		if pod.Status.Phase == corev1.PodRunning {
 			for _, status := range pod.Status.ContainerStatuses {
 				if status.Name != containerName {
@@ -194,24 +246,183 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, err
 		}
 
-		runnerBusy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
-		if err != nil {
-			log.Error(err, "Failed to check if runner is busy")
-			return ctrl.Result{}, nil
-		}
+		var registrationRecheckDelay time.Duration
 
-		// See the `newPod` function called above for more information
-		// about when this hash changes.
-		curHash := pod.Labels[LabelKeyPodTemplateHash]
-		newHash := newPod.Labels[LabelKeyPodTemplateHash]
+		// all checks done below only decide whether a restart is needed
+		// if a restart was already decided before, there is no need for the checks
+		// saving API calls and scary{ log messages
+		if !restart {
+			registrationCheckInterval := time.Minute
+			if r.RegistrationRecheckInterval > 0 {
+				registrationCheckInterval = r.RegistrationRecheckInterval
+			}
 
-		if !runnerBusy && curHash != newHash {
-			restart = true
+			// We want to call ListRunners GitHub Actions API only once per runner per minute.
+			// This if block, in conjunction with:
+			//   return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil
+			// achieves that.
+			if lastCheckTime := runner.Status.LastRegistrationCheckTime; lastCheckTime != nil {
+				nextCheckTime := lastCheckTime.Add(registrationCheckInterval)
+				now := time.Now()
+
+				// Requeue scheduled by RequeueAfter can happen a bit earlier (like dozens of milliseconds)
+				// so to avoid excessive, in-effective retry, we heuristically ignore the remaining delay in case it is
+				// shorter than 1s
+				requeueAfter := nextCheckTime.Sub(now) - time.Second
+				if requeueAfter > 0 {
+					log.Info(
+						fmt.Sprintf("Skipped registration check because it's deferred until %s. Retrying in %s at latest", nextCheckTime, requeueAfter),
+						"lastRegistrationCheckTime", lastCheckTime,
+						"registrationCheckInterval", registrationCheckInterval,
+					)
+
+					// Without RequeueAfter, the controller may not retry on scheduled. Instead, it must wait until the
+					// next sync period passes, which can be too much later than nextCheckTime.
+					//
+					// We need to requeue on this reconcilation even though we have already scheduled the initial
+					// requeue previously with `return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil`.
+					// Apparently, the workqueue used by controller-runtime seems to deduplicate and resets the delay on
+					// other requeues- so the initial scheduled requeue may have been reset due to requeue on
+					// spec/status change.
+					return ctrl.Result{RequeueAfter: requeueAfter}, nil
+				}
+			}
+
+			notFound := false
+			offline := false
+
+			runnerBusy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+
+			currentTime := time.Now()
+
+			if err != nil {
+				var notFoundException *github.RunnerNotFound
+				var offlineException *github.RunnerOffline
+				if errors.As(err, &notFoundException) {
+					notFound = true
+				} else if errors.As(err, &offlineException) {
+					offline = true
+				} else {
+					var e *gogithub.RateLimitError
+					if errors.As(err, &e) {
+						// We log the underlying error when we failed calling GitHub API to list or unregisters,
+						// or the runner is still busy.
+						log.Error(
+							err,
+							fmt.Sprintf(
+								"Failed to check if runner is busy due to Github API rate limit. Retrying in %s to avoid excessive GitHub API calls",
+								retryDelayOnGitHubAPIRateLimitError,
+							),
+						)
+
+						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+					}
+
+					return ctrl.Result{}, err
+				}
+			}
+
+			// See the `newPod` function called above for more information
+			// about when this hash changes.
+			curHash := pod.Labels[LabelKeyPodTemplateHash]
+			newHash := newPod.Labels[LabelKeyPodTemplateHash]
+
+			if !runnerBusy && curHash != newHash {
+				restart = true
+			}
+
+			registrationTimeout := 10 * time.Minute
+			durationAfterRegistrationTimeout := currentTime.Sub(pod.CreationTimestamp.Add(registrationTimeout))
+			registrationDidTimeout := durationAfterRegistrationTimeout > 0
+
+			if notFound {
+				if registrationDidTimeout {
+					log.Info(
+						"Runner failed to register itself to GitHub in timely manner. "+
+							"Recreating the pod to see if it resolves the issue. "+
+							"CAUTION: If you see this a lot, you should investigate the root cause. "+
+							"See https://github.com/summerwind/actions-runner-controller/issues/288",
+						"podCreationTimestamp", pod.CreationTimestamp,
+						"currentTime", currentTime,
+						"configuredRegistrationTimeout", registrationTimeout,
+					)
+
+					restart = true
+				} else {
+					log.V(1).Info(
+						"Runner pod exists but we failed to check if runner is busy. Apparently it still needs more time.",
+						"runnerName", runner.Name,
+					)
+				}
+			} else if offline {
+				if registrationDidTimeout {
+					log.Info(
+						"Already existing GitHub runner still appears offline . "+
+							"Recreating the pod to see if it resolves the issue. "+
+							"CAUTION: If you see this a lot, you should investigate the root cause. ",
+						"podCreationTimestamp", pod.CreationTimestamp,
+						"currentTime", currentTime,
+						"configuredRegistrationTimeout", registrationTimeout,
+					)
+
+					restart = true
+				} else {
+					log.V(1).Info(
+						"Runner pod exists but the GitHub runner appears to be still offline. Waiting for runner to get online ...",
+						"runnerName", runner.Name,
+					)
+				}
+			}
+
+			if (notFound || offline) && !registrationDidTimeout {
+				registrationRecheckJitter := 10 * time.Second
+				if r.RegistrationRecheckJitter > 0 {
+					registrationRecheckJitter = r.RegistrationRecheckJitter
+				}
+
+				registrationRecheckDelay = registrationCheckInterval + wait.Jitter(registrationRecheckJitter, 0.1)
+			}
 		}
 
 		// Don't do anything if there's no need to restart the runner
 		if !restart {
-			return ctrl.Result{}, err
+			// This guard enables us to update runner.Status.Phase to `Running` only after
+			// the runner is registered to GitHub.
+			if registrationRecheckDelay > 0 {
+				log.V(1).Info(fmt.Sprintf("Rechecking the runner registration in %s", registrationRecheckDelay))
+
+				updated := runner.DeepCopy()
+				updated.Status.LastRegistrationCheckTime = &metav1.Time{Time: time.Now()}
+
+				if err := r.Status().Patch(ctx, updated, client.MergeFrom(&runner)); err != nil {
+					log.Error(err, "Failed to update runner status for LastRegistrationCheckTime")
+					return ctrl.Result{}, err
+				}
+
+				return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil
+			}
+
+			if runner.Status.Phase != string(pod.Status.Phase) {
+				if pod.Status.Phase == corev1.PodRunning {
+					// Seeing this message, you can expect the runner to become `Running` soon.
+					log.Info(
+						"Runner appears to have registered and running.",
+						"podCreationTimestamp", pod.CreationTimestamp,
+					)
+				}
+
+				updated := runner.DeepCopy()
+				updated.Status.Phase = string(pod.Status.Phase)
+				updated.Status.Reason = pod.Status.Reason
+				updated.Status.Message = pod.Status.Message
+
+				if err := r.Status().Patch(ctx, updated, client.MergeFrom(&runner)); err != nil {
+					log.Error(err, "Failed to update runner status for Phase/Reason/Message")
+					return ctrl.Result{}, err
+				}
+			}
+
+			return ctrl.Result{}, nil
 		}
 
 		// Delete current pod if recreation is needed
@@ -227,23 +438,8 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	return ctrl.Result{}, nil
 }
 
-func (r *RunnerReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
-	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
-	if err != nil {
-		return false, err
-	}
-
-	for _, runner := range runners {
-		if runner.GetName() == name {
-			return runner.GetBusy(), nil
-		}
-	}
-
-	return false, fmt.Errorf("runner not found")
-}
-
-func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name string) (bool, error) {
-	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
+func (r *RunnerReconciler) unregisterRunner(ctx context.Context, enterprise, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, enterprise, org, repo)
 	if err != nil {
 		return false, err
 	}
@@ -263,7 +459,7 @@ func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name
 		return false, nil
 	}
 
-	if err := r.GitHubClient.RemoveRunner(ctx, org, repo, id); err != nil {
+	if err := r.GitHubClient.RemoveRunner(ctx, enterprise, org, repo, id); err != nil {
 		return false, err
 	}
 
@@ -277,7 +473,7 @@ func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v
 
 	log := r.Log.WithValues("runner", runner.Name)
 
-	rt, err := r.GitHubClient.GetRegistrationToken(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+	rt, err := r.GitHubClient.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 	if err != nil {
 		r.Recorder.Event(&runner, corev1.EventTypeWarning, "FailedUpdateRegistrationToken", "Updating registration token failed")
 		log.Error(err, "Failed to get new registration token")
@@ -293,8 +489,8 @@ func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v
 		ExpiresAt:    metav1.NewTime(rt.GetExpiresAt().Time),
 	}
 
-	if err := r.Status().Update(ctx, updated); err != nil {
-		log.Error(err, "Failed to update runner status")
+	if err := r.Status().Patch(ctx, updated, client.MergeFrom(&runner)); err != nil {
+		log.Error(err, "Failed to update runner status for Registration")
 		return false, err
 	}
 
@@ -339,6 +535,10 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 			Name:  "RUNNER_REPO",
 			Value: runner.Spec.Repository,
 		},
+		{
+			Name:  "RUNNER_ENTERPRISE",
+			Value: runner.Spec.Enterprise,
+		},
 		{
 			Name:  "RUNNER_LABELS",
 			Value: strings.Join(runner.Spec.Labels, ","),
@@ -425,6 +625,15 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		},
 	}
 
+	if mtu := runner.Spec.DockerMTU; mtu != nil && dockerdInRunner {
+		pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, []corev1.EnvVar{
+			{
+				Name:  "MTU",
+				Value: fmt.Sprintf("%d", *runner.Spec.DockerMTU),
+			},
+		}...)
+	}
+
 	if !dockerdInRunner && dockerEnabled {
 		runnerVolumeName := "runner"
 		runnerVolumeMountPath := "/runner"
@@ -504,8 +713,18 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 			SecurityContext: &corev1.SecurityContext{
 				Privileged: &privileged,
 			},
+			Resources: runner.Spec.DockerdContainerResources,
 		})
 
+		if mtu := runner.Spec.DockerMTU; mtu != nil {
+			pod.Spec.Containers[1].Env = append(pod.Spec.Containers[1].Env, []corev1.EnvVar{
+				{
+					Name:  "DOCKERD_ROOTLESS_ROOTLESSKIT_MTU",
+					Value: fmt.Sprintf("%d", *runner.Spec.DockerMTU),
+				},
+			}...)
+		}
+
 	}
 
 	if len(runner.Spec.Containers) != 0 {
@@ -574,11 +793,17 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 }
 
 func (r *RunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("runner-controller")
+	name := "runner-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.Runner{}).
 		Owns(&corev1.Pod{}).
+		Named(name).
 		Complete(r)
 }
 

controllers/runnerdeployment_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"reflect"
 	"sort"
 	"time"
 
@@ -37,10 +38,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/controllers/metrics"
 )
 
 const (
-	LabelKeyRunnerTemplateHash = "runner-template-hash"
+	LabelKeyRunnerTemplateHash   = "runner-template-hash"
+	LabelKeyRunnerDeploymentName = "runner-deployment-name"
 
 	runnerSetOwnerKey = ".metadata.controller"
 )
@@ -48,9 +51,11 @@ const (
 // RunnerDeploymentReconciler reconciles a Runner object
 type RunnerDeploymentReconciler struct {
 	client.Client
-	Log      logr.Logger
-	Recorder record.EventRecorder
-	Scheme   *runtime.Scheme
+	Log                logr.Logger
+	Recorder           record.EventRecorder
+	Scheme             *runtime.Scheme
+	CommonRunnerLabels []string
+	Name               string
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
@@ -73,6 +78,8 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		return ctrl.Result{}, nil
 	}
 
+	metrics.SetRunnerDeployment(rd)
+
 	var myRunnerReplicaSetList v1alpha1.RunnerReplicaSetList
 	if err := r.List(ctx, &myRunnerReplicaSetList, client.InNamespace(req.Namespace), client.MatchingFields{runnerSetOwnerKey: req.Name}); err != nil {
 		return ctrl.Result{}, err
@@ -141,6 +148,28 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
 	}
 
+	if !reflect.DeepEqual(newestSet.Spec.Selector, desiredRS.Spec.Selector) {
+		updateSet := newestSet.DeepCopy()
+		updateSet.Spec = *desiredRS.Spec.DeepCopy()
+
+		// A selector update change doesn't trigger replicaset replacement,
+		// but we still need to update the existing replicaset with it.
+		// Otherwise selector-based runner query will never work on replicasets created before the controller v0.17.0
+		// See https://github.com/summerwind/actions-runner-controller/pull/355#discussion_r585379259
+		if err := r.Client.Update(ctx, updateSet); err != nil {
+			log.Error(err, "Failed to update runnerreplicaset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		// At this point, we are already sure that there's no need to create a new replicaset
+		// as the runner template hash is not changed.
+		//
+		// But we still need to requeue for the (possibly rare) cases that there are still old replicasets that needs
+		// to be cleaned up.
+		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+	}
+
 	const defaultReplicas = 1
 
 	currentDesiredReplicas := getIntOrDefault(newestSet.Spec.Replicas, defaultReplicas)
@@ -163,21 +192,35 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	if len(oldSets) > 0 {
 		readyReplicas := newestSet.Status.ReadyReplicas
 
-		if readyReplicas < currentDesiredReplicas {
-			log.WithValues("runnerreplicaset", types.NamespacedName{
+		oldSetsCount := len(oldSets)
+
+		logWithDebugInfo := log.WithValues(
+			"newest_runnerreplicaset", types.NamespacedName{
 				Namespace: newestSet.Namespace,
 				Name:      newestSet.Name,
-			}).
-				Info("Waiting until the newest runner replica set to be 100% available")
+			},
+			"newest_runnerreplicaset_replicas_ready", readyReplicas,
+			"newest_runnerreplicaset_replicas_desired", currentDesiredReplicas,
+			"old_runnerreplicasets_count", oldSetsCount,
+		)
 
-			return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+		if readyReplicas < currentDesiredReplicas {
+			logWithDebugInfo.
+				Info("Waiting until the newest runnerreplicaset to be 100% available")
+
+			return ctrl.Result{}, nil
+		}
+
+		if oldSetsCount > 0 {
+			logWithDebugInfo.
+				Info("The newest runnerreplicaset is 100% available. Deleting old runnerreplicasets")
 		}
 
 		for i := range oldSets {
 			rs := oldSets[i]
 
 			if err := r.Client.Delete(ctx, &rs); err != nil {
-				log.Error(err, "Failed to delete runner resource")
+				log.Error(err, "Failed to delete runnerreplicaset resource")
 
 				return ctrl.Result{}, err
 			}
@@ -256,28 +299,94 @@ func CloneAndAddLabel(labels map[string]string, labelKey, labelValue string) map
 	return newLabels
 }
 
-func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeployment) (*v1alpha1.RunnerReplicaSet, error) {
-	newRSTemplate := *rd.Spec.Template.DeepCopy()
-	templateHash := ComputeHash(&newRSTemplate)
-	// Add template hash label to selector.
-	labels := CloneAndAddLabel(rd.Spec.Template.Labels, LabelKeyRunnerTemplateHash, templateHash)
+// Clones the given selector and returns a new selector with the given key and value added.
+// Returns the given selector, if labelKey is empty.
+//
+// Proudly copied from k8s.io/kubernetes/pkg/util/labels.CloneSelectorAndAddLabel
+func CloneSelectorAndAddLabel(selector *metav1.LabelSelector, labelKey, labelValue string) *metav1.LabelSelector {
+	if labelKey == "" {
+		// Don't need to add a label.
+		return selector
+	}
 
-	newRSTemplate.Labels = labels
+	// Clone.
+	newSelector := new(metav1.LabelSelector)
+
+	newSelector.MatchLabels = make(map[string]string)
+	if selector.MatchLabels != nil {
+		for key, val := range selector.MatchLabels {
+			newSelector.MatchLabels[key] = val
+		}
+	}
+	newSelector.MatchLabels[labelKey] = labelValue
+
+	if selector.MatchExpressions != nil {
+		newMExps := make([]metav1.LabelSelectorRequirement, len(selector.MatchExpressions))
+		for i, me := range selector.MatchExpressions {
+			newMExps[i].Key = me.Key
+			newMExps[i].Operator = me.Operator
+			if me.Values != nil {
+				newMExps[i].Values = make([]string, len(me.Values))
+				copy(newMExps[i].Values, me.Values)
+			} else {
+				newMExps[i].Values = nil
+			}
+		}
+		newSelector.MatchExpressions = newMExps
+	} else {
+		newSelector.MatchExpressions = nil
+	}
+
+	return newSelector
+}
+
+func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeployment) (*v1alpha1.RunnerReplicaSet, error) {
+	return newRunnerReplicaSet(&rd, r.CommonRunnerLabels, r.Scheme)
+}
+
+func getSelector(rd *v1alpha1.RunnerDeployment) *metav1.LabelSelector {
+	selector := rd.Spec.Selector
+	if selector == nil {
+		selector = &metav1.LabelSelector{MatchLabels: map[string]string{LabelKeyRunnerDeploymentName: rd.Name}}
+	}
+
+	return selector
+}
+
+func newRunnerReplicaSet(rd *v1alpha1.RunnerDeployment, commonRunnerLabels []string, scheme *runtime.Scheme) (*v1alpha1.RunnerReplicaSet, error) {
+	newRSTemplate := *rd.Spec.Template.DeepCopy()
+
+	for _, l := range commonRunnerLabels {
+		newRSTemplate.Spec.Labels = append(newRSTemplate.Spec.Labels, l)
+	}
+
+	templateHash := ComputeHash(&newRSTemplate)
+
+	// Add template hash label to selector.
+	newRSTemplate.ObjectMeta.Labels = CloneAndAddLabel(newRSTemplate.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash)
+
+	// This label selector is used by default when rd.Spec.Selector is empty.
+	newRSTemplate.ObjectMeta.Labels = CloneAndAddLabel(newRSTemplate.ObjectMeta.Labels, LabelKeyRunnerDeploymentName, rd.Name)
+
+	selector := getSelector(rd)
+
+	newRSSelector := CloneSelectorAndAddLabel(selector, LabelKeyRunnerTemplateHash, templateHash)
 
 	rs := v1alpha1.RunnerReplicaSet{
 		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: rd.ObjectMeta.Name + "-",
 			Namespace:    rd.ObjectMeta.Namespace,
-			Labels:       labels,
+			Labels:       newRSTemplate.ObjectMeta.Labels,
 		},
 		Spec: v1alpha1.RunnerReplicaSetSpec{
 			Replicas: rd.Spec.Replicas,
+			Selector: newRSSelector,
 			Template: newRSTemplate,
 		},
 	}
 
-	if err := ctrl.SetControllerReference(&rd, &rs, r.Scheme); err != nil {
+	if err := ctrl.SetControllerReference(rd, &rs, scheme); err != nil {
 		return &rs, err
 	}
 
@@ -285,7 +394,12 @@ func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeplo
 }
 
 func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("runnerdeployment-controller")
+	name := "runnerdeployment-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.RunnerReplicaSet{}, runnerSetOwnerKey, func(rawObj runtime.Object) []string {
 		runnerSet := rawObj.(*v1alpha1.RunnerReplicaSet)
@@ -306,5 +420,6 @@ func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.RunnerDeployment{}).
 		Owns(&v1alpha1.RunnerReplicaSet{}).
+		Named(name).
 		Complete(r)
 }

controllers/runnerdeployment_controller_test.go

@@ -2,8 +2,13 @@ package controllers
 
 import (
 	"context"
+	"fmt"
+	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+	"k8s.io/apimachinery/pkg/runtime"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -18,6 +23,103 @@ import (
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
 )
 
+func TestNewRunnerReplicaSet(t *testing.T) {
+	scheme := runtime.NewScheme()
+	if err := actionsv1alpha1.AddToScheme(scheme); err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	r := &RunnerDeploymentReconciler{
+		CommonRunnerLabels: []string{"dev"},
+		Scheme:             scheme,
+	}
+	rd := actionsv1alpha1.RunnerDeployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "example",
+		},
+		Spec: actionsv1alpha1.RunnerDeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"foo": "bar",
+				},
+			},
+			Template: actionsv1alpha1.RunnerTemplate{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"foo": "bar",
+					},
+				},
+				Spec: actionsv1alpha1.RunnerSpec{
+					Labels: []string{"project1"},
+				},
+			},
+		},
+	}
+
+	rs, err := r.newRunnerReplicaSet(rd)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	if val, ok := rs.Labels["foo"]; ok {
+		if val != "bar" {
+			t.Errorf("foo label does not have bar but %v", val)
+		}
+	} else {
+		t.Errorf("foo label does not exist")
+	}
+
+	hash1, ok := rs.Labels[LabelKeyRunnerTemplateHash]
+	if !ok {
+		t.Errorf("missing runner-template-hash label")
+	}
+
+	runnerLabel := []string{"project1", "dev"}
+	if d := cmp.Diff(runnerLabel, rs.Spec.Template.Spec.Labels); d != "" {
+		t.Errorf("%s", d)
+	}
+
+	rd2 := rd.DeepCopy()
+	rd2.Spec.Template.Spec.Labels = []string{"project2"}
+
+	rs2, err := r.newRunnerReplicaSet(*rd2)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	hash2, ok := rs2.Labels[LabelKeyRunnerTemplateHash]
+	if !ok {
+		t.Errorf("missing runner-template-hash label")
+	}
+
+	if hash1 == hash2 {
+		t.Errorf(
+			"runner replica sets from runner deployments with varying labels must have different template hash, but got %s and %s",
+			hash1, hash2,
+		)
+	}
+
+	rd3 := rd.DeepCopy()
+	rd3.Spec.Template.Labels["foo"] = "baz"
+
+	rs3, err := r.newRunnerReplicaSet(*rd3)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	hash3, ok := rs3.Labels[LabelKeyRunnerTemplateHash]
+	if !ok {
+		t.Errorf("missing runner-template-hash label")
+	}
+
+	if hash1 == hash3 {
+		t.Errorf(
+			"runner replica sets from runner deployments with varying meta labels must have different template hash, but got %s and %s",
+			hash1, hash3,
+		)
+	}
+}
+
 // SetupDeploymentTest will set up a testing environment.
 // This includes:
 // * creating a Namespace to be used during the test
@@ -37,7 +139,9 @@ func SetupDeploymentTest(ctx context.Context) *corev1.Namespace {
 		err := k8sClient.Create(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
 
-		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+			Namespace: ns.Name,
+		})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
 		controller := &RunnerDeploymentReconciler{
@@ -45,6 +149,7 @@ func SetupDeploymentTest(ctx context.Context) *corev1.Namespace {
 			Scheme:   scheme.Scheme,
 			Log:      logf.Log,
 			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     "runnerdeployment-" + ns.Name,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -74,7 +179,7 @@ var _ = Context("Inside of a new namespace", func() {
 	Describe("when no existing resources exist", func() {
 
 		It("should create a new RunnerReplicaSet resource from the specified template, add a another RunnerReplicaSet on template modification, and eventually removes old runnerreplicasets", func() {
-			name := "example-runnerdeploy"
+			name := "example-runnerdeploy-1"
 
 			{
 				rs := &actionsv1alpha1.RunnerDeployment{
@@ -84,9 +189,19 @@ var _ = Context("Inside of a new namespace", func() {
 					},
 					Spec: actionsv1alpha1.RunnerDeploymentSpec{
 						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
 						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
 							Spec: actionsv1alpha1.RunnerSpec{
-								Repository: "foo/bar",
+								Repository: "test/valid",
 								Image:      "bar",
 								Env: []corev1.EnvVar{
 									{Name: "FOO", Value: "FOOVALUE"},
@@ -103,29 +218,25 @@ var _ = Context("Inside of a new namespace", func() {
 				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
 
 				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
 						if err != nil {
-							logf.Log.Error(err, "list runner sets")
+							return 0, err
 						}
-
-						return len(runnerSets.Items)
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
 						if err != nil {
-							logf.Log.Error(err, "list runner sets")
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
 						}
 
-						if len(runnerSets.Items) == 0 {
-							logf.Log.Info("No runnerreplicasets exist yet")
-							return -1
-						}
-
-						return *runnerSets.Items[0].Spec.Replicas
+						return *runnerSets.Items[0].Spec.Replicas, nil
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
 			}
@@ -134,13 +245,12 @@ var _ = Context("Inside of a new namespace", func() {
 				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
 				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
 				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+				var rd actionsv1alpha1.RunnerDeployment
 				Eventually(func() error {
-					var rd actionsv1alpha1.RunnerDeployment
-
 					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
-
-					Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerReplicaSet resource")
-
+					if err != nil {
+						return fmt.Errorf("failed to get test RunnerReplicaSet resource: %v\n", err)
+					}
 					rd.Spec.Replicas = intPtr(2)
 
 					return k8sClient.Update(ctx, &rd)
@@ -150,27 +260,222 @@ var _ = Context("Inside of a new namespace", func() {
 				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
 
 				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rd.Spec.Selector)
 						if err != nil {
-							logf.Log.Error(err, "list runner sets")
+							return 0, err
+						}
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
+						if err != nil {
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
 						}
 
-						return len(runnerSets.Items)
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						return *runnerSets.Items[0].Spec.Replicas
+						return *runnerSets.Items[0].Spec.Replicas, nil
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
 			}
 		})
+
+		It("should create a new RunnerReplicaSet resource from the specified template without labels and selector, add a another RunnerReplicaSet on template modification, and eventually removes old runnerreplicasets", func() {
+			name := "example-runnerdeploy-2"
+
+			{
+				rs := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								Repository: "test/valid",
+								Image:      "bar",
+								Env: []corev1.EnvVar{
+									{Name: "FOO", Value: "FOOVALUE"},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
+						if err != nil {
+							return 0, err
+						}
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
+						if err != nil {
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas, nil
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+			}
+
+			{
+				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+				var rd actionsv1alpha1.RunnerDeployment
+				Eventually(func() error {
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
+					if err != nil {
+						return fmt.Errorf("failed to get test RunnerReplicaSet resource: %v\n", err)
+					}
+					rd.Spec.Replicas = intPtr(2)
+
+					return k8sClient.Update(ctx, &rd)
+				},
+					time.Second*1, time.Millisecond*500).Should(BeNil())
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rd.Spec.Selector)
+						if err != nil {
+							return 0, err
+						}
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
+						if err != nil {
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas, nil
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
+			}
+		})
+
+		It("should adopt RunnerReplicaSet created before 0.18.0 to have Spec.Selector", func() {
+			name := "example-runnerdeploy-2"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								Repository: "test/valid",
+								Image:      "bar",
+								Env: []corev1.EnvVar{
+									{Name: "FOO", Value: "FOOVALUE"},
+								},
+							},
+						},
+					},
+				}
+
+				createRDErr := k8sClient.Create(ctx, rd)
+				Expect(createRDErr).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				Eventually(
+					func() (int, error) {
+						runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+						err := k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+						)
+						if err != nil {
+							return 0, err
+						}
+
+						return len(runnerSets.Items), nil
+					},
+					time.Second*1, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				var rs17 *actionsv1alpha1.RunnerReplicaSet
+
+				Consistently(
+					func() (*metav1.LabelSelector, error) {
+						runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+						err := k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+						)
+						if err != nil {
+							return nil, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return nil, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						rs17 = &runnerSets.Items[0]
+
+						return runnerSets.Items[0].Spec.Selector, nil
+					},
+					time.Second*1, time.Millisecond*500).Should(Not(BeNil()))
+
+				// We simulate the old, pre 0.18.0 RunnerReplicaSet by updating it.
+				// I've tried to use controllerutil.Set{Owner,Controller}Reference and k8sClient.Create(rs17)
+				// but it didn't work due to missing RD UID, where UID is generated on K8s API server on k8sCLient.Create(rd)
+				rs17.Spec.Selector = nil
+
+				updateRSErr := k8sClient.Update(ctx, rs17)
+				Expect(updateRSErr).NotTo(HaveOccurred())
+
+				Eventually(
+					func() (*metav1.LabelSelector, error) {
+						runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+						err := k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+						)
+						if err != nil {
+							return nil, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return nil, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return runnerSets.Items[0].Spec.Selector, nil
+					},
+					time.Second*1, time.Millisecond*500).Should(Not(BeNil()))
+			}
+		})
+
 	})
 })

controllers/runnerreplicaset_controller.go

@@ -18,10 +18,14 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"time"
+
+	gogithub "github.com/google/go-github/v33/github"
 
 	"github.com/go-logr/logr"
-	"k8s.io/apimachinery/pkg/api/errors"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -41,6 +45,7 @@ type RunnerReplicaSetReconciler struct {
 	Recorder     record.EventRecorder
 	Scheme       *runtime.Scheme
 	GitHubClient *github.Client
+	Name         string
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
@@ -63,18 +68,33 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		return ctrl.Result{}, nil
 	}
 
+	selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	// Get the Runners managed by the target RunnerReplicaSet
 	var allRunners v1alpha1.RunnerList
-	if err := r.List(ctx, &allRunners, client.InNamespace(req.Namespace)); err != nil {
-		if !errors.IsNotFound(err) {
+	if err := r.List(
+		ctx,
+		&allRunners,
+		client.InNamespace(req.Namespace),
+		client.MatchingLabelsSelector{Selector: selector},
+	); err != nil {
+		if !kerrors.IsNotFound(err) {
 			return ctrl.Result{}, err
 		}
 	}
 
 	var myRunners []v1alpha1.Runner
 
-	var available, ready int
+	var (
+		available int
+		ready     int
+	)
 
 	for _, r := range allRunners.Items {
+		// This guard is required to avoid the RunnerReplicaSet created by the controller v0.17.0 or before
+		// to not treat all the runners in the namespace as its children.
 		if metav1.IsControlledBy(&r, &rs) {
 			myRunners = append(myRunners, r)
 
@@ -94,41 +114,92 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		desired = 1
 	}
 
-	log.V(0).Info("debug", "desired", desired, "available", available)
-
 	if available > desired {
 		n := available - desired
 
-		// get runners that are currently not busy
-		var notBusy []v1alpha1.Runner
-		for _, runner := range myRunners {
-			busy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+		log.V(0).Info(fmt.Sprintf("Deleting %d runners", n), "desired", desired, "available", available, "ready", ready)
+
+		// get runners that are currently offline/not busy/timed-out to register
+		var deletionCandidates []v1alpha1.Runner
+
+		for _, runner := range allRunners.Items {
+			busy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 			if err != nil {
-				log.Error(err, "Failed to check if runner is busy")
-				return ctrl.Result{}, err
-			}
-			if !busy {
-				notBusy = append(notBusy, runner)
+				notRegistered := false
+				offline := false
+
+				var notFoundException *github.RunnerNotFound
+				var offlineException *github.RunnerOffline
+				if errors.As(err, &notFoundException) {
+					log.V(1).Info("Failed to check if runner is busy. Either this runner has never been successfully registered to GitHub or it still needs more time.", "runnerName", runner.Name)
+					notRegistered = true
+				} else if errors.As(err, &offlineException) {
+					offline = true
+				} else {
+					var e *gogithub.RateLimitError
+					if errors.As(err, &e) {
+						// We log the underlying error when we failed calling GitHub API to list or unregisters,
+						// or the runner is still busy.
+						log.Error(
+							err,
+							fmt.Sprintf(
+								"Failed to check if runner is busy due to GitHub API rate limit. Retrying in %s to avoid excessive GitHub API calls",
+								retryDelayOnGitHubAPIRateLimitError,
+							),
+						)
+
+						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+					}
+
+					return ctrl.Result{}, err
+				}
+
+				registrationTimeout := 15 * time.Minute
+				currentTime := time.Now()
+				registrationDidTimeout := currentTime.Sub(runner.CreationTimestamp.Add(registrationTimeout)) > 0
+
+				if notRegistered && registrationDidTimeout {
+					log.Info(
+						"Runner failed to register itself to GitHub in timely manner. "+
+							"Marking the runner for scale down. "+
+							"CAUTION: If you see this a lot, you should investigate the root cause. "+
+							"See https://github.com/summerwind/actions-runner-controller/issues/288",
+						"runnerCreationTimestamp", runner.CreationTimestamp,
+						"currentTime", currentTime,
+						"configuredRegistrationTimeout", registrationTimeout,
+					)
+
+					deletionCandidates = append(deletionCandidates, runner)
+				}
+
+				// offline runners should always be a great target for scale down
+				if offline {
+					deletionCandidates = append(deletionCandidates, runner)
+				}
+			} else if !busy {
+				deletionCandidates = append(deletionCandidates, runner)
 			}
 		}
 
-		if len(notBusy) < n {
-			n = len(notBusy)
+		if len(deletionCandidates) < n {
+			n = len(deletionCandidates)
 		}
 
 		for i := 0; i < n; i++ {
-			if err := r.Client.Delete(ctx, &notBusy[i]); err != nil {
+			if err := r.Client.Delete(ctx, &deletionCandidates[i]); client.IgnoreNotFound(err) != nil {
 				log.Error(err, "Failed to delete runner resource")
 
 				return ctrl.Result{}, err
 			}
 
-			r.Recorder.Event(&rs, corev1.EventTypeNormal, "RunnerDeleted", fmt.Sprintf("Deleted runner '%s'", myRunners[i].Name))
-			log.Info("Deleted runner", "runnerreplicaset", rs.ObjectMeta.Name)
+			r.Recorder.Event(&rs, corev1.EventTypeNormal, "RunnerDeleted", fmt.Sprintf("Deleted runner '%s'", deletionCandidates[i].Name))
+			log.Info("Deleted runner")
 		}
 	} else if desired > available {
 		n := desired - available
 
+		log.V(0).Info(fmt.Sprintf("Creating %d runner(s)", n), "desired", desired, "available", available, "ready", ready)
+
 		for i := 0; i < n; i++ {
 			newRunner, err := r.newRunner(rs)
 			if err != nil {
@@ -151,8 +222,10 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		updated.Status.ReadyReplicas = ready
 
 		if err := r.Status().Update(ctx, updated); err != nil {
-			log.Error(err, "Failed to update runner status")
-			return ctrl.Result{}, err
+			log.Info("Failed to update status. Retrying immediately", "error", err.Error())
+			return ctrl.Result{
+				Requeue: true,
+			}, nil
 		}
 	}
 
@@ -179,26 +252,16 @@ func (r *RunnerReplicaSetReconciler) newRunner(rs v1alpha1.RunnerReplicaSet) (v1
 }
 
 func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("runnerreplicaset-controller")
+	name := "runnerreplicaset-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.RunnerReplicaSet{}).
 		Owns(&v1alpha1.Runner{}).
+		Named(name).
 		Complete(r)
 }
-
-func (r *RunnerReplicaSetReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
-	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
-	r.Log.Info("runners", "github", runners)
-	if err != nil {
-		return false, err
-	}
-
-	for _, runner := range runners {
-		if runner.GetName() == name {
-			return runner.GetBusy(), nil
-		}
-	}
-
-	return false, fmt.Errorf("runner not found")
-}

controllers/runnerreplicaset_controller_test.go

@@ -47,7 +47,9 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 		err := k8sClient.Create(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
 
-		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+			Namespace: ns.Name,
+		})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
 		runnersList = fake.NewRunnersList()
@@ -60,6 +62,7 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 			Log:          logf.Log,
 			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
 			GitHubClient: ghClient,
+			Name:         "runnerreplicaset-" + ns.Name,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -114,9 +117,19 @@ var _ = Context("Inside of a new namespace", func() {
 					},
 					Spec: actionsv1alpha1.RunnerReplicaSetSpec{
 						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
 						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
 							Spec: actionsv1alpha1.RunnerSpec{
-								Repository: "foo/bar",
+								Repository: "test/valid",
 								Image:      "bar",
 								Env: []corev1.EnvVar{
 									{Name: "FOO", Value: "FOOVALUE"},
@@ -134,9 +147,26 @@ var _ = Context("Inside of a new namespace", func() {
 
 				Eventually(
 					func() int {
-						err := k8sClient.List(ctx, &runners, client.InNamespace(ns.Name))
+						selector, err := metav1.LabelSelectorAsSelector(
+							&metav1.LabelSelector{
+								MatchLabels: map[string]string{
+									"foo": "bar",
+								},
+							},
+						)
+						if err != nil {
+							logf.Log.Error(err, "failed to create labelselector")
+							return -1
+						}
+						err = k8sClient.List(
+							ctx,
+							&runners,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
 						if err != nil {
 							logf.Log.Error(err, "list runners")
+							return -1
 						}
 
 						for i, runner := range runners.Items {
@@ -175,7 +205,23 @@ var _ = Context("Inside of a new namespace", func() {
 
 				Eventually(
 					func() int {
-						err := k8sClient.List(ctx, &runners, client.InNamespace(ns.Name))
+						selector, err := metav1.LabelSelectorAsSelector(
+							&metav1.LabelSelector{
+								MatchLabels: map[string]string{
+									"foo": "bar",
+								},
+							},
+						)
+						if err != nil {
+							logf.Log.Error(err, "failed to create labelselector")
+							return -1
+						}
+						err = k8sClient.List(
+							ctx,
+							&runners,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
 						if err != nil {
 							logf.Log.Error(err, "list runners")
 						}
@@ -219,6 +265,7 @@ var _ = Context("Inside of a new namespace", func() {
 						err := k8sClient.List(ctx, &runners, client.InNamespace(ns.Name))
 						if err != nil {
 							logf.Log.Error(err, "list runners")
+							return -1
 						}
 
 						for i, runner := range runners.Items {

controllers/suite_test.go

@@ -17,6 +17,8 @@ limitations under the License.
 package controllers
 
 import (
+	"github.com/onsi/ginkgo/config"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -43,6 +45,8 @@ var testEnv *envtest.Environment
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
 
+	config.GinkgoConfig.FocusString = os.Getenv("GINKGO_FOCUS")
+
 	RunSpecsWithDefaultAndCustomReporters(t,
 		"Controller Suite",
 		[]Reporter{envtest.NewlineReporter{}})
@@ -51,9 +55,17 @@ func TestAPIs(t *testing.T) {
 var _ = BeforeSuite(func(done Done) {
 	logf.SetLogger(zap.LoggerTo(GinkgoWriter, true))
 
+	var apiServerFlags []string
+
+	apiServerFlags = append(apiServerFlags, envtest.DefaultKubeAPIServerFlags...)
+	// Avoids the following error:
+	// 2021-03-19T15:14:11.673+0900    ERROR   controller-runtime.controller   Reconciler error      {"controller": "testns-tvjzjrunner", "request": "testns-gdnyx/example-runnerdeploy-zps4z-j5562", "error": "Pod \"example-runnerdeploy-zps4z-j5562\" is invalid: [spec.containers[1].image: Required value, spec.containers[1].securityContext.privileged: Forbidden: disallowed by cluster policy]"}
+	apiServerFlags = append(apiServerFlags, "--allow-privileged=true")
+
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
+		CRDDirectoryPaths:  []string{filepath.Join("..", "config", "crd", "bases")},
+		KubeAPIServerFlags: apiServerFlags,
 	}
 
 	var err error

controllers/testdata/org_webhook_check_run_payload.json

@@ -0,0 +1,373 @@
+{
+  "action": "created",
+  "check_run": {
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "head_sha": "1234567890123456789012345678901234567890",
+    "external_id": "92058b04-f16a-5035-546c-cae3ad5e2f5f",
+    "url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890",
+    "html_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "details_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "status": "queued",
+    "conclusion": null,
+    "started_at": "2021-02-18T06:16:31Z",
+    "completed_at": null,
+    "output": {
+      "title": null,
+      "summary": null,
+      "text": null,
+      "annotations_count": 0,
+      "annotations_url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890/annotations"
+    },
+    "name": "validate",
+    "check_suite": {
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "head_branch": "MYNAME/actions-runner-controller-webhook",
+      "head_sha": "1234567890123456789012345678901234567890",
+      "status": "queued",
+      "conclusion": null,
+      "url": "https://api.github.com/repos/MYORG/MYREPO/check-suites/1234567890",
+      "before": "1234567890123456789012345678901234567890",
+      "after": "1234567890123456789012345678901234567890",
+      "pull_requests": [
+        {
+          "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/2033",
+          "id": 1234567890,
+          "number": 1234567890,
+          "head": {
+            "ref": "feature",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          },
+          "base": {
+            "ref": "master",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          }
+        }
+      ],
+      "app": {
+        "id": 1234567890,
+        "slug": "github-actions",
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "owner": {
+          "login": "github",
+          "id": 1234567890,
+          "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+          "avatar_url": "https://avatars.githubusercontent.com/u/123467890?v=4",
+          "gravatar_id": "",
+          "url": "https://api.github.com/users/github",
+          "html_url": "https://github.com/github",
+          "followers_url": "https://api.github.com/users/github/followers",
+          "following_url": "https://api.github.com/users/github/following{/other_user}",
+          "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+          "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+          "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+          "organizations_url": "https://api.github.com/users/github/orgs",
+          "repos_url": "https://api.github.com/users/github/repos",
+          "events_url": "https://api.github.com/users/github/events{/privacy}",
+          "received_events_url": "https://api.github.com/users/github/received_events",
+          "type": "Organization",
+          "site_admin": false
+        },
+        "name": "GitHub Actions",
+        "description": "Automate your workflow from idea to production",
+        "external_url": "https://help.github.com/en/actions",
+        "html_url": "https://github.com/apps/github-actions",
+        "created_at": "2018-07-30T09:30:17Z",
+        "updated_at": "2019-12-10T19:04:12Z",
+        "permissions": {
+          "actions": "write",
+          "checks": "write",
+          "contents": "write",
+          "deployments": "write",
+          "issues": "write",
+          "metadata": "read",
+          "organization_packages": "write",
+          "packages": "write",
+          "pages": "write",
+          "pull_requests": "write",
+          "repository_hooks": "write",
+          "repository_projects": "write",
+          "security_events": "write",
+          "statuses": "write",
+          "vulnerability_alerts": "read"
+        },
+        "events": [
+          "check_run",
+          "check_suite",
+          "create",
+          "delete",
+          "deployment",
+          "deployment_status",
+          "fork",
+          "gollum",
+          "issues",
+          "issue_comment",
+          "label",
+          "milestone",
+          "page_build",
+          "project",
+          "project_card",
+          "project_column",
+          "public",
+          "pull_request",
+          "pull_request_review",
+          "pull_request_review_comment",
+          "push",
+          "registry_package",
+          "release",
+          "repository",
+          "repository_dispatch",
+          "status",
+          "watch",
+          "workflow_dispatch",
+          "workflow_run"
+        ]
+      },
+      "created_at": "2021-02-18T06:15:32Z",
+      "updated_at": "2021-02-18T06:16:31Z"
+    },
+    "app": {
+    "id": 1234567890,
+    "slug": "github-actions",
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "owner": {
+      "login": "github",
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/github",
+      "html_url": "https://github.com/github",
+      "followers_url": "https://api.github.com/users/github/followers",
+      "following_url": "https://api.github.com/users/github/following{/other_user}",
+      "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+      "organizations_url": "https://api.github.com/users/github/orgs",
+      "repos_url": "https://api.github.com/users/github/repos",
+      "events_url": "https://api.github.com/users/github/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/github/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "name": "GitHub Actions",
+    "description": "Automate your workflow from idea to production",
+    "external_url": "https://help.github.com/en/actions",
+    "html_url": "https://github.com/apps/github-actions",
+    "created_at": "2018-07-30T09:30:17Z",
+    "updated_at": "2019-12-10T19:04:12Z",
+    "permissions": {
+      "actions": "write",
+      "checks": "write",
+      "contents": "write",
+      "deployments": "write",
+      "issues": "write",
+      "metadata": "read",
+      "organization_packages": "write",
+      "packages": "write",
+      "pages": "write",
+      "pull_requests": "write",
+      "repository_hooks": "write",
+      "repository_projects": "write",
+      "security_events": "write",
+      "statuses": "write",
+      "vulnerability_alerts": "read"
+    },
+    "events": [
+      "check_run",
+      "check_suite",
+      "create",
+      "delete",
+      "deployment",
+      "deployment_status",
+      "fork",
+      "gollum",
+      "issues",
+      "issue_comment",
+      "label",
+      "milestone",
+      "page_build",
+      "project",
+      "project_card",
+      "project_column",
+      "public",
+      "pull_request",
+      "pull_request_review",
+      "pull_request_review_comment",
+      "push",
+      "registry_package",
+      "release",
+      "repository",
+      "repository_dispatch",
+      "status",
+      "watch",
+      "workflow_dispatch",
+      "workflow_run"
+    ]
+  },
+  "pull_requests": [
+    {
+      "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/1234567890",
+      "id": 1234567890,
+      "number": 1234567890,
+      "head": {
+        "ref": "feature",
+        "sha": "1234567890123456789012345678901234567890",
+        "repo": {
+          "id": 1234567890,
+          "url": "https://api.github.com/repos/MYORG/MYREPO",
+          "name": "MYREPO"
+        }
+      },
+      "base": {
+        "ref": "master",
+        "sha": "1234567890123456789012345678901234567890",
+        "repo": {
+          "id": 1234567890,
+          "url": "https://api.github.com/repos/MYORG/MYREPO",
+          "name": "MYREPO"
+        }
+      }
+    }
+  ]
+},
+"repository": {
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "name": "MYREPO",
+    "full_name": "MYORG/MYREPO",
+    "private": true,
+    "owner": {
+      "login": "MYORG",
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/MYORG",
+      "html_url": "https://github.com/MYORG",
+      "followers_url": "https://api.github.com/users/MYORG/followers",
+      "following_url": "https://api.github.com/users/MYORG/following{/other_user}",
+      "gists_url": "https://api.github.com/users/MYORG/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/MYORG/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/MYORG/subscriptions",
+      "organizations_url": "https://api.github.com/users/MYORG/orgs",
+      "repos_url": "https://api.github.com/users/MYORG/repos",
+      "events_url": "https://api.github.com/users/MYORG/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/MYORG/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "html_url": "https://github.com/MYORG/MYREPO",
+    "description": "MYREPO",
+    "fork": false,
+    "url": "https://api.github.com/repos/MYORG/MYREPO",
+    "forks_url": "https://api.github.com/repos/MYORG/MYREPO/forks",
+    "keys_url": "https://api.github.com/repos/MYORG/MYREPO/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/MYORG/MYREPO/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/MYORG/MYREPO/teams",
+    "hooks_url": "https://api.github.com/repos/MYORG/MYREPO/hooks",
+    "issue_events_url": "https://api.github.com/repos/MYORG/MYREPO/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/MYORG/MYREPO/events",
+    "assignees_url": "https://api.github.com/repos/MYORG/MYREPO/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/MYORG/MYREPO/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/MYORG/MYREPO/tags",
+    "blobs_url": "https://api.github.com/repos/MYORG/MYREPO/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/MYORG/MYREPO/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/MYORG/MYREPO/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/MYORG/MYREPO/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/MYORG/MYREPO/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/MYORG/MYREPO/languages",
+    "stargazers_url": "https://api.github.com/repos/MYORG/MYREPO/stargazers",
+    "contributors_url": "https://api.github.com/repos/MYORG/MYREPO/contributors",
+    "subscribers_url": "https://api.github.com/repos/MYORG/MYREPO/subscribers",
+    "subscription_url": "https://api.github.com/repos/MYORG/MYREPO/subscription",
+    "commits_url": "https://api.github.com/repos/MYORG/MYREPO/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/MYORG/MYREPO/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/MYORG/MYREPO/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/MYORG/MYREPO/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/MYORG/MYREPO/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/MYORG/MYREPO/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/MYORG/MYREPO/merges",
+    "archive_url": "https://api.github.com/repos/MYORG/MYREPO/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/MYORG/MYREPO/downloads",
+    "issues_url": "https://api.github.com/repos/MYORG/MYREPO/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/MYORG/MYREPO/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/MYORG/MYREPO/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/MYORG/MYREPO/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/MYORG/MYREPO/labels{/name}",
+    "releases_url": "https://api.github.com/repos/MYORG/MYREPO/releases{/id}",
+    "deployments_url": "https://api.github.com/repos/MYORG/MYREPO/deployments",
+    "created_at": "2017-08-10T02:21:10Z",
+    "updated_at": "2021-02-18T04:40:55Z",
+    "pushed_at": "2021-02-18T06:15:30Z",
+    "git_url": "git://github.com/MYORG/MYREPO.git",
+    "ssh_url": "git@github.com:MYORG/MYREPO.git",
+    "clone_url": "https://github.com/MYORG/MYREPO.git",
+    "svn_url": "https://github.com/MYORG/MYREPO",
+    "homepage": null,
+    "size": 30782,
+    "stargazers_count": 2,
+    "watchers_count": 2,
+    "language": "Shell",
+    "has_issues": false,
+    "has_projects": true,
+    "has_downloads": true,
+    "has_wiki": false,
+    "has_pages": false,
+    "forks_count": 0,
+    "mirror_url": null,
+    "archived": false,
+    "disabled": false,
+    "open_issues_count": 6,
+    "license": null,
+    "forks": 0,
+    "open_issues": 6,
+    "watchers": 2,
+    "default_branch": "master"
+  },
+  "organization": {
+    "login": "MYORG",
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "url": "https://api.github.com/orgs/MYORG",
+    "repos_url": "https://api.github.com/orgs/MYORG/repos",
+    "events_url": "https://api.github.com/orgs/MYORG/events",
+    "hooks_url": "https://api.github.com/orgs/MYORG/hooks",
+    "issues_url": "https://api.github.com/orgs/MYORG/issues",
+    "members_url": "https://api.github.com/orgs/MYORG/members{/member}",
+    "public_members_url": "https://api.github.com/orgs/MYORG/public_members{/member}",
+    "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+    "description": ""
+  },
+  "sender": {
+    "login": "MYNAME",
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/MYNAME",
+    "html_url": "https://github.com/MYNAME",
+    "followers_url": "https://api.github.com/users/MYNAME/followers",
+    "following_url": "https://api.github.com/users/MYNAME/following{/other_user}",
+    "gists_url": "https://api.github.com/users/MYNAME/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/MYNAME/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/MYNAME/subscriptions",
+    "organizations_url": "https://api.github.com/users/MYNAME/orgs",
+    "repos_url": "https://api.github.com/users/MYNAME/repos",
+    "events_url": "https://api.github.com/users/MYNAME/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/MYNAME/received_events",
+    "type": "User",
+    "site_admin": false
+  }
+}

controllers/testdata/repo_webhook_check_run_payload.json

@@ -0,0 +1,360 @@
+{
+  "action": "completed",
+  "check_run": {
+    "id": 1949438388,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "head_sha": "1234567890123456789012345678901234567890",
+    "external_id": "ca395085-040a-526b-2ce8-bdc85f692774",
+    "url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890",
+    "html_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "details_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "status": "queued",
+    "conclusion": null,
+    "started_at": "2021-02-18T06:16:31Z",
+    "completed_at": null,
+    "output": {
+      "title": null,
+      "summary": null,
+      "text": null,
+      "annotations_count": 0,
+      "annotations_url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890/annotations"
+    },
+    "name": "build",
+    "name": "validate",
+    "check_suite": {
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "head_branch": "MYNAME/actions-runner-controller-webhook",
+      "head_sha": "1234567890123456789012345678901234567890",
+      "status": "queued",
+      "conclusion": null,
+      "url": "https://api.github.com/repos/MYORG/MYREPO/check-suites/1234567890",
+      "before": "1234567890123456789012345678901234567890",
+      "after": "1234567890123456789012345678901234567890",
+      "pull_requests": [
+        {
+          "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/2033",
+          "id": 1234567890,
+          "number": 1234567890,
+          "head": {
+            "ref": "feature",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          },
+          "base": {
+            "ref": "master",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          }
+        }
+      ],
+      "app": {
+        "id": 1234567890,
+        "slug": "github-actions",
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "owner": {
+          "login": "github",
+          "id": 1234567890,
+          "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+          "avatar_url": "https://avatars.githubusercontent.com/u/123467890?v=4",
+          "gravatar_id": "",
+          "url": "https://api.github.com/users/github",
+          "html_url": "https://github.com/github",
+          "followers_url": "https://api.github.com/users/github/followers",
+          "following_url": "https://api.github.com/users/github/following{/other_user}",
+          "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+          "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+          "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+          "organizations_url": "https://api.github.com/users/github/orgs",
+          "repos_url": "https://api.github.com/users/github/repos",
+          "events_url": "https://api.github.com/users/github/events{/privacy}",
+          "received_events_url": "https://api.github.com/users/github/received_events",
+          "type": "Organization",
+          "site_admin": false
+        },
+        "name": "GitHub Actions",
+        "description": "Automate your workflow from idea to production",
+        "external_url": "https://help.github.com/en/actions",
+        "html_url": "https://github.com/apps/github-actions",
+        "created_at": "2018-07-30T09:30:17Z",
+        "updated_at": "2019-12-10T19:04:12Z",
+        "permissions": {
+          "actions": "write",
+          "checks": "write",
+          "contents": "write",
+          "deployments": "write",
+          "issues": "write",
+          "metadata": "read",
+          "organization_packages": "write",
+          "packages": "write",
+          "pages": "write",
+          "pull_requests": "write",
+          "repository_hooks": "write",
+          "repository_projects": "write",
+          "security_events": "write",
+          "statuses": "write",
+          "vulnerability_alerts": "read"
+        },
+        "events": [
+          "check_run",
+          "check_suite",
+          "create",
+          "delete",
+          "deployment",
+          "deployment_status",
+          "fork",
+          "gollum",
+          "issues",
+          "issue_comment",
+          "label",
+          "milestone",
+          "page_build",
+          "project",
+          "project_card",
+          "project_column",
+          "public",
+          "pull_request",
+          "pull_request_review",
+          "pull_request_review_comment",
+          "push",
+          "registry_package",
+          "release",
+          "repository",
+          "repository_dispatch",
+          "status",
+          "watch",
+          "workflow_dispatch",
+          "workflow_run"
+        ]
+      },
+      "created_at": "2021-02-18T06:15:32Z",
+      "updated_at": "2021-02-18T06:16:31Z"
+    },
+    "app": {
+      "id": 1234567890,
+      "slug": "github-actions",
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "owner": {
+        "login": "github",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/github",
+        "html_url": "https://github.com/github",
+        "followers_url": "https://api.github.com/users/github/followers",
+        "following_url": "https://api.github.com/users/github/following{/other_user}",
+        "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+        "organizations_url": "https://api.github.com/users/github/orgs",
+        "repos_url": "https://api.github.com/users/github/repos",
+        "events_url": "https://api.github.com/users/github/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/github/received_events",
+        "type": "Organization",
+        "site_admin": false
+      },
+      "name": "GitHub Actions",
+      "description": "Automate your workflow from idea to production",
+      "external_url": "https://help.github.com/en/actions",
+      "html_url": "https://github.com/apps/github-actions",
+      "created_at": "2018-07-30T09:30:17Z",
+      "updated_at": "2019-12-10T19:04:12Z",
+      "permissions": {
+        "actions": "write",
+        "checks": "write",
+        "contents": "write",
+        "deployments": "write",
+        "issues": "write",
+        "metadata": "read",
+        "organization_packages": "write",
+        "packages": "write",
+        "pages": "write",
+        "pull_requests": "write",
+        "repository_hooks": "write",
+        "repository_projects": "write",
+        "security_events": "write",
+        "statuses": "write",
+        "vulnerability_alerts": "read"
+      },
+      "events": [
+        "check_run",
+        "check_suite",
+        "create",
+        "delete",
+        "deployment",
+        "deployment_status",
+        "fork",
+        "gollum",
+        "issues",
+        "issue_comment",
+        "label",
+        "milestone",
+        "page_build",
+        "project",
+        "project_card",
+        "project_column",
+        "public",
+        "pull_request",
+        "pull_request_review",
+        "pull_request_review_comment",
+        "push",
+        "registry_package",
+        "release",
+        "repository",
+        "repository_dispatch",
+        "status",
+        "watch",
+        "workflow_dispatch",
+        "workflow_run"
+      ]
+    },
+    "pull_requests": [
+      {
+        "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/1234567890",
+        "id": 1234567890,
+        "number": 1234567890,
+        "head": {
+          "ref": "feature",
+          "sha": "1234567890123456789012345678901234567890",
+          "repo": {
+            "id": 1234567890,
+            "url": "https://api.github.com/repos/MYORG/MYREPO",
+            "name": "MYREPO"
+          }
+        },
+        "base": {
+          "ref": "master",
+          "sha": "1234567890123456789012345678901234567890",
+          "repo": {
+            "id": 1234567890,
+            "url": "https://api.github.com/repos/MYORG/MYREPO",
+            "name": "MYREPO"
+          }
+        }
+      }
+    ]
+  },
+  "repository": {
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "name": "MYREPO",
+    "full_name": "MYORG/MYREPO",
+    "private": true,
+    "owner": {
+      "login": "MYUSER",
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/MYUSER",
+      "html_url": "https://github.com/MYUSER",
+      "followers_url": "https://api.github.com/users/MYUSER/followers",
+      "following_url": "https://api.github.com/users/MYUSER/following{/other_user}",
+      "gists_url": "https://api.github.com/users/MYUSER/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/MYUSER/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/MYUSER/subscriptions",
+      "organizations_url": "https://api.github.com/users/MYUSER/orgs",
+      "repos_url": "https://api.github.com/users/MYUSER/repos",
+      "events_url": "https://api.github.com/users/MYUSER/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/MYUSER/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "html_url": "https://github.com/MYUSER/MYREPO",
+    "description": null,
+    "fork": false,
+    "url": "https://api.github.com/repos/MYUSER/MYREPO",
+    "forks_url": "https://api.github.com/repos/MYUSER/MYREPO/forks",
+    "keys_url": "https://api.github.com/repos/MYUSER/MYREPO/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/MYUSER/MYREPO/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/MYUSER/MYREPO/teams",
+    "hooks_url": "https://api.github.com/repos/MYUSER/MYREPO/hooks",
+    "issue_events_url": "https://api.github.com/repos/MYUSER/MYREPO/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/MYUSER/MYREPO/events",
+    "assignees_url": "https://api.github.com/repos/MYUSER/MYREPO/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/MYUSER/MYREPO/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/MYUSER/MYREPO/tags",
+    "blobs_url": "https://api.github.com/repos/MYUSER/MYREPO/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/MYUSER/MYREPO/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/MYUSER/MYREPO/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/MYUSER/MYREPO/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/MYUSER/MYREPO/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/MYUSER/MYREPO/languages",
+    "stargazers_url": "https://api.github.com/repos/MYUSER/MYREPO/stargazers",
+    "contributors_url": "https://api.github.com/repos/MYUSER/MYREPO/contributors",
+    "subscribers_url": "https://api.github.com/repos/MYUSER/MYREPO/subscribers",
+    "subscription_url": "https://api.github.com/repos/MYUSER/MYREPO/subscription",
+    "commits_url": "https://api.github.com/repos/MYUSER/MYREPO/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/MYUSER/MYREPO/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/MYUSER/MYREPO/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/MYUSER/MYREPO/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/MYUSER/MYREPO/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/MYUSER/MYREPO/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/MYUSER/MYREPO/merges",
+    "archive_url": "https://api.github.com/repos/MYUSER/MYREPO/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/MYUSER/MYREPO/downloads",
+    "issues_url": "https://api.github.com/repos/MYUSER/MYREPO/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/MYUSER/MYREPO/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/MYUSER/MYREPO/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/MYUSER/MYREPO/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/MYUSER/MYREPO/labels{/name}",
+    "releases_url": "https://api.github.com/repos/MYUSER/MYREPO/releases{/id}",
+    "deployments_url": "https://api.github.com/repos/MYUSER/MYREPO/deployments",
+    "created_at": "2021-02-18T06:16:31Z",
+    "updated_at": "2021-02-18T06:16:31Z",
+    "pushed_at": "2021-02-18T06:16:31Z",
+    "git_url": "git://github.com/MYUSER/MYREPO.git",
+    "ssh_url": "git@github.com:MYUSER/MYREPO.git",
+    "clone_url": "https://github.com/MYUSER/MYREPO.git",
+    "svn_url": "https://github.com/MYUSER/MYREPO",
+    "homepage": null,
+    "size": 4,
+    "stargazers_count": 0,
+    "watchers_count": 0,
+    "language": null,
+    "has_issues": true,
+    "has_projects": true,
+    "has_downloads": true,
+    "has_wiki": true,
+    "has_pages": false,
+    "forks_count": 0,
+    "mirror_url": null,
+    "archived": false,
+    "disabled": false,
+    "open_issues_count": 0,
+    "license": null,
+    "forks": 0,
+    "open_issues": 0,
+    "watchers": 0,
+    "default_branch": "main"
+  },
+  "sender": {
+    "login": "MYUSER",
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/MYUSER",
+    "html_url": "https://github.com/MYUSER",
+    "followers_url": "https://api.github.com/users/MYUSER/followers",
+    "following_url": "https://api.github.com/users/MYUSER/following{/other_user}",
+    "gists_url": "https://api.github.com/users/MYUSER/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/MYUSER/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/MYUSER/subscriptions",
+    "organizations_url": "https://api.github.com/users/MYUSER/orgs",
+    "repos_url": "https://api.github.com/users/MYUSER/repos",
+    "events_url": "https://api.github.com/users/MYUSER/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/MYUSER/received_events",
+    "type": "User",
+    "site_admin": false
+  }
+}

github/fake/fake.go

@@ -24,13 +24,34 @@ const (
 `
 )
 
-type Handler struct {
+type ListRunnersHandler struct {
 	Status int
 	Body   string
 }
 
+func (h *ListRunnersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.WriteHeader(h.Status)
+	fmt.Fprintf(w, h.Body)
+}
+
+type Handler struct {
+	Status int
+	Body   string
+
+	Statuses map[string]string
+}
+
 func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	w.WriteHeader(h.Status)
+
+	status := req.URL.Query().Get("status")
+	if h.Statuses != nil {
+		if body, ok := h.Statuses[status]; ok {
+			fmt.Fprintf(w, body)
+			return
+		}
+	}
+
 	fmt.Fprintf(w, h.Body)
 }
 
@@ -92,12 +113,21 @@ func NewServer(opts ...Option) *httptest.Server {
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
+		"/enterprises/test/actions/runners/registration-token": &Handler{
+			Status: http.StatusCreated,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/enterprises/invalid/actions/runners/registration-token": &Handler{
+			Status: http.StatusOK,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/enterprises/error/actions/runners/registration-token": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
 
 		// For ListRunners
-		"/repos/test/valid/actions/runners": &Handler{
-			Status: http.StatusOK,
-			Body:   RunnersListBody,
-		},
+		"/repos/test/valid/actions/runners": config.FixedResponses.ListRunners,
 		"/repos/test/invalid/actions/runners": &Handler{
 			Status: http.StatusNoContent,
 			Body:   "",
@@ -118,6 +148,18 @@ func NewServer(opts ...Option) *httptest.Server {
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
+		"/enterprises/test/actions/runners": &Handler{
+			Status: http.StatusOK,
+			Body:   RunnersListBody,
+		},
+		"/enterprises/invalid/actions/runners": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/enterprises/error/actions/runners": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
 
 		// For RemoveRunner
 		"/repos/test/valid/actions/runners/1": &Handler{
@@ -144,6 +186,18 @@ func NewServer(opts ...Option) *httptest.Server {
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
+		"/enterprises/test/actions/runners/1": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/enterprises/invalid/actions/runners/1": &Handler{
+			Status: http.StatusOK,
+			Body:   "",
+		},
+		"/enterprises/error/actions/runners/1": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
 
 		// For auto-scaling based on the number of queued(pending) workflow runs
 		"/repos/test/valid/actions/runs": config.FixedResponses.ListRepositoryWorkflowRuns,
@@ -159,3 +213,10 @@ func NewServer(opts ...Option) *httptest.Server {
 
 	return httptest.NewServer(mux)
 }
+
+func DefaultListRunnersHandler() *ListRunnersHandler {
+	return &ListRunnersHandler{
+		Status: http.StatusOK,
+		Body:   RunnersListBody,
+	}
+}

github/fake/options.go

@@ -1,17 +1,24 @@
 package fake
 
+import "net/http"
+
 type FixedResponses struct {
 	ListRepositoryWorkflowRuns *Handler
 	ListWorkflowJobs           *MapHandler
+	ListRunners                http.Handler
 }
 
 type Option func(*ServerConfig)
 
-func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
+func WithListRepositoryWorkflowRunsResponse(status int, body, queued, in_progress string) Option {
 	return func(c *ServerConfig) {
 		c.FixedResponses.ListRepositoryWorkflowRuns = &Handler{
 			Status: status,
 			Body:   body,
+			Statuses: map[string]string{
+				"queued":      queued,
+				"in_progress": in_progress,
+			},
 		}
 	}
 }
@@ -25,6 +32,15 @@ func WithListWorkflowJobsResponse(status int, bodies map[int]string) Option {
 	}
 }
 
+func WithListRunnersResponse(status int, body string) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListRunners = &ListRunnersHandler{
+			Status: status,
+			Body:   body,
+		}
+	}
+}
+
 func WithFixedResponses(responses *FixedResponses) Option {
 	return func(c *ServerConfig) {
 		c.FixedResponses = responses

github/fake/runners.go

@@ -2,6 +2,7 @@ package fake
 
 import (
 	"encoding/json"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -29,15 +30,15 @@ func (r *RunnersList) Add(runner *github.Runner) {
 func (r *RunnersList) GetServer() *httptest.Server {
 	router := mux.NewRouter()
 
-	router.Handle("/repos/{owner}/{repo}/actions/runners", r.handleList())
+	router.Handle("/repos/{owner}/{repo}/actions/runners", r.HandleList())
 	router.Handle("/repos/{owner}/{repo}/actions/runners/{id}", r.handleRemove())
-	router.Handle("/orgs/{org}/actions/runners", r.handleList())
+	router.Handle("/orgs/{org}/actions/runners", r.HandleList())
 	router.Handle("/orgs/{org}/actions/runners/{id}", r.handleRemove())
 
 	return httptest.NewServer(router)
 }
 
-func (r *RunnersList) handleList() http.HandlerFunc {
+func (r *RunnersList) HandleList() http.HandlerFunc {
 	return func(w http.ResponseWriter, res *http.Request) {
 		j, err := json.Marshal(github.Runners{
 			TotalCount: len(r.runners),
@@ -64,6 +65,20 @@ func (r *RunnersList) handleRemove() http.HandlerFunc {
 	}
 }
 
+func (r *RunnersList) Sync(runners []v1alpha1.Runner) {
+	r.runners = nil
+
+	for i, want := range runners {
+		r.Add(&github.Runner{
+			ID:     github.Int64(int64(i)),
+			Name:   github.String(want.Name),
+			OS:     github.String("linux"),
+			Status: github.String("online"),
+			Busy:   github.Bool(false),
+		})
+	}
+}
+
 func exists(runners []*github.Runner, runner *github.Runner) bool {
 	for _, r := range runners {
 		if *r.Name == *runner.Name {

github/github.go

@@ -5,12 +5,14 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/bradleyfalzon/ghinstallation"
 	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/github/metrics"
 	"golang.org/x/oauth2"
 )
 
@@ -34,20 +36,24 @@ type Client struct {
 
 // NewClient creates a Github Client
 func (c *Config) NewClient() (*Client, error) {
-	var (
-		httpClient *http.Client
-		client     *github.Client
-	)
-	githubBaseURL := "https://github.com/"
+	var transport http.RoundTripper
 	if len(c.Token) > 0 {
-		httpClient = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(
-			&oauth2.Token{AccessToken: c.Token},
-		))
+		transport = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(&oauth2.Token{AccessToken: c.Token})).Transport
 	} else {
-		tr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.AppID, c.AppInstallationID, c.AppPrivateKey)
-		if err != nil {
-			return nil, fmt.Errorf("authentication failed: %v", err)
+		var tr *ghinstallation.Transport
+
+		if _, err := os.Stat(c.AppPrivateKey); err == nil {
+			tr, err = ghinstallation.NewKeyFromFile(http.DefaultTransport, c.AppID, c.AppInstallationID, c.AppPrivateKey)
+			if err != nil {
+				return nil, fmt.Errorf("authentication failed: using private key at %s: %v", c.AppPrivateKey, err)
+			}
+		} else {
+			tr, err = ghinstallation.New(http.DefaultTransport, c.AppID, c.AppInstallationID, []byte(c.AppPrivateKey))
+			if err != nil {
+				return nil, fmt.Errorf("authentication failed: using private key of size %d (%s...): %v", len(c.AppPrivateKey), strings.Split(c.AppPrivateKey, "\n")[0], err)
+			}
 		}
+
 		if len(c.EnterpriseURL) > 0 {
 			githubAPIURL, err := getEnterpriseApiUrl(c.EnterpriseURL)
 			if err != nil {
@@ -55,9 +61,13 @@ func (c *Config) NewClient() (*Client, error) {
 			}
 			tr.BaseURL = githubAPIURL
 		}
-		httpClient = &http.Client{Transport: tr}
+		transport = tr
 	}
+	transport = metrics.Transport{Transport: transport}
+	httpClient := &http.Client{Transport: transport}
 
+	var client *github.Client
+	var githubBaseURL string
 	if len(c.EnterpriseURL) > 0 {
 		var err error
 		client, err = github.NewEnterpriseClient(c.EnterpriseURL, c.EnterpriseURL, httpClient)
@@ -67,6 +77,7 @@ func (c *Config) NewClient() (*Client, error) {
 		githubBaseURL = fmt.Sprintf("%s://%s%s", client.BaseURL.Scheme, client.BaseURL.Host, strings.TrimSuffix(client.BaseURL.Path, "api/v3/"))
 	} else {
 		client = github.NewClient(httpClient)
+		githubBaseURL = "https://github.com/"
 	}
 
 	return &Client{
@@ -78,24 +89,27 @@ func (c *Config) NewClient() (*Client, error) {
 }
 
 // GetRegistrationToken returns a registration token tied with the name of repository and runner.
-func (c *Client) GetRegistrationToken(ctx context.Context, org, repo, name string) (*github.RegistrationToken, error) {
+func (c *Client) GetRegistrationToken(ctx context.Context, enterprise, org, repo, name string) (*github.RegistrationToken, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	key := getRegistrationKey(org, repo)
+	key := getRegistrationKey(org, repo, enterprise)
 	rt, ok := c.regTokens[key]
 
-	if ok && rt.GetExpiresAt().After(time.Now()) {
+	// we like to give runners a chance that are just starting up and may miss the expiration date by a bit
+	runnerStartupTimeout := 3 * time.Minute
+
+	if ok && rt.GetExpiresAt().After(time.Now().Add(runnerStartupTimeout)) {
 		return rt, nil
 	}
 
-	owner, repo, err := getOwnerAndRepo(org, repo)
+	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
 
 	if err != nil {
 		return rt, err
 	}
 
-	rt, res, err := c.createRegistrationToken(ctx, owner, repo)
+	rt, res, err := c.createRegistrationToken(ctx, enterprise, owner, repo)
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to create registration token: %v", err)
@@ -114,17 +128,17 @@ func (c *Client) GetRegistrationToken(ctx context.Context, org, repo, name strin
 }
 
 // RemoveRunner removes a runner with specified runner ID from repository.
-func (c *Client) RemoveRunner(ctx context.Context, org, repo string, runnerID int64) error {
-	owner, repo, err := getOwnerAndRepo(org, repo)
+func (c *Client) RemoveRunner(ctx context.Context, enterprise, org, repo string, runnerID int64) error {
+	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
 
 	if err != nil {
 		return err
 	}
 
-	res, err := c.removeRunner(ctx, owner, repo, runnerID)
+	res, err := c.removeRunner(ctx, enterprise, owner, repo, runnerID)
 
 	if err != nil {
-		return fmt.Errorf("failed to remove runner: %v", err)
+		return fmt.Errorf("failed to remove runner: %w", err)
 	}
 
 	if res.StatusCode != 204 {
@@ -135,8 +149,8 @@ func (c *Client) RemoveRunner(ctx context.Context, org, repo string, runnerID in
 }
 
 // ListRunners returns a list of runners of specified owner/repository name.
-func (c *Client) ListRunners(ctx context.Context, org, repo string) ([]*github.Runner, error) {
-	owner, repo, err := getOwnerAndRepo(org, repo)
+func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string) ([]*github.Runner, error) {
+	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
 
 	if err != nil {
 		return nil, err
@@ -144,12 +158,12 @@ func (c *Client) ListRunners(ctx context.Context, org, repo string) ([]*github.R
 
 	var runners []*github.Runner
 
-	opts := github.ListOptions{PerPage: 10}
+	opts := github.ListOptions{PerPage: 100}
 	for {
-		list, res, err := c.listRunners(ctx, owner, repo, &opts)
+		list, res, err := c.listRunners(ctx, enterprise, owner, repo, &opts)
 
 		if err != nil {
-			return runners, fmt.Errorf("failed to list runners: %v", err)
+			return runners, fmt.Errorf("failed to list runners: %w", err)
 		}
 
 		runners = append(runners, list.Runners...)
@@ -174,49 +188,102 @@ func (c *Client) cleanup() {
 	}
 }
 
-// wrappers for github functions (switch between organization/repository mode)
+// wrappers for github functions (switch between enterprise/organization/repository mode)
 // so the calling functions don't need to switch and their code is a bit cleaner
 
-func (c *Client) createRegistrationToken(ctx context.Context, owner, repo string) (*github.RegistrationToken, *github.Response, error) {
+func (c *Client) createRegistrationToken(ctx context.Context, enterprise, org, repo string) (*github.RegistrationToken, *github.Response, error) {
 	if len(repo) > 0 {
-		return c.Client.Actions.CreateRegistrationToken(ctx, owner, repo)
-	}
-
-	return c.Client.Actions.CreateOrganizationRegistrationToken(ctx, owner)
-}
-
-func (c *Client) removeRunner(ctx context.Context, owner, repo string, runnerID int64) (*github.Response, error) {
-	if len(repo) > 0 {
-		return c.Client.Actions.RemoveRunner(ctx, owner, repo, runnerID)
-	}
-
-	return c.Client.Actions.RemoveOrganizationRunner(ctx, owner, runnerID)
-}
-
-func (c *Client) listRunners(ctx context.Context, owner, repo string, opts *github.ListOptions) (*github.Runners, *github.Response, error) {
-	if len(repo) > 0 {
-		return c.Client.Actions.ListRunners(ctx, owner, repo, opts)
-	}
-
-	return c.Client.Actions.ListOrganizationRunners(ctx, owner, opts)
-}
-
-// Validates owner and repo arguments. Both are optional, but at least one should be specified
-func getOwnerAndRepo(org, repo string) (string, string, error) {
-	if len(repo) > 0 {
-		return splitOwnerAndRepo(repo)
+		return c.Client.Actions.CreateRegistrationToken(ctx, org, repo)
 	}
 	if len(org) > 0 {
-		return org, "", nil
+		return c.Client.Actions.CreateOrganizationRegistrationToken(ctx, org)
 	}
-	return "", "", fmt.Errorf("organization and repository are both empty")
+	return c.Client.Enterprise.CreateRegistrationToken(ctx, enterprise)
 }
 
-func getRegistrationKey(org, repo string) string {
-	if len(org) > 0 {
-		return org
+func (c *Client) removeRunner(ctx context.Context, enterprise, org, repo string, runnerID int64) (*github.Response, error) {
+	if len(repo) > 0 {
+		return c.Client.Actions.RemoveRunner(ctx, org, repo, runnerID)
 	}
-	return repo
+	if len(org) > 0 {
+		return c.Client.Actions.RemoveOrganizationRunner(ctx, org, runnerID)
+	}
+	return c.Client.Enterprise.RemoveRunner(ctx, enterprise, runnerID)
+}
+
+func (c *Client) listRunners(ctx context.Context, enterprise, org, repo string, opts *github.ListOptions) (*github.Runners, *github.Response, error) {
+	if len(repo) > 0 {
+		return c.Client.Actions.ListRunners(ctx, org, repo, opts)
+	}
+	if len(org) > 0 {
+		return c.Client.Actions.ListOrganizationRunners(ctx, org, opts)
+	}
+	return c.Client.Enterprise.ListRunners(ctx, enterprise, opts)
+}
+
+func (c *Client) ListRepositoryWorkflowRuns(ctx context.Context, user string, repoName string) ([]*github.WorkflowRun, error) {
+	queued, err := c.listRepositoryWorkflowRuns(ctx, user, repoName, "queued")
+	if err != nil {
+		return nil, fmt.Errorf("listing queued workflow runs: %w", err)
+	}
+
+	inProgress, err := c.listRepositoryWorkflowRuns(ctx, user, repoName, "in_progress")
+	if err != nil {
+		return nil, fmt.Errorf("listing in_progress workflow runs: %w", err)
+	}
+
+	var workflowRuns []*github.WorkflowRun
+
+	workflowRuns = append(workflowRuns, queued...)
+	workflowRuns = append(workflowRuns, inProgress...)
+
+	return workflowRuns, nil
+}
+
+func (c *Client) listRepositoryWorkflowRuns(ctx context.Context, user string, repoName, status string) ([]*github.WorkflowRun, error) {
+	var workflowRuns []*github.WorkflowRun
+
+	opts := github.ListWorkflowRunsOptions{
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+		Status: status,
+	}
+
+	for {
+		list, res, err := c.Client.Actions.ListRepositoryWorkflowRuns(ctx, user, repoName, &opts)
+
+		if err != nil {
+			return workflowRuns, fmt.Errorf("failed to list workflow runs: %v", err)
+		}
+
+		workflowRuns = append(workflowRuns, list.WorkflowRuns...)
+		if res.NextPage == 0 {
+			break
+		}
+		opts.Page = res.NextPage
+	}
+
+	return workflowRuns, nil
+}
+
+// Validates enterprise, organisation and repo arguments. Both are optional, but at least one should be specified
+func getEnterpriseOrganisationAndRepo(enterprise, org, repo string) (string, string, string, error) {
+	if len(repo) > 0 {
+		owner, repository, err := splitOwnerAndRepo(repo)
+		return "", owner, repository, err
+	}
+	if len(org) > 0 {
+		return "", org, "", nil
+	}
+	if len(enterprise) > 0 {
+		return enterprise, "", "", nil
+	}
+	return "", "", "", fmt.Errorf("enterprise, organization and repository are all empty")
+}
+
+func getRegistrationKey(org, repo, enterprise string) string {
+	return fmt.Sprintf("org=%s,repo=%s,enterprise=%s", org, repo, enterprise)
 }
 
 func splitOwnerAndRepo(repo string) (string, string, error) {
@@ -244,3 +311,37 @@ func getEnterpriseApiUrl(baseURL string) (string, error) {
 	// Trim trailing slash, otherwise there's double slash added to token endpoint
 	return fmt.Sprintf("%s://%s%s", baseEndpoint.Scheme, baseEndpoint.Host, strings.TrimSuffix(baseEndpoint.Path, "/")), nil
 }
+
+type RunnerNotFound struct {
+	runnerName string
+}
+
+func (e *RunnerNotFound) Error() string {
+	return fmt.Sprintf("runner %q not found", e.runnerName)
+}
+
+type RunnerOffline struct {
+	runnerName string
+}
+
+func (e *RunnerOffline) Error() string {
+	return fmt.Sprintf("runner %q offline", e.runnerName)
+}
+
+func (r *Client) IsRunnerBusy(ctx context.Context, enterprise, org, repo, name string) (bool, error) {
+	runners, err := r.ListRunners(ctx, enterprise, org, repo)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			if runner.GetStatus() == "offline" {
+				return false, &RunnerOffline{runnerName: name}
+			}
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, &RunnerNotFound{runnerName: name}
+}

github/github_test.go

@@ -32,29 +32,36 @@ func newTestClient() *Client {
 }
 
 func TestMain(m *testing.M) {
-	server = fake.NewServer()
+	res := &fake.FixedResponses{
+		ListRunners: fake.DefaultListRunnersHandler(),
+	}
+	server = fake.NewServer(fake.WithFixedResponses(res))
 	defer server.Close()
 	m.Run()
 }
 
 func TestGetRegistrationToken(t *testing.T) {
 	tests := []struct {
-		org   string
-		repo  string
-		token string
-		err   bool
+		enterprise string
+		org        string
+		repo       string
+		token      string
+		err        bool
 	}{
-		{org: "", repo: "test/valid", token: fake.RegistrationToken, err: false},
-		{org: "", repo: "test/invalid", token: "", err: true},
-		{org: "", repo: "test/error", token: "", err: true},
-		{org: "test", repo: "", token: fake.RegistrationToken, err: false},
-		{org: "invalid", repo: "", token: "", err: true},
-		{org: "error", repo: "", token: "", err: true},
+		{enterprise: "", org: "", repo: "test/valid", token: fake.RegistrationToken, err: false},
+		{enterprise: "", org: "", repo: "test/invalid", token: "", err: true},
+		{enterprise: "", org: "", repo: "test/error", token: "", err: true},
+		{enterprise: "", org: "test", repo: "", token: fake.RegistrationToken, err: false},
+		{enterprise: "", org: "invalid", repo: "", token: "", err: true},
+		{enterprise: "", org: "error", repo: "", token: "", err: true},
+		{enterprise: "test", org: "", repo: "", token: fake.RegistrationToken, err: false},
+		{enterprise: "invalid", org: "", repo: "", token: "", err: true},
+		{enterprise: "error", org: "", repo: "", token: "", err: true},
 	}
 
 	client := newTestClient()
 	for i, tt := range tests {
-		rt, err := client.GetRegistrationToken(context.Background(), tt.org, tt.repo, "test")
+		rt, err := client.GetRegistrationToken(context.Background(), tt.enterprise, tt.org, tt.repo, "test")
 		if !tt.err && err != nil {
 			t.Errorf("[%d] unexpected error: %v", i, err)
 		}
@@ -66,22 +73,26 @@ func TestGetRegistrationToken(t *testing.T) {
 
 func TestListRunners(t *testing.T) {
 	tests := []struct {
-		org    string
-		repo   string
-		length int
-		err    bool
+		enterprise string
+		org        string
+		repo       string
+		length     int
+		err        bool
 	}{
-		{org: "", repo: "test/valid", length: 2, err: false},
-		{org: "", repo: "test/invalid", length: 0, err: true},
-		{org: "", repo: "test/error", length: 0, err: true},
-		{org: "test", repo: "", length: 2, err: false},
-		{org: "invalid", repo: "", length: 0, err: true},
-		{org: "error", repo: "", length: 0, err: true},
+		{enterprise: "", org: "", repo: "test/valid", length: 2, err: false},
+		{enterprise: "", org: "", repo: "test/invalid", length: 0, err: true},
+		{enterprise: "", org: "", repo: "test/error", length: 0, err: true},
+		{enterprise: "", org: "test", repo: "", length: 2, err: false},
+		{enterprise: "", org: "invalid", repo: "", length: 0, err: true},
+		{enterprise: "", org: "error", repo: "", length: 0, err: true},
+		{enterprise: "test", org: "", repo: "", length: 2, err: false},
+		{enterprise: "invalid", org: "", repo: "", length: 0, err: true},
+		{enterprise: "error", org: "", repo: "", length: 0, err: true},
 	}
 
 	client := newTestClient()
 	for i, tt := range tests {
-		runners, err := client.ListRunners(context.Background(), tt.org, tt.repo)
+		runners, err := client.ListRunners(context.Background(), tt.enterprise, tt.org, tt.repo)
 		if !tt.err && err != nil {
 			t.Errorf("[%d] unexpected error: %v", i, err)
 		}
@@ -93,21 +104,25 @@ func TestListRunners(t *testing.T) {
 
 func TestRemoveRunner(t *testing.T) {
 	tests := []struct {
-		org  string
-		repo string
-		err  bool
+		enterprise string
+		org        string
+		repo       string
+		err        bool
 	}{
-		{org: "", repo: "test/valid", err: false},
-		{org: "", repo: "test/invalid", err: true},
-		{org: "", repo: "test/error", err: true},
-		{org: "test", repo: "", err: false},
-		{org: "invalid", repo: "", err: true},
-		{org: "error", repo: "", err: true},
+		{enterprise: "", org: "", repo: "test/valid", err: false},
+		{enterprise: "", org: "", repo: "test/invalid", err: true},
+		{enterprise: "", org: "", repo: "test/error", err: true},
+		{enterprise: "", org: "test", repo: "", err: false},
+		{enterprise: "", org: "invalid", repo: "", err: true},
+		{enterprise: "", org: "error", repo: "", err: true},
+		{enterprise: "test", org: "", repo: "", err: false},
+		{enterprise: "invalid", org: "", repo: "", err: true},
+		{enterprise: "error", org: "", repo: "", err: true},
 	}
 
 	client := newTestClient()
 	for i, tt := range tests {
-		err := client.RemoveRunner(context.Background(), tt.org, tt.repo, int64(1))
+		err := client.RemoveRunner(context.Background(), tt.enterprise, tt.org, tt.repo, int64(1))
 		if !tt.err && err != nil {
 			t.Errorf("[%d] unexpected error: %v", i, err)
 		}

github/metrics/transport.go

@@ -0,0 +1,63 @@
+// Package metrics provides monitoring of the GitHub related metrics.
+//
+// This depends on the metrics exporter of kubebuilder.
+// See https://book.kubebuilder.io/reference/metrics.html for details.
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+func init() {
+	metrics.Registry.MustRegister(metricRateLimit, metricRateLimitRemaining)
+}
+
+var (
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+	metricRateLimit = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit",
+			Help: "The maximum number of requests you're permitted to make per hour",
+		},
+	)
+	metricRateLimitRemaining = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit_remaining",
+			Help: "The number of requests remaining in the current rate limit window",
+		},
+	)
+)
+
+const (
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+	headerRateLimit          = "X-RateLimit-Limit"
+	headerRateLimitRemaining = "X-RateLimit-Remaining"
+)
+
+// Transport wraps a transport with metrics monitoring
+type Transport struct {
+	Transport http.RoundTripper
+}
+
+func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := t.Transport.RoundTrip(req)
+	if resp != nil {
+		parseResponse(resp)
+	}
+	return resp, err
+}
+
+func parseResponse(resp *http.Response) {
+	rateLimit, err := strconv.Atoi(resp.Header.Get(headerRateLimit))
+	if err == nil {
+		metricRateLimit.Set(float64(rateLimit))
+	}
+	rateLimitRemaining, err := strconv.Atoi(resp.Header.Get(headerRateLimitRemaining))
+	if err == nil {
+		metricRateLimitRemaining.Set(float64(rateLimitRemaining))
+	}
+}

go.mod

@@ -6,14 +6,13 @@ require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v0.1.0
-	github.com/google/go-github v17.0.0+incompatible // indirect
-	github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04
-	github.com/google/go-github/v33 v33.0.0
-	github.com/google/go-querystring v1.0.0
+	github.com/google/go-cmp v0.3.1
+	github.com/google/go-github/v33 v33.0.1-0.20210204004227-319dcffb518a
 	github.com/gorilla/mux v1.8.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.5.0
+	github.com/prometheus/client_golang v0.9.2
 	github.com/stretchr/testify v1.4.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f

go.sum

@@ -116,14 +116,10 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
-github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04 h1:wEYk2h/GwOhImcVjiTIceP88WxVbXw2F+ARYUQMEsfg=
-github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
-github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM=
-github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg=
+github.com/google/go-github/v33 v33.0.1-0.20210204004227-319dcffb518a h1:Z9Nzq8ntvvXCLnFGOkzzcD8HDOzOo+obuwE5oK85vNQ=
+github.com/google/go-github/v33 v33.0.1-0.20210204004227-319dcffb518a/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=

main.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/kelseyhightower/envconfig"
@@ -40,8 +41,8 @@ const (
 )
 
 var (
-	scheme   = runtime.NewScheme()
-	setupLog = ctrl.Log.WithName("setup")
+	scheme = runtime.NewScheme()
+	log    = ctrl.Log.WithName("actions-runner-controller")
 )
 
 func init() {
@@ -62,6 +63,9 @@ func main() {
 
 		runnerImage string
 		dockerImage string
+		namespace   string
+
+		commonRunnerLabels commaSeparatedStringSlice
 	)
 
 	var c github.Config
@@ -80,6 +84,8 @@ func main() {
 	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
 	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
 	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
+	flag.Var(&commonRunnerLabels, "common-runner-labels", "Runner labels in the K1=V1,K2=V2,... format that are inherited all the runners created by the controller. See https://github.com/summerwind/actions-runner-controller/issues/321 for more information")
+	flag.StringVar(&namespace, "watch-namespace", "", "The namespace to watch for custom resources. Set to empty for letting it watch for all namespaces.")
 	flag.Parse()
 
 	logger := zap.New(func(o *zap.Options) {
@@ -100,15 +106,16 @@ func main() {
 		LeaderElection:     enableLeaderElection,
 		Port:               9443,
 		SyncPeriod:         &syncPeriod,
+		Namespace:          namespace,
 	})
 	if err != nil {
-		setupLog.Error(err, "unable to start manager")
+		log.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
 
 	runnerReconciler := &controllers.RunnerReconciler{
 		Client:       mgr.GetClient(),
-		Log:          ctrl.Log.WithName("controllers").WithName("Runner"),
+		Log:          log.WithName("runner"),
 		Scheme:       mgr.GetScheme(),
 		GitHubClient: ghClient,
 		RunnerImage:  runnerImage,
@@ -116,62 +123,81 @@ func main() {
 	}
 
 	if err = runnerReconciler.SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "Runner")
+		log.Error(err, "unable to create controller", "controller", "Runner")
 		os.Exit(1)
 	}
 
 	runnerSetReconciler := &controllers.RunnerReplicaSetReconciler{
 		Client:       mgr.GetClient(),
-		Log:          ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
+		Log:          log.WithName("runnerreplicaset"),
 		Scheme:       mgr.GetScheme(),
 		GitHubClient: ghClient,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "RunnerReplicaSet")
+		log.Error(err, "unable to create controller", "controller", "RunnerReplicaSet")
 		os.Exit(1)
 	}
 
 	runnerDeploymentReconciler := &controllers.RunnerDeploymentReconciler{
-		Client: mgr.GetClient(),
-		Log:    ctrl.Log.WithName("controllers").WithName("RunnerDeployment"),
-		Scheme: mgr.GetScheme(),
+		Client:             mgr.GetClient(),
+		Log:                log.WithName("runnerdeployment"),
+		Scheme:             mgr.GetScheme(),
+		CommonRunnerLabels: commonRunnerLabels,
 	}
 
 	if err = runnerDeploymentReconciler.SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "RunnerDeployment")
+		log.Error(err, "unable to create controller", "controller", "RunnerDeployment")
 		os.Exit(1)
 	}
 
 	horizontalRunnerAutoscaler := &controllers.HorizontalRunnerAutoscalerReconciler{
-		Client:       mgr.GetClient(),
-		Log:          ctrl.Log.WithName("controllers").WithName("HorizontalRunnerAutoscaler"),
-		Scheme:       mgr.GetScheme(),
-		GitHubClient: ghClient,
+		Client:        mgr.GetClient(),
+		Log:           log.WithName("horizontalrunnerautoscaler"),
+		Scheme:        mgr.GetScheme(),
+		GitHubClient:  ghClient,
+		CacheDuration: syncPeriod - 10*time.Second,
 	}
 
 	if err = horizontalRunnerAutoscaler.SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "HorizontalRunnerAutoscaler")
+		log.Error(err, "unable to create controller", "controller", "HorizontalRunnerAutoscaler")
 		os.Exit(1)
 	}
 
 	if err = (&actionsv1alpha1.Runner{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create webhook", "webhook", "Runner")
+		log.Error(err, "unable to create webhook", "webhook", "Runner")
 		os.Exit(1)
 	}
 	if err = (&actionsv1alpha1.RunnerDeployment{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create webhook", "webhook", "RunnerDeployment")
+		log.Error(err, "unable to create webhook", "webhook", "RunnerDeployment")
 		os.Exit(1)
 	}
 	if err = (&actionsv1alpha1.RunnerReplicaSet{}).SetupWebhookWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create webhook", "webhook", "RunnerReplicaSet")
+		log.Error(err, "unable to create webhook", "webhook", "RunnerReplicaSet")
 		os.Exit(1)
 	}
 	// +kubebuilder:scaffold:builder
 
-	setupLog.Info("starting manager")
+	log.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-		setupLog.Error(err, "problem running manager")
+		log.Error(err, "problem running manager")
 		os.Exit(1)
 	}
 }
+
+type commaSeparatedStringSlice []string
+
+func (s *commaSeparatedStringSlice) String() string {
+	return fmt.Sprintf("%v", *s)
+}
+
+func (s *commaSeparatedStringSlice) Set(value string) error {
+	for _, v := range strings.Split(value, ",") {
+		if v == "" {
+			continue
+		}
+
+		*s = append(*s, v)
+	}
+	return nil
+}

pkg/actionsglob/README.md

@@ -0,0 +1,8 @@
+This package is an implementation of glob that is intended to simulate the behaviour of 
+https://github.com/actions/toolkit/tree/master/packages/glob in many cases.
+
+This isn't a complete reimplementation of the referenced nodejs package.
+
+Differences:
+
+- This package doesn't implement `**`

pkg/actionsglob/actionsglob.go

@@ -0,0 +1,78 @@
+package actionsglob
+
+import (
+	"fmt"
+	"strings"
+)
+
+func Match(pat string, s string) bool {
+	if len(pat) == 0 {
+		panic(fmt.Sprintf("unexpected length of pattern: %d", len(pat)))
+	}
+
+	var inverse bool
+
+	if pat[0] == '!' {
+		pat = pat[1:]
+		inverse = true
+	}
+
+	tokens := strings.SplitAfter(pat, "*")
+
+	var wildcardInHead bool
+
+	for i := 0; i < len(tokens); i++ {
+		p := tokens[i]
+
+		if p == "" {
+			s = ""
+			break
+		}
+
+		if p == "*" {
+			if i == len(tokens)-1 {
+				s = ""
+				break
+			}
+
+			wildcardInHead = true
+
+			continue
+		}
+
+		wildcardInTail := p[len(p)-1] == '*'
+		if wildcardInTail {
+			p = p[:len(p)-1]
+		}
+
+		subs := strings.SplitN(s, p, 2)
+
+		if len(subs) == 0 {
+			break
+		}
+
+		if subs[0] != "" {
+			if !wildcardInHead {
+				break
+			}
+		}
+
+		if subs[1] != "" {
+			if !wildcardInTail {
+				break
+			}
+		}
+
+		s = subs[1]
+
+		wildcardInHead = wildcardInTail
+	}
+
+	r := s == ""
+
+	if inverse {
+		r = !r
+	}
+
+	return r
+}

pkg/actionsglob/match_test.go

@@ -0,0 +1,214 @@
+package actionsglob
+
+import (
+	"testing"
+)
+
+func TestMatch(t *testing.T) {
+	type testcase struct {
+		Pattern, Target string
+		Want            bool
+	}
+
+	run := func(t *testing.T, tc testcase) {
+		t.Helper()
+
+		got := Match(tc.Pattern, tc.Target)
+
+		if got != tc.Want {
+			t.Errorf("%s against %s: want %v, got %v", tc.Pattern, tc.Target, tc.Want, got)
+		}
+	}
+
+	t.Run("foo == foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "foo",
+			Target:  "foo",
+			Want:    true,
+		})
+	})
+
+	t.Run("!foo == foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!foo",
+			Target:  "foo",
+			Want:    false,
+		})
+	})
+
+	t.Run("foo == foo1", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "foo",
+			Target:  "foo1",
+			Want:    false,
+		})
+	})
+
+	t.Run("!foo == foo1", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!foo",
+			Target:  "foo1",
+			Want:    true,
+		})
+	})
+
+	t.Run("*foo == foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "*foo",
+			Target:  "foo",
+			Want:    true,
+		})
+	})
+
+	t.Run("!*foo == foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!*foo",
+			Target:  "foo",
+			Want:    false,
+		})
+	})
+
+	t.Run("*foo == 1foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "*foo",
+			Target:  "1foo",
+			Want:    true,
+		})
+	})
+
+	t.Run("!*foo == 1foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!*foo",
+			Target:  "1foo",
+			Want:    false,
+		})
+	})
+
+	t.Run("*foo == foo1", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "*foo",
+			Target:  "foo1",
+			Want:    false,
+		})
+	})
+
+	t.Run("!*foo == foo1", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!*foo",
+			Target:  "foo1",
+			Want:    true,
+		})
+	})
+
+	t.Run("*foo* == foo1", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "*foo*",
+			Target:  "foo1",
+			Want:    true,
+		})
+	})
+
+	t.Run("!*foo* == foo1", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!*foo*",
+			Target:  "foo1",
+			Want:    false,
+		})
+	})
+
+	t.Run("*foo == foobar", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "*foo",
+			Target:  "foobar",
+			Want:    false,
+		})
+	})
+
+	t.Run("!*foo == foobar", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!*foo",
+			Target:  "foobar",
+			Want:    true,
+		})
+	})
+
+	t.Run("*foo* == foobar", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "*foo*",
+			Target:  "foobar",
+			Want:    true,
+		})
+	})
+
+	t.Run("!*foo* == foobar", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!*foo*",
+			Target:  "foobar",
+			Want:    false,
+		})
+	})
+
+	t.Run("foo* == foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "foo*",
+			Target:  "foo",
+			Want:    true,
+		})
+	})
+
+	t.Run("!foo* == foo", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!foo*",
+			Target:  "foo",
+			Want:    false,
+		})
+	})
+
+	t.Run("foo* == foobar", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "foo*",
+			Target:  "foobar",
+			Want:    true,
+		})
+	})
+
+	t.Run("!foo* == foobar", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!foo*",
+			Target:  "foobar",
+			Want:    false,
+		})
+	})
+
+	t.Run("foo (* == foo ( 1 / 2 )", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "foo (*",
+			Target:  "foo ( 1 / 2 )",
+			Want:    true,
+		})
+	})
+
+	t.Run("!foo (* == foo ( 1 / 2 )", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!foo (*",
+			Target:  "foo ( 1 / 2 )",
+			Want:    false,
+		})
+	})
+
+	t.Run("actions-*-metrics == actions-workflow-metrics", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "actions-*-metrics",
+			Target:  "actions-workflow-metrics",
+			Want:    true,
+		})
+	})
+
+	t.Run("!actions-*-metrics == actions-workflow-metrics", func(t *testing.T) {
+		run(t, testcase{
+			Pattern: "!actions-*-metrics",
+			Target:  "actions-workflow-metrics",
+			Want:    false,
+		})
+	})
+}

runner/Dockerfile

@@ -37,6 +37,7 @@ RUN apt update -y \
   wget \
   zip \
   zstd \
+  && cd /usr/bin && ln -sf python3 python \
   && rm -rf /var/lib/apt/lists/*
 
 RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
@@ -79,11 +80,11 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 
 RUN echo AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache > .env \
   && mkdir /opt/hostedtoolcache \
-  && chgrp runner /opt/hostedtoolcache \
+  && chgrp docker /opt/hostedtoolcache \
   && chmod g+rwx /opt/hostedtoolcache
 
 COPY entrypoint.sh /
-COPY patched $RUNNER_ASSETS_DIR/patched
+COPY --chown=runner:docker patched $RUNNER_ASSETS_DIR/patched
 
 USER runner
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]

runner/dindrunner.Dockerfile

@@ -21,6 +21,7 @@ RUN apt update \
     netcat \
     openssh-client \
     parallel \
+    python-is-python3 \
     rsync \
     shellcheck \
     sudo \
@@ -88,7 +89,7 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 
 RUN echo AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache > /runner.env \
   && mkdir /opt/hostedtoolcache \
-  && chgrp runner /opt/hostedtoolcache \
+  && chgrp docker /opt/hostedtoolcache \
   && chmod g+rwx /opt/hostedtoolcache
 
 COPY modprobe startup.sh /usr/local/bin/
@@ -104,7 +105,7 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 
 VOLUME /var/lib/docker
 
-COPY patched $RUNNER_ASSETS_DIR/patched
+COPY --chown=runner:docker patched $RUNNER_ASSETS_DIR/patched
 
 # No group definition, as that makes it harder to run docker.
 USER runner

runner/entrypoint.sh

@@ -16,14 +16,16 @@ if [ -z "${RUNNER_NAME}" ]; then
   exit 1
 fi
 
-if [ -n "${RUNNER_ORG}" ] && [ -n "${RUNNER_REPO}" ]; then
+if [ -n "${RUNNER_ORG}" ] && [ -n "${RUNNER_REPO}" ] && [ -n "${RUNNER_ENTERPRISE}" ]; then
   ATTACH="${RUNNER_ORG}/${RUNNER_REPO}"
 elif [ -n "${RUNNER_ORG}" ]; then
   ATTACH="${RUNNER_ORG}"
 elif [ -n "${RUNNER_REPO}" ]; then
   ATTACH="${RUNNER_REPO}"
+elif [ -n "${RUNNER_ENTERPRISE}" ]; then
+  ATTACH="enterprises/${RUNNER_ENTERPRISE}"
 else
-  echo "At least one of RUNNER_ORG or RUNNER_REPO must be set" 1>&2
+  echo "At least one of RUNNER_ORG or RUNNER_REPO or RUNNER_ENTERPRISE must be set" 1>&2
   exit 1
 fi
 
@@ -40,7 +42,7 @@ if [ -z "${RUNNER_TOKEN}" ]; then
   exit 1
 fi
 
-if [ -z "${RUNNER_REPO}" ] && [ -n "${RUNNER_ORG}" ] && [ -n "${RUNNER_GROUP}" ];then
+if [ -z "${RUNNER_REPO}" ] && [ -n "${RUNNER_GROUP}" ];then
   RUNNER_GROUP_ARG="--runnergroup ${RUNNER_GROUP}"
 fi
 

runner/startup.sh

@@ -33,5 +33,9 @@ for process in "${processes[@]}"; do
     fi
 done
 
+if [ -n "${MTU}" ]; then
+  ifconfig docker0 mtu ${MTU} up
+fi
+
 # Wait processes to be running
 entrypoint.sh