Work-around for offline+busy runners (#993)

Ref #911

.github/actions/setup-docker-environment/action.yaml

@@ -0,0 +1,38 @@
+name: "Setup Docker"
+
+inputs:
+  username:
+    description: "Username"
+    required: true
+  password:
+    description: "Password"
+    required: true
+
+outputs:
+  sha_short:
+    description: "The short SHA used for image builds"
+    value: ${{ steps.vars.outputs.sha_short }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get Short SHA
+      id: vars
+      run: |
+        echo ::set-output name=sha_short::${GITHUB_SHA::7}
+      shell: bash
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
+
+    - name: Login to DockerHub
+      if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+      uses: docker/login-action@v1
+      with:
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}

.github/renovate.json5

@@ -12,10 +12,12 @@
   "regexManagers": [
     {
       // use https://github.com/actions/runner/releases
-      "fileMatch": [".github/workflows/build-and-release-runners.yml"],
+      "fileMatch": [
+        ".github/workflows/runners.yml"
+        ],
       "matchStrings": ["RUNNER_VERSION: +(?<currentValue>.*?)\\n"],
       "depNameTemplate": "actions/runner",
       "datasourceTemplate": "github-releases"
     }
   ]
-}
+}

.github/workflows/build-and-release-runners.yml

@@ -1,123 +0,0 @@
-name: Build and Release Runners
-
-on:
-  pull_request:
-    branches:
-      - '**'
-    paths:
-      - 'runner/**'
-      - .github/workflows/build-and-release-runners.yml
-      - '!**.md'
-  push:
-    branches:
-      - master
-    paths:
-      - runner/patched/*
-      - runner/Dockerfile
-      - runner/Dockerfile.ubuntu.1804
-      - runner/Dockerfile.dindrunner
-      - runner/entrypoint.sh
-      - .github/workflows/build-and-release-runners.yml
-      - '!**.md'
-
-env:
-  RUNNER_VERSION: 2.284.0
-  DOCKER_VERSION: 20.10.8
-  DOCKERHUB_USERNAME: summerwind
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    name: Build ${{ matrix.name }}-ubuntu-${{ matrix.os-version }}
-    strategy:
-      matrix:
-        include:
-          - name: actions-runner
-            os-version: 20.04
-            dockerfile: Dockerfile
-          - name: actions-runner
-            os-version: 18.04
-            dockerfile: Dockerfile.ubuntu.1804
-          - name: actions-runner-dind
-            os-version: 20.04
-            dockerfile: Dockerfile.dindrunner
-
-    steps:
-      - name: Set outputs
-        id: vars
-        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
-
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        if: ${{ github.event_name == 'push' || github.event_name == 'release' }}
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
-      - name: Build and Push Versioned Tags
-        uses: docker/build-push-action@v2
-        with:
-          context: ./runner
-          file: ./runner/${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          build-args: |
-            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
-            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-ubuntu-${{ matrix.os-version }}
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-ubuntu-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
-
-  latest-tags:
-    if: ${{ github.event_name == 'push' || github.event_name == 'release' }}
-    runs-on: ubuntu-latest
-    name: Build ${{ matrix.name }}-latest
-    strategy:
-      matrix:
-        include:
-          - name: actions-runner
-            dockerfile: Dockerfile
-          - name: actions-runner-dind
-            dockerfile: Dockerfile.dindrunner
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
-      - name: Build and Push Latest Tag
-        uses: docker/build-push-action@v2
-        with:
-          context: ./runner
-          file: ./runner/${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
-            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:latest

.github/workflows/runners.yml

@@ -0,0 +1,65 @@
+name: Runners
+
+on:
+  pull_request:
+    types: 
+      - opened
+      - synchronize
+      - reopened
+      - closed
+    branches:
+      - 'master'
+    paths:
+      - 'runner/**'
+      - .github/workflows/runners.yml
+      - '!**.md'
+
+env:
+  RUNNER_VERSION: 2.285.1
+  DOCKER_VERSION: 20.10.8
+  DOCKERHUB_USERNAME: summerwind
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Build ${{ matrix.name }}-${{ matrix.os-name }}-${{ matrix.os-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: actions-runner
+            os-name: ubuntu
+            os-version: 20.04
+            dockerfile: Dockerfile
+          - name: actions-runner-dind
+            os-name: ubuntu
+            os-version: 20.04
+            dockerfile: Dockerfile.dindrunner
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Docker Environment
+        id: vars
+        uses: ./.github/actions/setup-docker-environment
+        with: 
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build and Push Versioned Tags
+        uses: docker/build-push-action@v2
+        with:
+          context: ./runner
+          file: ./runner/${{ matrix.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+          build-args: |
+            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
+            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/test.yaml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
     paths-ignore:
-      - .github/workflows/build-and-release-runners.yml
+      - .github/workflows/runners.yml
       - .github/workflows/on-push-lint-charts.yml
       - .github/workflows/on-push-master-publish-chart.yml
       - .github/workflows/release.yml

.github/workflows/wip.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
     paths-ignore:
-      - .github/workflows/build-and-release-runners.yml
+      - .github/workflows/runners.yml
       - .github/workflows/on-push-lint-charts.yml
       - .github/workflows/on-push-master-publish-chart.yml
       - .github/workflows/release.yml

Makefile

@@ -17,6 +17,7 @@ RUNNER_FEATURE_FLAG_EPHEMERAL ?=
 KUBECONTEXT ?= kind-acceptance
 CLUSTER ?= acceptance
 CERT_MANAGER_VERSION ?= v1.1.1
+KUBE_RBAC_PROXY_VERSION ?= v0.11.0
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,generateEmbeddedObjectMeta=true"
@@ -156,7 +157,7 @@ acceptance/kind:
 # See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap
 acceptance/load:
 	kind load docker-image ${NAME}:${VERSION} --name ${CLUSTER}
-	kind load docker-image quay.io/brancz/kube-rbac-proxy:v0.10.0 --name ${CLUSTER}
+	kind load docker-image quay.io/brancz/kube-rbac-proxy:$(KUBE_RBAC_PROXY_VERSION) --name ${CLUSTER}
 	kind load docker-image ${RUNNER_NAME}:${RUNNER_TAG} --name ${CLUSTER}
 	kind load docker-image docker:dind --name ${CLUSTER}
 	kind load docker-image quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION) --name ${CLUSTER}
@@ -166,7 +167,7 @@ acceptance/load:
 
 # Pull the docker images for acceptance
 acceptance/pull:
-	docker pull quay.io/brancz/kube-rbac-proxy:v0.10.0
+	docker pull quay.io/brancz/kube-rbac-proxy:$(KUBE_RBAC_PROXY_VERSION)
 	docker pull docker:dind
 	docker pull quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION)
 	docker pull quay.io/jetstack/cert-manager-cainjector:$(CERT_MANAGER_VERSION)

README.md

@@ -32,6 +32,7 @@ ToC:
   - [Stateful Runners](#stateful-runners)
   - [Ephemeral Runners](#ephemeral-runners)
   - [Software Installed in the Runner Image](#software-installed-in-the-runner-image)
+  - [Using without cert-manager](#using-without-cert-manager)
   - [Common Errors](#common-errors)
 - [Contributing](#contributing)
 
@@ -43,7 +44,7 @@ ToC:
 
 ## Installation
 
-actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) for certificate management of Admission Webhook. Make sure you have already installed cert-manager before you install. The installation instructions for cert-manager can be found below.
+By default, actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) for certificate management of Admission Webhook. Make sure you have already installed cert-manager before you install. The installation instructions for cert-manager can be found below.
 
 - [Installing cert-manager on Kubernetes](https://cert-manager.io/docs/installation/kubernetes/)
 
@@ -68,11 +69,11 @@ helm upgrade --install --namespace actions-runner-system --create-namespace \
 
 ### GitHub Enterprise Support
 
-The solution supports both GitHub Enterprise Cloud and Server editions as well as regular GitHub. Both PAT (personal access token) and GitHub App authentication works for installations that will be deploying either repository level and / or organization level runners. If you need to deploy enterprise level runners then you are restricted to PAT based authentication as GitHub doesn't support GitHub App based authentication for enterprise runners currently.
+The solution supports both GHEC (GitHub Enterprise Cloud) and GHES (GitHub Enterprise Server) editions as well as regular GitHub. Both PAT (personal access token) and GitHub App authentication works for installations that will be deploying either repository level and / or organization level runners. If you need to deploy enterprise level runners then you are restricted to PAT based authentication as GitHub doesn't support GitHub App based authentication for enterprise runners currently.
 
-If you are deploying this solution into a GitHub Enterprise Server environment then you will need version >= [3.0.0](https://docs.github.com/en/enterprise-server@3.0/admin/release-notes#3.0.0).
+If you are deploying this solution into a GHES environment then you will need version >= [3.0.0](https://docs.github.com/en/enterprise-server@3.0/admin/release-notes) as a minimum, in order to use all the features of actions-runner-controller >= [3.3.0](https://docs.github.com/en/enterprise-server@3.3/admin/release-notes) is required.
 
-When deploying the solution for a GitHub Enterprise Server environment you need to provide an additional environment variable as part of the controller deployment:
+When deploying the solution for a GHES environment you need to provide an additional environment variable as part of the controller deployment:
 
 ```shell
 kubectl set env deploy controller-manager -c manager GITHUB_ENTERPRISE_URL=<GHEC/S URL> --namespace actions-runner-system
@@ -89,7 +90,7 @@ There are two ways for actions-runner-controller to authenticate with the GitHub
 
 Functionality wise, there isn't much of a difference between the 2 authentication methods. The primarily benefit of authenticating via a GitHub App is an [increased API quota](https://docs.github.com/en/developers/apps/rate-limits-for-github-apps).
 
-If you are deploying the solution for a GitHub Enterprise Server environment you are able to [configure your rate limit settings](https://docs.github.com/en/enterprise-server@3.0/admin/configuration/configuring-rate-limits) making the main benefit irrelevant. If you're deploying the solution for a GitHub Enterprise Cloud or regular GitHub environment and you run into rate limit issues, consider deploying the solution using the GitHub App authentication method instead.
+If you are deploying the solution for a GHES environment you are able to [configure your rate limit settings](https://docs.github.com/en/enterprise-server@3.0/admin/configuration/configuring-rate-limits) making the main benefit irrelevant. If you're deploying the solution for a GHEC or regular GitHub environment and you run into rate limit issues, consider deploying the solution using the GitHub App authentication method instead.
 
 ### Deploying Using GitHub App Authentication
 
@@ -360,10 +361,12 @@ example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 
 ### Autoscaling
 
-> Since the release of GitHub's [`workflow_job` webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job), webhook driven scaling is the preferred way of autoscaling as it enables targeted scaling of your `RunnerDeployments` / `RunnerSets` as it includes the `runs-on` information needed to scale the appropriate runners for that workflow run. More broadly, webhook driven scaling is the preferred scaling option as it is far quicker compared to the pull driven scaling and is easy to setup.
+> Since the release of GitHub's [`workflow_job` webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job), webhook driven scaling is the preferred way of autoscaling as it enables targeted scaling of your `RunnerDeployment` / `RunnerSet` as it includes the `runs-on` information needed to scale the appropriate runners for that workflow run. More broadly, webhook driven scaling is the preferred scaling option as it is far quicker compared to the pull driven scaling and is easy to setup.
 
 A `RunnerDeployment` or `RunnerSet` (see [stateful runners](#stateful-runners) for more details on this kind) can scale the number of runners between `minReplicas` and `maxReplicas` fields driven by either pull based scaling metrics or via a webhook event (see limitations section of [stateful runners](#stateful-runners) for cavaets of this kind). Whether the autoscaling is driven from a webhook event or pull based metrics it is implemented by backing a `RunnerDeployment` or `RunnerSet` kind with a `HorizontalRunnerAutoscaler` kind.
 
+**_Important!!! If you opt to configure autoscaling, ensure you remove the `replicas:` attribute in the `RunnerDeployment` / `RunnerSet` kinds that are configured for autoscaling [#206](https://github.com/actions-runner-controller/actions-runner-controller/issues/206#issuecomment-748601907)_**
+
 #### Anti-Flapping Configuration
 
 For both pull driven or webhook driven scaling an anti-flapping implementation is included, by default a runner won't be scaled down within 10 minutes of it having been scaled up. This delay is configurable by including the attribute `scaleDownDelaySecondsAfterScaleOut:` in a `HorizontalRunnerAutoscaler` kind's `spec:`.
@@ -387,7 +390,8 @@ kind: HorizontalRunnerAutoscaler
 metadata:
   name: example-runner-deployment-autoscaler
 spec:
-  # Runners in the targeted RunnerDeployment won't be scaled down for 5 minutes instead of the default 10 minutes now
+  # Runners in the targeted RunnerDeployment won't be scaled down
+  # for 5 minutes instead of the default 10 minutes now
   scaleDownDelaySecondsAfterScaleOut: 300
   scaleTargetRef:
     name: example-runner-deployment
@@ -441,8 +445,6 @@ The `TotalNumberOfQueuedAndInProgressWorkflowRuns` metric polls GitHub for all p
 
 Example `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`:
 
-**_Important!!! We no longer include the attribute `replicas` in our `RunnerDeployment` if we are configuring autoscaling!_**
-
 ```yaml
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
@@ -484,8 +486,6 @@ The `HorizontalRunnerAutoscaler` will poll GitHub for the number of runners in t
 
 Examples of each scaling type implemented with a `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`:
 
-**_Important!!! We no longer include the attribute `replicas` in our `RunnerDeployment` if we are configuring autoscaling!_**
-
 ```yaml
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
@@ -588,6 +588,8 @@ by learning the following configuration examples.
 
 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
 
+_Note: GitHub does not include the runner group information of a repository in the payload of `workflow_job` event in the initial `queued` event. The runner group information is only include for `workflow_job` events when the job has already been allocated to a runner (events with a status of `in_progress` or `completed`). Please do raise feature requests against [GitHub](https://support.github.com/tickets/personal/0) for this information to be included in the initial `queued` event if this would improve autoscaling runners for you._
+
 The most flexible webhook GitHub offers is the `workflow_job` webhook, it includes the `runs-on` information in the payload allowing scaling based on runner labels.
 
 This webhook should cover most people's needs, please experiment with this webhook first before considering the others.
@@ -717,6 +719,7 @@ spec:
     amount: 1
     duration: "5m"
 ```
+
 #### Autoscaling to/from 0
 
 > This feature requires controller version => [v0.19.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.19.0)
@@ -1113,7 +1116,6 @@ kind: RunnerSet
 metadata:
   name: example
 spec:
-  # NOTE: RunnerSet supports non-ephemeral runners only today
   ephemeral: false
   replicas: 2
   repository: mumoshu/actions-runner-controller-ci
@@ -1159,7 +1161,7 @@ We envision that `RunnerSet` will eventually replace `RunnerDeployment`, as `Run
 **Limitations**
 
 * For autoscaling the `RunnerSet` kind only supports pull driven scaling or the `workflow_job` event for webhook driven scaling.
-* For autoscaling the `RunnerSet` kind doesn't support the [registration-only runner](#autoscaling-tofrom-0)
+* For autoscaling the `RunnerSet` kind doesn't support the [registration-only runner](#autoscaling-tofrom-0), these are deprecated however and to be [removed](https://github.com/actions-runner-controller/actions-runner-controller/issues/859)
 * A known down-side of relying on `StatefulSet` is that it misses a support for `maxUnavailable`. A `StatefulSet` basically works like `maxUnavailable: 1` in `Deployment`, which means that it can take down only one pod concurrently while doing a rolling-update of pods. Kubernetes 1.22 doesn't support customizing it yet so probably it takes more releases to arrive. See https://github.com/kubernetes/kubernetes/issues/68397 for more information.
 
 ### Ephemeral Runners
@@ -1168,19 +1170,9 @@ Both `RunnerDeployment` and `RunnerSet` has ability to configure `ephemeral: tru
 
 When it is configured, it passes a `--once` flag to every runner.
 
-`--once` is an experimental `actions/runner` feature that instructs the runner to stop after the first job run. But it is a known race issue that may fetch a job even when it's being terminated. If a runner fetched a job while terminating, the job is very likely to fail because the terminating runner doesn't wait for the job to complete. This is tracked in #466.
+`--once` is an experimental `actions/runner` feature that instructs the runner to stop after the first job run. It has a known race condition issue that means the runner may fetch a job even when it's being terminated. If a runner fetched a job while terminating, the job is very likely to fail because the terminating runner doesn't wait for the job to complete. This is tracked in issue [#466](https://github.com/actions-runner-controller/actions-runner-controller/issues/466).
 
-> The below feature depends on an unreleased GitHub feature
-
-GitHub seems to be adding an another flag called `--ephemeral` that is race-free. The pull request to add it to `actions/runner` can be found at https://github.com/actions/runner/pull/660.
-
-`actions-runner-controller` has a feature flag backend by an environment variable to enable using `--ephemeral` instead of `--once`. The environment variable is `RUNNER_FEATURE_FLAG_EPHEMERAL`. You can se it to `true` on runner containers in your runner pods to enable the feature.
-
-> At the time of writing this, you need to wait until GitHub rolls out the server-side feature for `--ephemeral`, AND you need to include your own `actions/runner` binary built from https://github.com/actions/runner/pull/660 into the runner container image to test this feature.
->
-> Please see comments in [`runner/Dockerfile`](/runner/Dockerfile) for more information about how to build a custom image using your own `actions/runner` binary.
-
-For example, a `RunnerSet` config with the flag enabled looks like:
+Since the implementation of the `--once` flag GitHub have implemented the `--ephemeral` flag which has no known race conditions and is much more supported by GitHub, this is the prefered flag for ephemeral runners. To have your `RunnerDeployment` and `RunnerSet` kinds use this new flag instead of the `--once` flag set `RUNNER_FEATURE_FLAG_EPHEMERAL` to `"true"`. For example, a `RunnerSet` configured to use the new flag looks like:
 
 ```yaml
 kind: RunnerSet
@@ -1201,9 +1193,9 @@ spec:
           value: "true"
 ```
 
-Note that once https://github.com/actions/runner/pull/660 becomes generally available on GitHub, you no longer need to build a custom runner image to use this feature. Just set `RUNNER_FEATURE_FLAG_EPHEMERAL` and it should use `--ephemeral`.
+You should configure all your ephemeral runners to use the new flag unless you have a reason for needing to use the old flag.
 
-In the future, `--once` might get removed in `actions/runner`. `actions-runner-controller` will make `--ephemeral` the default option for `ephemeral: true` runners until the legacy flag is removed.
+Once able, `actions-runner-controller` will make `--ephemeral` the default option for `ephemeral: true` runners and potentially remove `--once` entirely. It is likely that in the future the `--once` flag will be officially deprecated by GitHub and subsquently removed in `actions/runner`.
 
 ### Software Installed in the Runner Image
 
@@ -1213,7 +1205,7 @@ The project supports being deployed on the various cloud Kubernetes platforms (e
 **Bundled Software**<br />
 The GitHub hosted runners include a large amount of pre-installed software packages. GitHub maintain a list in README files at <https://github.com/actions/virtual-environments/tree/main/images/linux>
 
-This solution maintains a few runner images with `latest` aligning with GitHub's Ubuntu version. Older images are maintained whilst GitHub also provides them as an option. These images do not contain all of the software installed on the GitHub runners. It contains the following subset of packages from the GitHub runners:
+This solution maintains a few runner images with `latest` aligning with GitHub's Ubuntu version, these images do not contain all of the software installed on the GitHub runners. The images contain the following subset of packages from the GitHub runners:
 
 - Basic CLI packages
 - git
@@ -1244,6 +1236,35 @@ spec:
   image: YOUR_CUSTOM_DOCKER_IMAGE
 ```
 
+### Using without cert-manager
+
+Assuming you are installing in the default namespace, ensure your certificate has SANs:
+
+* `webhook-service.actions-runner-system.svc`
+* `webhook-service.actions-runner-system.svc.cluster.local`
+
+It is possible to use a self-signed certificate by following a guide like
+[this one](https://mariadb.com/docs/security/encryption/in-transit/create-self-signed-certificates-keys-openssl/)
+using `openssl`.
+
+Install your certificate as a TLS secret:
+
+```shell
+$ kubectl create secret tls webhook-server-cert \
+  -n actions-runner-system \
+  --cert=path/to/cert/file \
+  --key=path/to/key/file
+```
+
+Set the Helm chart values as follows:
+
+```shell
+$ CA_BUNDLE=$(cat path/to/ca.pem | base64)
+$ helm --upgrade install actions-runner-controller/actions-runner-controller \
+  certManagerEnabled=false \
+  admissionWebHooks.caBundle=${CA_BUNDLE}
+```
+
 ### Common Errors
 
 #### invalid header field value

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.14.0
+version: 0.15.1
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.20.2
+appVersion: 0.20.3
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/README.md

@@ -18,6 +18,8 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `githubAPICacheDuration`                                 | Set the cache period for API calls                                                                                         |                                                                      |
 | `githubEnterpriseServerURL`                              | Set the URL for a self-hosted GitHub Enterprise Server                                                                     |                                                                      |
 | `logLevel`                                               | Set the log level of the controller container                                                                              |                                                                      |
+| `additionalVolumes`                                      | Set additional volumes to add to the manager container                                                                     |                                                                      |
+| `additionalVolumeMounts`                                 | Set additional volume mounts to add to the manager container                                                               |                                                                      |
 | `authSecret.create`                                      | Deploy the controller auth secret                                                                                          | false                                                                |
 | `authSecret.name`                                        | Set the name of the auth secret                                                                                            | controller-manager                                                   |
 | `authSecret.annotations`                                 | Set annotations for the auth Secret                                                                                        |                                                                      |
@@ -54,7 +56,10 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `topologySpreadConstraints`                              | Set the controller pod topologySpreadConstraints                                                                           |                                                                      |
 | `nodeSelector`                                           | Set the controller pod nodeSelector                                                                                        |                                                                      |
 | `resources`                                              | Set the controller pod resources                                                                                           |                                                                      |
-| `affinity`                                               | Set the controller pod affinity rules                                                                                      |                                                                      |
+| `affinity`                                               | Set the controller pod affinity rules                                                                                      |
+| `podDisruptionBudget.enabled`                                               | Enables a PDB to ensure HA of controller pods                                                                                     |      false                                                                    |
+| `podDisruptionBudget.minAvailable`                                               | Minimum number of pods that must be available after eviction                                                                                     |                                                                          |
+| `podDisruptionBudget.maxUnavailable`                                               | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                                                                     |                                                                          |
 | `tolerations`                                            | Set the controller pod tolerations                                                                                         |                                                                      |
 | `env`                                                    | Set environment variables for the controller container                                                                     |                                                                      |
 | `priorityClassName`                                      | Set the controller pod priorityClassName                                                                                   |                                                                      |
@@ -89,3 +94,8 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `githubWebhookServer.ingress.annotations`                | Set annotations for the ingress kind                                                                                       |                                                                      |
 | `githubWebhookServer.ingress.hosts`                      | Set hosts configuration for ingress                                                                                        | `[{"host": "chart-example.local", "paths": []}]`                     |
 | `githubWebhookServer.ingress.tls`                        | Set tls configuration for ingress                                                                                          |                                                                      |
+| `githubWebhookServer.podDisruptionBudget.enabled`        | Enables a PDB to ensure HA of githubwebhook pods                                                                           |      false                                                           |
+| `githubWebhookServer.podDisruptionBudget.minAvailable`   | Minimum number of pods that must be available after eviction                                                               |                                                                      |
+| `githubWebhookServer.podDisruptionBudget.maxUnavailable` | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                      |
+| `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                   | true                                                                 |
+| `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                  |                                                                      |

charts/actions-runner-controller/templates/_github_webhook_server_helpers.tpl

@@ -58,3 +58,7 @@ Create the name of the service account to use
 {{- define "actions-runner-controller-github-webhook-server.serviceMonitorName" -}}
 {{- include "actions-runner-controller-github-webhook-server.fullname" . | trunc 47 }}-service-monitor
 {{- end }}
+
+{{- define "actions-runner-controller-github-webhook-server.pdbName" -}}
+{{- include "actions-runner-controller-github-webhook-server.fullname" . | trunc 59 }}-pdb
+{{- end }}

charts/actions-runner-controller/templates/_helpers.tpl

@@ -107,3 +107,7 @@ Create the name of the service account to use
 {{- define "actions-runner-controller.servingCertName" -}}
 {{- include "actions-runner-controller.fullname" . }}-serving-cert
 {{- end }}
+
+{{- define "actions-runner-controller.pdbName" -}}
+{{- include "actions-runner-controller.fullname" . | trunc 59 }}-pdb
+{{- end }}

charts/actions-runner-controller/templates/certificate.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.certManagerEnabled }}
 # The following manifests contain a self-signed issuer CR and a certificate CR.
 # More document can be found at https://docs.cert-manager.io
 # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
@@ -22,3 +23,4 @@ spec:
     kind: Issuer
     name: {{ include "actions-runner-controller.selfsignedIssuerName" . }}
   secretName: {{ include "actions-runner-controller.servingCertName" . }}
+{{- end }}

charts/actions-runner-controller/templates/controller.pdb.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.podDisruptionBudget.enabled }}
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  name: {{ include "actions-runner-controller.pdbName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  {{- if .Values.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.podDisruptionBudget.minAvailable }}
+  {{- end  }}
+  {{- if .Values.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }}
+  {{- end  }}
+  selector:
+    matchLabels:
+      {{- include "actions-runner-controller.selectorLabels" . | nindent 6 }}
+{{- end -}}

charts/actions-runner-controller/templates/deployment.yaml

@@ -114,6 +114,9 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
+        {{- if .Values.additionalVolumeMounts }}
+          {{- toYaml .Values.additionalVolumeMounts | nindent 8 }} 
+        {{- end }}
       {{- if .Values.metrics.proxy.enabled }}
       - args:
         - "--secure-listen-address=0.0.0.0:{{ .Values.metrics.port }}"
@@ -142,6 +145,9 @@ spec:
           secretName: {{ include "actions-runner-controller.servingCertName" . }}
       - name: tmp
         emptyDir: {}
+      {{- if .Values.additionalVolumes }}
+        {{- toYaml .Values.additionalVolumes | nindent 6}}
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/actions-runner-controller/templates/githubwebhook.pdb.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.githubWebhookServer.podDisruptionBudget.enabled }}
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  name: {{ include "actions-runner-controller-github-webhook-server.pdbName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  {{- if .Values.githubWebhookServer.podDisruptionBudget.minAvailable }}
+  minAvailable: {{ .Values.githubWebhookServer.podDisruptionBudget.minAvailable }}
+  {{- end  }}
+  {{- if .Values.githubWebhookServer.podDisruptionBudget.maxUnavailable }}
+  maxUnavailable: {{ .Values.githubWebhookServer.podDisruptionBudget.maxUnavailable }}
+  {{- end  }}
+  selector:
+    matchLabels:
+      {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 6 }}
+{{- end -}}

charts/actions-runner-controller/templates/webhook_configs.yaml

@@ -5,12 +5,17 @@ kind: MutatingWebhookConfiguration
 metadata:
   creationTimestamp: null
   name: {{ include "actions-runner-controller.fullname" . }}-mutating-webhook-configuration
+  {{- if .Values.certManagerEnabled }}
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "actions-runner-controller.servingCertName" . }}
+  {{- end }}
 webhooks:
 - admissionReviewVersions:
   - v1beta1
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -31,6 +36,9 @@ webhooks:
 - admissionReviewVersions:
   - v1beta1
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -51,6 +59,9 @@ webhooks:
 - admissionReviewVersions:
   - v1beta1
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -71,6 +82,9 @@ webhooks:
 - admissionReviewVersions:
   - v1beta1
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -96,12 +110,17 @@ kind: ValidatingWebhookConfiguration
 metadata:
   creationTimestamp: null
   name: {{ include "actions-runner-controller.fullname" . }}-validating-webhook-configuration
+  {{- if .Values.certManagerEnabled }}
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "actions-runner-controller.servingCertName" . }}
+  {{- end }}
 webhooks:
 - admissionReviewVersions:
   - v1beta1
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -122,6 +141,9 @@ webhooks:
 - admissionReviewVersions:
   - v1beta1
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -142,6 +164,9 @@ webhooks:
 - admissionReviewVersions:
   - v1beta1
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}

charts/actions-runner-controller/values.yaml

@@ -88,7 +88,7 @@ metrics:
     enabled: true
     image:
       repository: quay.io/brancz/kube-rbac-proxy
-      tag: v0.10.0
+      tag: v0.11.0
 
 resources:
   {}
@@ -109,6 +109,12 @@ tolerations: []
 
 affinity: {}
 
+# Only one of minAvailable or maxUnavailable can be set
+podDisruptionBudget:
+  enabled: false
+  # minAvailable: 1
+  # maxUnavailable: 3
+
 # Leverage a PriorityClass to ensure your pods survive resource shortages
 # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
 # PriorityClass: system-cluster-critical
@@ -120,6 +126,14 @@ env:
   # https_proxy: "proxy.com:8080"
   # no_proxy: ""
 
+## specify additional volumes to mount in the manager container, this can be used
+## to specify additional storage of material or to inject files from ConfigMaps
+## into the running container
+additionalVolumes: []
+
+## specify where the additional volumes are mounted in the manager container
+additionalVolumeMounts: []
+
 scope:
   # If true, the controller will only watch custom resources in a single namespace
   singleNamespace: false
@@ -179,3 +193,15 @@ githubWebhookServer:
     #  - secretName: chart-example-tls
     #    hosts:
     #      - chart-example.local
+
+  # Only one of minAvailable or maxUnavailable can be set
+  podDisruptionBudget:
+    enabled: false
+    # minAvailable: 1
+    # maxUnavailable: 3
+
+certManagerEnabled: true
+
+admissionWebHooks:
+  {}
+  #caBundle: "Ci0tLS0tQk...<base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate>...tLS0K"

controllers/runner_controller.go

@@ -422,16 +422,26 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 						"configuredRegistrationTimeout", registrationTimeout,
 					)
 				} else if registrationDidTimeout {
-					log.Info(
-						"Already existing GitHub runner still appears offline . "+
-							"Recreating the pod to see if it resolves the issue. "+
-							"CAUTION: If you see this a lot, you should investigate the root cause. ",
-						"podCreationTimestamp", pod.CreationTimestamp,
-						"currentTime", currentTime,
-						"configuredRegistrationTimeout", registrationTimeout,
-					)
+					if runnerBusy {
+						log.Info(
+							"Timeout out while waiting for the runner to be online, but observed that it's busy at the same time."+
+								"This is a known (unintuitive) behaviour of a runner that is already running a job. Please see https://github.com/actions-runner-controller/actions-runner-controller/issues/911",
+							"podCreationTimestamp", pod.CreationTimestamp,
+							"currentTime", currentTime,
+							"configuredRegistrationTimeout", registrationTimeout,
+						)
+					} else {
+						log.Info(
+							"Already existing GitHub runner still appears offline . "+
+								"Recreating the pod to see if it resolves the issue. "+
+								"CAUTION: If you see this a lot, you should investigate the root cause. ",
+							"podCreationTimestamp", pod.CreationTimestamp,
+							"currentTime", currentTime,
+							"configuredRegistrationTimeout", registrationTimeout,
+						)
 
-					restart = true
+						restart = true
+					}
 				} else {
 					log.V(1).Info(
 						"Runner pod exists but the GitHub runner appears to be still offline. Waiting for runner to get online ...",

runner/Dockerfile.dindrunner

@@ -3,7 +3,7 @@ FROM ubuntu:20.04
 ARG TARGETPLATFORM
 ARG RUNNER_VERSION=2.280.3
 ARG DOCKER_CHANNEL=stable
-ARG DOCKER_VERSION=19.03.13
+ARG DOCKER_VERSION=20.10.8
 ARG DUMB_INIT_VERSION=1.2.5
 
 RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)