chore: bump chart to latest (#1280)

* fix(chart): add namespace selector to webhooks when in singleNamespace mode

* docs: expand multi controller setup

Co-authored-by: Callum Tait <15716903+toast-gear@users.noreply.github.com>

.dockerignore

@@ -11,3 +11,4 @@ charts
 *.md
 *.txt
 *.sh
+test/e2e/.docker-build

.github/actions/setup-docker-environment/action.yaml

@@ -0,0 +1,52 @@
+name: "Setup Docker"
+
+inputs:
+  username:
+    description: "Username"
+    required: true
+  password:
+    description: "Password"
+    required: true
+  ghcr_username:
+    description: "GHCR username. Usually set from the github.actor variable"
+    required: true
+  ghcr_password:
+    description: "GHCR password. Usually set from the secrets.GITHUB_TOKEN variable"
+    required: true
+
+outputs:
+  sha_short:
+    description: "The short SHA used for image builds"
+    value: ${{ steps.vars.outputs.sha_short }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get Short SHA
+      id: vars
+      run: |
+        echo ::set-output name=sha_short::${GITHUB_SHA::7}
+      shell: bash
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
+
+    - name: Login to DockerHub
+      if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+      uses: docker/login-action@v1
+      with:
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v1
+      if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.ghcr_username }}
+        password: ${{ inputs.ghcr_password }}

.github/renovate.json5

@@ -12,10 +12,12 @@
   "regexManagers": [
     {
       // use https://github.com/actions/runner/releases
-      "fileMatch": [".github/workflows/build-and-release-runners.yml"],
+      "fileMatch": [
+        ".github/workflows/runners.yml"
+        ],
       "matchStrings": ["RUNNER_VERSION: +(?<currentValue>.*?)\\n"],
       "depNameTemplate": "actions/runner",
       "datasourceTemplate": "github-releases"
     }
   ]
-}
+}

.github/workflows/build-and-release-runners.yml

@@ -1,123 +0,0 @@
-name: Build and Release Runners
-
-on:
-  pull_request:
-    branches:
-      - '**'
-    paths:
-      - 'runner/**'
-      - .github/workflows/build-and-release-runners.yml
-      - '!**.md'
-  push:
-    branches:
-      - master
-    paths:
-      - runner/patched/*
-      - runner/Dockerfile
-      - runner/Dockerfile.ubuntu.1804
-      - runner/Dockerfile.dindrunner
-      - runner/entrypoint.sh
-      - .github/workflows/build-and-release-runners.yml
-      - '!**.md'
-
-env:
-  RUNNER_VERSION: 2.284.0
-  DOCKER_VERSION: 20.10.8
-  DOCKERHUB_USERNAME: summerwind
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    name: Build ${{ matrix.name }}-ubuntu-${{ matrix.os-version }}
-    strategy:
-      matrix:
-        include:
-          - name: actions-runner
-            os-version: 20.04
-            dockerfile: Dockerfile
-          - name: actions-runner
-            os-version: 18.04
-            dockerfile: Dockerfile.ubuntu.1804
-          - name: actions-runner-dind
-            os-version: 20.04
-            dockerfile: Dockerfile.dindrunner
-
-    steps:
-      - name: Set outputs
-        id: vars
-        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
-
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        if: ${{ github.event_name == 'push' || github.event_name == 'release' }}
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
-      - name: Build and Push Versioned Tags
-        uses: docker/build-push-action@v2
-        with:
-          context: ./runner
-          file: ./runner/${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          build-args: |
-            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
-            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-ubuntu-${{ matrix.os-version }}
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-ubuntu-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
-
-  latest-tags:
-    if: ${{ github.event_name == 'push' || github.event_name == 'release' }}
-    runs-on: ubuntu-latest
-    name: Build ${{ matrix.name }}-latest
-    strategy:
-      matrix:
-        include:
-          - name: actions-runner
-            dockerfile: Dockerfile
-          - name: actions-runner-dind
-            dockerfile: Dockerfile.dindrunner
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
-      - name: Build and Push Latest Tag
-        uses: docker/build-push-action@v2
-        with:
-          context: ./runner
-          file: ./runner/${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
-            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:latest

.github/workflows/on-push-lint-charts.yml

@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
 env:
   KUBE_SCORE_VERSION: 1.10.0
-  HELM_VERSION: v3.4.1
+  HELM_VERSION: v3.8.0
 
 jobs:
   lint-test:
@@ -18,12 +18,12 @@ jobs:
     name: Lint Chart
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v1
+        uses: azure/setup-helm@v2.0
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -44,12 +44,12 @@ jobs:
               --enable-optional-test container-security-context-readonlyrootfilesystem
 
       # python is a requirement for the chart-testing action below (supports yamllint among other tests)
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
           python-version: 3.7
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.1.0
+        uses: helm/chart-testing-action@v2.2.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed

.github/workflows/on-push-master-publish-chart.yml

@@ -13,7 +13,7 @@ on:
 
 env:
   KUBE_SCORE_VERSION: 1.10.0
-  HELM_VERSION: v3.4.1
+  HELM_VERSION: v3.8.0
 
 jobs:
   lint-chart:
@@ -23,12 +23,12 @@ jobs:
       publish-chart: ${{ steps.publish-chart-step.outputs.publish }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v1
+        uses: azure/setup-helm@v2.0
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -49,12 +49,12 @@ jobs:
               --enable-optional-test container-security-context-readonlyrootfilesystem
 
       # python is a requirement for the chart-testing action below (supports yamllint among other tests)
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
           python-version: 3.7
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.1.0
+        uses: helm/chart-testing-action@v2.2.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -104,7 +104,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
@@ -114,7 +114,7 @@ jobs:
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
 
       - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.2.1
+        uses: helm/chart-releaser-action@v1.3.0
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/release.yml

@@ -16,7 +16,11 @@ jobs:
         run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '^1.17.7'
 
       - name: Install tools
         run: |

.github/workflows/runners.yml

@@ -0,0 +1,73 @@
+name: Runners
+
+on:
+  pull_request:
+    types: 
+      - opened
+      - synchronize
+      - reopened
+      - closed
+    branches:
+      - 'master'
+    paths:
+      - 'runner/**'
+      - .github/workflows/runners.yml
+      - '!**.md'
+
+env:
+  RUNNER_VERSION: 2.289.1
+  DOCKER_VERSION: 20.10.12
+  DOCKERHUB_USERNAME: summerwind
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    name: Build ${{ matrix.name }}-${{ matrix.os-name }}-${{ matrix.os-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: actions-runner
+            os-name: ubuntu
+            os-version: 20.04
+            dockerfile: Dockerfile
+          - name: actions-runner-dind
+            os-name: ubuntu
+            os-version: 20.04
+            dockerfile: Dockerfile.dindrunner
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Docker Environment
+        id: vars
+        uses: ./.github/actions/setup-docker-environment
+        with: 
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+          ghcr_username: ${{ github.actor }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and Push Versioned Tags
+        uses: docker/build-push-action@v2
+        with:
+          context: ./runner
+          file: ./runner/${{ matrix.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+          build-args: |
+            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
+            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:latest
+            ghcr.io/${{ github.repository }}/${{ matrix.name }}:latest
+            ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}
+            ghcr.io/${{ github.repository }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ matrix.os-name }}-${{ matrix.os-version }}-${{ steps.vars.outputs.sha_short }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/test-entrypoint.yaml

@@ -15,8 +15,7 @@ jobs:
     name: Test entrypoint
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Run unit tests for entrypoint.sh
       run: |
-        cd test/entrypoint
-        bash entrypoint_unittest.sh
+        make acceptance/runner/entrypoint

.github/workflows/test.yaml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
     paths-ignore:
-      - .github/workflows/build-and-release-runners.yml
+      - .github/workflows/runners.yml
       - .github/workflows/on-push-lint-charts.yml
       - .github/workflows/on-push-master-publish-chart.yml
       - .github/workflows/release.yml
@@ -21,11 +21,17 @@ jobs:
     name: Test
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
-    - uses: actions/setup-go@v2
+      uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
       with:
-        go-version: '^1.16.5'
+        go-version: '^1.17.7'
     - run: go version
+    - uses: actions/cache@v3
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
     - name: Install kubebuilder
       run: |
         curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_linux_amd64.tar.gz

.github/workflows/wip.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
     paths-ignore:
-      - .github/workflows/build-and-release-runners.yml
+      - .github/workflows/runners.yml
       - .github/workflows/on-push-lint-charts.yml
       - .github/workflows/on-push-master-publish-chart.yml
       - .github/workflows/release.yml
@@ -22,7 +22,7 @@ jobs:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USER }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

CONTRIBUTING.md

@@ -95,6 +95,7 @@ To make your development cycle faster, use the below command to update deploy an
 # you either need to bump VERSION and RUNNER_TAG on each run,
 # or manually run `kubectl delete pod $POD` on respective pods for changes to actually take effect.
 
+# Makefile
 VERSION=controller1 \
   RUNNER_TAG=runner1 \
   make acceptance/pull acceptance/kind docker-build acceptance/load acceptance/deploy
@@ -103,14 +104,16 @@ VERSION=controller1 \
 If you've already deployed actions-runner-controller and only want to recreate pods to use the newer image, you can run:
 
 ```shell
+# Makefile
 NAME=$DOCKER_USER/actions-runner-controller \
   make docker-build acceptance/load && \
   kubectl -n actions-runner-system delete po $(kubectl -n actions-runner-system get po -ojsonpath={.items[*].metadata.name})
 ```
 
-Similarly, if you'd like to recreate runner pods with the newer runner image,
+Similarly, if you'd like to recreate runner pods with the newer runner image you can use the runner specific [Makefile](runner/Makefile) to build and / or push new runner images
 
 ```shell
+# runner/Makefile
 NAME=$DOCKER_USER/actions-runner make \
   -C runner docker-{build,push}-ubuntu && \
   (kubectl get po -ojsonpath={.items[*].metadata.name} | xargs -n1 kubectl delete po)

Dockerfile

@@ -1,29 +1,44 @@
 # Build the manager binary
-FROM golang:1.17 as builder
-
-ARG TARGETPLATFORM
+FROM --platform=$BUILDPLATFORM golang:1.17 as builder
 
 WORKDIR /workspace
 
-ENV GO111MODULE=on \
-  CGO_ENABLED=0
+# Make it runnable on a distroless image/without libc
+ENV CGO_ENABLED=0
 
 # Copy the Go Modules manifests
 COPY go.mod go.sum ./
 
 # cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
+# and so that source changes don't invalidate our downloaded layer.
+#
+# Also, we need to do this before setting TARGETPLATFORM/TARGETOS/TARGETARCH/TARGETVARIANT
+# so that go mod cache is shared across platforms.
 RUN go mod download
 
 # Copy the go source
-COPY . .
+# COPY . .
+
+# Usage:
+#   docker buildx build --tag repo/img:tag -f ./Dockerfile . --platform linux/amd64,linux/arm64,linux/arm/v7
+#
+# With the above commmand,
+# TARGETOS can be "linux", TARGETARCH can be "amd64", "arm64", and "arm", TARGETVARIANT can be "v7".
+
+ARG TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT
+
+# We intentionally avoid `--mount=type=cache,mode=0777,target=/go/pkg/mod` in the `go mod download` and the `go build` runs
+# to avoid https://github.com/moby/buildkit/issues/2334
+# We can use docker layer cache so the build is fast enogh anyway
+# We also use per-platform GOCACHE for the same reason.
+env GOCACHE /build/${TARGETPLATFORM}/root/.cache/go-build
 
 # Build
-RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
-  export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
-  GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \
-  go build -a -o manager main.go && \
-  go build -a -o github-webhook-server ./cmd/githubwebhookserver
+RUN --mount=target=. \
+  --mount=type=cache,mode=0777,target=${GOCACHE} \
+  export GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOARM=${TARGETVARIANT#v} && \
+  go build -o /out/manager main.go && \
+  go build -o /out/github-webhook-server ./cmd/githubwebhookserver
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -31,8 +46,8 @@ FROM gcr.io/distroless/static:nonroot
 
 WORKDIR /
 
-COPY --from=builder /workspace/manager .
-COPY --from=builder /workspace/github-webhook-server .
+COPY --from=builder /out/manager .
+COPY --from=builder /out/github-webhook-server .
 
 USER nonroot:nonroot
 

Makefile

@@ -5,6 +5,7 @@ else
 endif
 DOCKER_USER ?= $(shell echo ${NAME} | cut -d / -f1)
 VERSION ?= latest
+TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}
 TEST_REPO ?= ${DOCKER_USER}/actions-runner-controller
@@ -17,9 +18,10 @@ RUNNER_FEATURE_FLAG_EPHEMERAL ?=
 KUBECONTEXT ?= kind-acceptance
 CLUSTER ?= acceptance
 CERT_MANAGER_VERSION ?= v1.1.1
+KUBE_RBAC_PROXY_VERSION ?= v0.11.0
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true,generateEmbeddedObjectMeta=true"
+CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=true"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -110,7 +112,7 @@ generate: controller-gen
 # Build the docker image
 docker-build:
 	docker build -t ${NAME}:${VERSION} .
-	docker build -t ${RUNNER_NAME}:${RUNNER_TAG} --build-arg TARGETPLATFORM=$(shell arch) runner
+	docker build -t ${RUNNER_NAME}:${RUNNER_TAG} --build-arg TARGETPLATFORM=${TARGETPLATFORM} runner
 
 docker-buildx:
 	export DOCKER_CLI_EXPERIMENTAL=enabled
@@ -156,7 +158,7 @@ acceptance/kind:
 # See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap
 acceptance/load:
 	kind load docker-image ${NAME}:${VERSION} --name ${CLUSTER}
-	kind load docker-image quay.io/brancz/kube-rbac-proxy:v0.10.0 --name ${CLUSTER}
+	kind load docker-image quay.io/brancz/kube-rbac-proxy:$(KUBE_RBAC_PROXY_VERSION) --name ${CLUSTER}
 	kind load docker-image ${RUNNER_NAME}:${RUNNER_TAG} --name ${CLUSTER}
 	kind load docker-image docker:dind --name ${CLUSTER}
 	kind load docker-image quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION) --name ${CLUSTER}
@@ -166,7 +168,7 @@ acceptance/load:
 
 # Pull the docker images for acceptance
 acceptance/pull:
-	docker pull quay.io/brancz/kube-rbac-proxy:v0.10.0
+	docker pull quay.io/brancz/kube-rbac-proxy:$(KUBE_RBAC_PROXY_VERSION)
 	docker pull docker:dind
 	docker pull quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION)
 	docker pull quay.io/jetstack/cert-manager-cainjector:$(CERT_MANAGER_VERSION)
@@ -195,6 +197,9 @@ acceptance/deploy:
 acceptance/tests:
 	acceptance/checks.sh
 
+acceptance/runner/entrypoint:
+	cd test/entrypoint/ && bash test.sh
+
 # We use -count=1 instead of `go clean -testcache`
 # See https://terratest.gruntwork.io/docs/testing-best-practices/avoid-test-caching/
 .PHONY: e2e
@@ -221,7 +226,7 @@ ifeq (, $(wildcard $(GOBIN)/controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.0 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 endif

README.md

@@ -18,6 +18,7 @@ ToC:
   - [Organization Runners](#organization-runners)
   - [Enterprise Runners](#enterprise-runners)
   - [RunnerDeployments](#runnerdeployments)
+  - [RunnerSets](#runnersets)
   - [Autoscaling](#autoscaling)
     - [Anti-Flapping Configuration](#anti-flapping-configuration)
     - [Pull Driven Scaling](#pull-driven-scaling)
@@ -26,13 +27,15 @@ ToC:
     - [Scheduled Overrides](#scheduled-overrides)
   - [Runner with DinD](#runner-with-dind)
   - [Additional Tweaks](#additional-tweaks)
+  - [Custom Volume mounts](#custom-volume-mounts)
   - [Runner Labels](#runner-labels)
   - [Runner Groups](#runner-groups)
+  - [Runner Entrypoint Features](#runner-entrypoint-features)
   - [Using IRSA (IAM Roles for Service Accounts) in EKS](#using-irsa-iam-roles-for-service-accounts-in-eks)
-  - [Stateful Runners](#stateful-runners)
-  - [Ephemeral Runners](#ephemeral-runners)
+  - [Persistent Runners](#persistent-runners)
   - [Software Installed in the Runner Image](#software-installed-in-the-runner-image)
-  - [Common Errors](#common-errors)
+  - [Using without cert-manager](#using-without-cert-manager)
+- [Troubleshooting](#troubleshooting)
 - [Contributing](#contributing)
 
 ## Motivation
@@ -43,7 +46,7 @@ ToC:
 
 ## Installation
 
-actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) for certificate management of Admission Webhook. Make sure you have already installed cert-manager before you install. The installation instructions for cert-manager can be found below.
+By default, actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) for certificate management of Admission Webhook. Make sure you have already installed cert-manager before you install. The installation instructions for cert-manager can be found below.
 
 - [Installing cert-manager on Kubernetes](https://cert-manager.io/docs/installation/kubernetes/)
 
@@ -52,8 +55,8 @@ Subsequent to this, install the custom resource definitions and actions-runner-c
 **Kubectl Deployment:**
 
 ```shell
-# REPLACE "v0.20.2" with the version you wish to deploy
-kubectl apply -f https://github.com/actions-runner-controller/actions-runner-controller/releases/download/v0.20.2/actions-runner-controller.yaml
+# REPLACE "v0.22.0" with the version you wish to deploy
+kubectl apply -f https://github.com/actions-runner-controller/actions-runner-controller/releases/download/v0.22.0/actions-runner-controller.yaml
 ```
 
 **Helm Deployment:**
@@ -68,11 +71,11 @@ helm upgrade --install --namespace actions-runner-system --create-namespace \
 
 ### GitHub Enterprise Support
 
-The solution supports both GitHub Enterprise Cloud and Server editions as well as regular GitHub. Both PAT (personal access token) and GitHub App authentication works for installations that will be deploying either repository level and / or organization level runners. If you need to deploy enterprise level runners then you are restricted to PAT based authentication as GitHub doesn't support GitHub App based authentication for enterprise runners currently.
+The solution supports both GHEC (GitHub Enterprise Cloud) and GHES (GitHub Enterprise Server) editions as well as regular GitHub. Both PAT (personal access token) and GitHub App authentication works for installations that will be deploying either repository level and / or organization level runners. If you need to deploy enterprise level runners then you are restricted to PAT based authentication as GitHub doesn't support GitHub App based authentication for enterprise runners currently.
 
-If you are deploying this solution into a GitHub Enterprise Server environment then you will need version >= [3.0.0](https://docs.github.com/en/enterprise-server@3.0/admin/release-notes#3.0.0).
+If you are deploying this solution into a GHES environment then you will need to be running version >= [3.3.0](https://docs.github.com/en/enterprise-server@3.3/admin/release-notes).
 
-When deploying the solution for a GitHub Enterprise Server environment you need to provide an additional environment variable as part of the controller deployment:
+When deploying the solution for a GHES environment you need to provide an additional environment variable as part of the controller deployment:
 
 ```shell
 kubectl set env deploy controller-manager -c manager GITHUB_ENTERPRISE_URL=<GHEC/S URL> --namespace actions-runner-system
@@ -89,7 +92,7 @@ There are two ways for actions-runner-controller to authenticate with the GitHub
 
 Functionality wise, there isn't much of a difference between the 2 authentication methods. The primarily benefit of authenticating via a GitHub App is an [increased API quota](https://docs.github.com/en/developers/apps/rate-limits-for-github-apps).
 
-If you are deploying the solution for a GitHub Enterprise Server environment you are able to [configure your rate limit settings](https://docs.github.com/en/enterprise-server@3.0/admin/configuration/configuring-rate-limits) making the main benefit irrelevant. If you're deploying the solution for a GitHub Enterprise Cloud or regular GitHub environment and you run into rate limit issues, consider deploying the solution using the GitHub App authentication method instead.
+If you are deploying the solution for a GHES environment you are able to [configure your rate limit settings](https://docs.github.com/en/enterprise-server@3.0/admin/configuration/configuring-rate-limits) making the main benefit irrelevant. If you're deploying the solution for a GHEC or regular GitHub environment and you run into rate limit issues, consider deploying the solution using the GitHub App authentication method instead.
 
 ### Deploying Using GitHub App Authentication
 
@@ -223,14 +226,16 @@ By default the controller will look for runners in all namespaces, the watch nam
 
 This feature is configured via the controller's `--watch-namespace` flag. When a namespace is provided via this flag, the controller will only monitor runners in that namespace.
 
-If you plan on installing all instances of the controller stack into a single namespace you will need to make the names of the resources unique to each stack. In the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
+You can deploy multiple controllers either in a single shared namespace, or in a unique namespace per controller.
 
-Alternatively, you can install each controller stack into its own unique namespace (relative to other controller stacks in the cluster), avoiding the need to uniquely prefix resources.
+If you plan on installing all instances of the controller stack into a single namespace there are a few things you need to do for this to work.
 
-When you go to the route of sharing the namespace while giving each a unique Helm release name, you must also ensure the following values are configured correctly:
+1. All resources per stack must have a unique, in the case of Helm this can be done by giving each install a unique release name, or via the `fullnameOverride` properties.
+2. `authSecret.name` needs be unique per stack when each stack is tied to runners in different GitHub organizations and repositories AND you want your GitHub credentials to narrowly scoped.
+3. `leaderElectionId` needs to be unique per stack. If this is not unique to the stack the controller tries to race onto the leader election lock resulting in only one stack working concurrently. Your controller will be stuck with a log message something like this `attempting to acquire leader lease arc-controllers/actions-runner-controller...`
+4. The MutatingWebhookConfiguration in each stack must include a namespace selector for that stacks corresponding runners namespace, this is already configured in the helm chart.
 
-- `authSecret.name` needs be unique per stack when each stack is tied to runners in different GitHub organizations and repositories AND you want your GitHub credentials to narrowly scoped.
-- `leaderElectionId` needs to be unique per stack. If this is not unique to the stack the controller tries to race onto the leader election lock and resulting in only one stack working concurrently.
+Alternatively, you can install each controller stack into a unique namespace (relative to other controller stacks in the cluster), avoiding these potential pitfalls.
 
 ## Usage
 
@@ -358,11 +363,103 @@ example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
 example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 ```
 
+  ### RunnerSets
+
+> This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
+
+For scenarios where you require the advantages of a `StatefulSet`, for example persistent storage, ARC implements a runner based on Kubernete's StatefulSets, the RunnerSet.
+
+A basic `RunnerSet` would look like this:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerSet
+metadata:
+  name: example
+spec:
+  ephemeral: false
+  replicas: 2
+  repository: mumoshu/actions-runner-controller-ci
+  # Other mandatory fields from StatefulSet
+  selector:
+    matchLabels:
+      app: example
+  serviceName: example
+  template:
+    metadata:
+      labels:
+        app: example
+```
+
+As it is based on `StatefulSet`, `selector` and `template.medatada.labels` needs to be defined and have the exact same set of labels. `serviceName` must be set to some non-empty string as it is also required by `StatefulSet`.
+
+Runner-related fields like `ephemeral`, `repository`, `organization`, `enterprise`, and so on should be written directly under `spec`.
+
+Fields like `volumeClaimTemplates` that originates from `StatefulSet` should also be written directly under `spec`.
+
+Pod-related fields like security contexts and volumes are written under `spec.template.spec` like `StatefulSet`.
+
+Similarly, container-related fields like resource requests and limits, container image names and tags, security context, and so on are written under `spec.template.spec.containers`. There are two reserved container `name`, `runner` and `docker`. The former is for the container that runs [actions runner](https://github.com/actions/runner) and the latter is for the container that runs a dockerd.
+
+For a more complex example, see the below:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerSet
+metadata:
+  name: example
+spec:
+  ephemeral: false
+  replicas: 2
+  repository: mumoshu/actions-runner-controller-ci
+  dockerdWithinRunnerContainer: true
+  template:
+    spec:
+      securityContext:
+        # All level/role/type/user values will vary based on your SELinux policies.
+        # See https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html/container_security_guide/docker_selinux_security_policy for information about SELinux with containers
+        seLinuxOptions: 
+          level: "s0"
+          role: "system_r"
+          type: "super_t"
+          user: "system_u"
+      containers:
+      - name: runner
+        env: []
+        resources:
+          limits:
+            cpu: "4.0"
+            memory: "8Gi"
+          requests:
+            cpu: "2.0"
+            memory: "4Gi"
+      - name: docker
+        resources:
+          limits:
+            cpu: "4.0"
+            memory: "8Gi"
+          requests:
+            cpu: "2.0"
+            memory: "4Gi"
+```
+
+You can also read the design and usage documentation written in the original pull request that introduced `RunnerSet` for more information [#629](https://github.com/actions-runner-controller/actions-runner-controller/pull/629).
+
+Under the hood, `RunnerSet` relies on Kubernetes's `StatefulSet` and Mutating Webhook. A statefulset is used to create a number of pods that has stable names and dynamically provisioned persistent volumes, so that each statefulset-managed pod gets the same persistent volume even after restarting. A mutating webhook is used to dynamically inject a runner's "registration token" which is used to call GitHub's "Create Runner" API.
+
+**Limitations**
+
+* For autoscaling the `RunnerSet` kind only supports pull driven scaling or the `workflow_job` event for webhook driven scaling.
+
 ### Autoscaling
 
-> Since the release of GitHub's [`workflow_job` webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job), webhook driven scaling is the preferred way of autoscaling as it enables targeted scaling of your `RunnerDeployments` / `RunnerSets` as it includes the `runs-on` information needed to scale the appropriate runners for that workflow run. More broadly, webhook driven scaling is the preferred scaling option as it is far quicker compared to the pull driven scaling and is easy to setup.
+> Since the release of GitHub's [`workflow_job` webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job), webhook driven scaling is the preferred way of autoscaling as it enables targeted scaling of your `RunnerDeployment` / `RunnerSet` as it includes the `runs-on` information needed to scale the appropriate runners for that workflow run. More broadly, webhook driven scaling is the preferred scaling option as it is far quicker compared to the pull driven scaling and is easy to setup.
 
-A `RunnerDeployment` or `RunnerSet` (see [stateful runners](#stateful-runners) for more details on this kind) can scale the number of runners between `minReplicas` and `maxReplicas` fields driven by either pull based scaling metrics or via a webhook event (see limitations section of [stateful runners](#stateful-runners) for cavaets of this kind). Whether the autoscaling is driven from a webhook event or pull based metrics it is implemented by backing a `RunnerDeployment` or `RunnerSet` kind with a `HorizontalRunnerAutoscaler` kind.
+> If you are using controller version < [v0.22.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.22.0) and you are not using GHES, and so can't set your rate limit budget, it is recommended that you use 100 replicas or fewer to prevent being rate limited.
+
+A `RunnerDeployment` or `RunnerSet` can scale the number of runners between `minReplicas` and `maxReplicas` fields driven by either pull based scaling metrics or via a webhook event (see limitations section of [stateful runners](#stateful-runners) for cavaets of this kind). Whether the autoscaling is driven from a webhook event or pull based metrics it is implemented by backing a `RunnerDeployment` or `RunnerSet` kind with a `HorizontalRunnerAutoscaler` kind.
+
+**_Important!!! If you opt to configure autoscaling, ensure you remove the `replicas:` attribute in the `RunnerDeployment` / `RunnerSet` kinds that are configured for autoscaling [#206](https://github.com/actions-runner-controller/actions-runner-controller/issues/206#issuecomment-748601907)_**
 
 #### Anti-Flapping Configuration
 
@@ -387,10 +484,13 @@ kind: HorizontalRunnerAutoscaler
 metadata:
   name: example-runner-deployment-autoscaler
 spec:
-  # Runners in the targeted RunnerDeployment won't be scaled down for 5 minutes instead of the default 10 minutes now
+  # Runners in the targeted RunnerDeployment won't be scaled down
+  # for 5 minutes instead of the default 10 minutes now
   scaleDownDelaySecondsAfterScaleOut: 300
   scaleTargetRef:
     name: example-runner-deployment
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -416,6 +516,8 @@ spec:
   scaleTargetRef:
     # Your RunnerDeployment Here
     name: example-runner-deployment
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   minReplicas: 1
   maxReplicas: 5
   # Your chosen scaling metrics here
@@ -441,8 +543,6 @@ The `TotalNumberOfQueuedAndInProgressWorkflowRuns` metric polls GitHub for all p
 
 Example `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`:
 
-**_Important!!! We no longer include the attribute `replicas` in our `RunnerDeployment` if we are configuring autoscaling!_**
-
 ```yaml
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerDeployment
@@ -460,6 +560,8 @@ metadata:
 spec:
   scaleTargetRef:
     name: example-runner-deployment
+    # IMPORTANT : If your HRA is targeting a RunnerSet you must specify the kind in the scaleTargetRef:, uncomment the below
+    #kind: RunnerSet
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -484,8 +586,6 @@ The `HorizontalRunnerAutoscaler` will poll GitHub for the number of runners in t
 
 Examples of each scaling type implemented with a `RunnerDeployment` backed by a `HorizontalRunnerAutoscaler`:
 
-**_Important!!! We no longer include the attribute `replicas` in our `RunnerDeployment` if we are configuring autoscaling!_**
-
 ```yaml
 ---
 apiVersion: actions.summerwind.dev/v1alpha1
@@ -495,6 +595,8 @@ metadata:
 spec:
   scaleTargetRef:
     name: example-runner-deployment
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -514,6 +616,8 @@ metadata:
 spec:
   scaleTargetRef:
     name: example-runner-deployment
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   minReplicas: 1
   maxReplicas: 5
   metrics:
@@ -541,6 +645,8 @@ kind: HorizontalRunnerAutoscaler
 spec:
   scaleTargetRef:
     name: example-runners
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scaleUpTriggers:
   - githubEvent:
       checkRun:
@@ -566,9 +672,9 @@ To enable this feature, you firstly need to install the webhook server, currentl
 _[see the values documentation for all configuration options](https://github.com/actions-runner-controller/actions-runner-controller/blob/master/charts/actions-runner-controller/README.md)_
 
 ```console
-$ helm --upgrade install actions-runner-controller/actions-runner-controller \
-  githubWebhookServer.enabled=true \
-  githubWebhookServer.ports[0].nodePort=33080
+$ helm upgrade --install --namespace actions-runner-system --create-namespace \
+             --wait actions-runner-controller actions-runner-controller/actions-runner-controller \
+             --set "githubWebhookServer.enabled=true,githubWebhookServer.ports[0].nodePort=33080"
 ```
 
 The above command will result in exposing the node port 33080 for Webhook events. Usually, you need to create an
@@ -584,10 +690,14 @@ by learning the following configuration examples.
 - [Example 3: Scale on each `pull_request` event against a given set of branches](#example-3-scale-on-each-pull_request-event-against-a-given-set-of-branches)
 - [Example 4: Scale on each `push` event](#example-4-scale-on-each-push-event)
 
+**Note:** All these examples should have **minReplicas** & **maxReplicas** as mandatory parameter even for webhook driven scaling. 
+
 ##### Example 1: Scale on each `workflow_job` event
 
 > This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
 
+_Note: GitHub does not include the runner group information of a repository in the payload of `workflow_job` event in the initial `queued` event. The runner group information is only include for `workflow_job` events when the job has already been allocated to a runner (events with a status of `in_progress` or `completed`). Please do raise feature requests against [GitHub](https://support.github.com/tickets/personal/0) for this information to be included in the initial `queued` event if this would improve autoscaling runners for you._
+
 The most flexible webhook GitHub offers is the `workflow_job` webhook, it includes the `runs-on` information in the payload allowing scaling based on runner labels.
 
 This webhook should cover most people's needs, please experiment with this webhook first before considering the others.
@@ -605,11 +715,15 @@ kind: HorizontalRunnerAutoscaler
 spec:
   scaleTargetRef:
     name: example-runners
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scaleUpTriggers:
   - githubEvent: {}
     duration: "30m"
 ```
 
+This webhook requires you to explicitly set the labels in the RunnerDeployment / RunnerSet if you are using them in your workflow to match the agents (field `runs-on`). Only `self-hosted` will be considered as included by default.
+
 You can configure your GitHub webhook settings to only include `Workflows Job` events, so that it sends us three kinds of `workflow_job` events per a job run.
 
 Each kind has a `status` of `queued`, `in_progress` and `completed`. With the above configuration, `actions-runner-controller` adds one runner for a `workflow_job` event whose `status` is `queued`. Similarly, it removes one runner for a `workflow_job` event whose `status` is `completed`. The cavaet to this to remember is that this the scale down is within the bounds of your `scaleDownDelaySecondsAfterScaleOut` configuration, if this time hasn't past the scale down will be defered.
@@ -633,6 +747,8 @@ kind: HorizontalRunnerAutoscaler
 spec:
   scaleTargetRef:
     name: example-runners
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scaleUpTriggers:
   - githubEvent:
       checkRun:
@@ -657,6 +773,8 @@ kind: HorizontalRunnerAutoscaler
 spec:
   scaleTargetRef:
     name: example-runners
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scaleUpTriggers:
   - githubEvent:
       checkRun:
@@ -685,6 +803,8 @@ kind: HorizontalRunnerAutoscaler
 spec:
   scaleTargetRef:
     name: example-runners
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scaleUpTriggers:
   - githubEvent:
       pullRequest:
@@ -711,19 +831,20 @@ kind: HorizontalRunnerAutoscaler
 spec:
   scaleTargetRef:
     name: example-runners
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scaleUpTriggers:
   - githubEvent:
       push:
     amount: 1
     duration: "5m"
 ```
+
 #### Autoscaling to/from 0
 
 > This feature requires controller version => [v0.19.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.19.0)
 
-_Note: The controller creates a "registration-only" runner per RunnerReplicaSet when it is being scaled to zero and retains it until there are one or more runners available. This is a deprecated feature for GitHub Cloud as "registration-only" runners are no longer needed due to GitHub changing their runner [routing logic](https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow#routing-precedence-for-self-hosted-runners) to no longer fail a workflow run if it targets a runner label that there are no registered runners for._
-
-The regular `RunnerDeployment` `replicas:` attribute as well as the `HorizontalRunnerAutoscaler` `minReplicas:` attribute supports being set to 0.
+The regular `RunnerDeployment` / `RunnerSet` `replicas:` attribute as well as the `HorizontalRunnerAutoscaler` `minReplicas:` attribute supports being set to 0.
 
 The main use case for scaling from 0 is with the `HorizontalRunnerAutoscaler` kind. To scale from 0 whilst still being able to provision runners as jobs are queued we must use the `HorizontalRunnerAutoscaler` with only certain scaling configurations, only the below configurations support scaling from 0 whilst also being able to provision runners as jobs are queued:
 
@@ -736,6 +857,8 @@ The main use case for scaling from 0 is with the `HorizontalRunnerAutoscaler` ki
 
 If a HorizontalRunnerAutoscaler is configured with a secondary metric of `TotalNumberOfQueuedAndInProgressWorkflowRuns` then be aware that the controller will check the primary metric of `PercentageRunnersBusy` first and will only use the secondary metric to calculate the desired replica count if the primary metric returns 0 desired replicas.
 
+Webhook-based autoscaling is the best option as it is relatively easy to configure and also it can scale scale quickly.
+
 #### Scheduled Overrides
 
 > This feature requires controller version => [v0.19.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.19.0)
@@ -755,6 +878,8 @@ metadata:
 spec:
   scaleTargetRef:
     name: example-runner-deployment
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scheduledOverrides:
   # Override minReplicas to 100 only between 2021-06-01T00:00:00+09:00 and 2021-06-03T00:00:00+09:00
   - startTime: "2021-06-01T00:00:00+09:00"
@@ -775,6 +900,8 @@ metadata:
 spec:
   scaleTargetRef:
     name: example-runner-deployment
+    # Uncomment the below in case the target is not RunnerDeployment but RunnerSet
+    #kind: RunnerSet
   scheduledOverrides:
   # Override minReplicas to 0 only between 0am sat to 0am mon
   - startTime: "2021-05-01T00:00:00+09:00"
@@ -915,6 +1042,11 @@ spec:
       # false (default) = Docker support is provided by a sidecar container deployed in the runner pod.
       # true = No docker sidecar container is deployed in the runner pod but docker can be used within the runner container instead. The image summerwind/actions-runner-dind is used by default.
       dockerdWithinRunnerContainer: true
+      #Optional environement variables for docker container
+      # Valid only when dockerdWithinRunnerContainer=false
+      dockerEnv:
+        - name: HTTP_PROXY
+          value: http://example.com      
       # Docker sidecar container image tweaks examples below, only applicable if dockerdWithinRunnerContainer = false
       dockerdContainerResources:
         limits:
@@ -975,6 +1107,84 @@ spec:
       runtimeClassName: "runc"
 ```
 
+### Custom Volume mounts
+You can configure your own custom volume mounts. For example to have the work/docker data in memory or on NVME ssd, for
+i/o intensive builds. Other custom volume mounts should be possible as well, see [kubernetes documentation](https://kubernetes.io/docs/concepts/storage/volumes/)
+
+**RAM Disk Runner**<br />
+Example how to place the runner work dir, docker sidecar and /tmp within the runner onto a ramdisk.
+```yaml
+kind: RunnerDeployment
+spec:
+  template:
+    spec:
+      dockerVolumeMounts:
+        - mountPath: /var/lib/docker
+          name: docker
+      volumeMounts:
+        - mountPath: /tmp
+          name: tmp
+      volumes:
+        - name: docker
+          emptyDir:
+            medium: Memory
+        - name: work # this volume gets automatically used up for the workdir
+          emptyDir:
+            medium: Memory
+        - name: tmp
+          emptyDir:
+            medium: Memory
+      emphemeral: true # recommended to not leak data between builds.
+```
+
+**NVME SSD Runner**<br />
+In this example we provide NVME backed storage for the workdir, docker sidecar and /tmp within the runner.
+Here we use a working example on GKE, which will provide the NVME disk at /mnt/disks/ssd0.  We will be placing the respective volumes in subdirs here and in order to be able to run multiple runners we will use the pod name as prefix for subdirectories. Also the disk will fill up over time and disk space will not be freed until the node is removed.
+
+**Beware** that running these persistent backend volumes **leave data behind** between 2 different jobs on the workdir and /tmp with emphemeral: false.
+
+```yaml
+kind: RunnerDeployment
+spec:
+  template:
+    spec:
+      env:
+      - name: POD_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.name
+      dockerVolumeMounts:
+      - mountPath: /var/lib/docker
+        name: docker
+        subPathExpr: $(POD_NAME)-docker
+      - mountPath: /runner/_work
+        name: work
+        subPathExpr: $(POD_NAME)-work
+      volumeMounts:
+      - mountPath: /runner/_work
+        name: work
+        subPathExpr: $(POD_NAME)-work
+      - mountPath: /tmp
+        name: tmp
+        subPathExpr: $(POD_NAME)-tmp
+      dockerEnv:
+      - name: POD_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.name
+      volumes:
+      - hostPath:
+          path: /mnt/disks/ssd0
+        name: docker
+      - hostPath:
+          path: /mnt/disks/ssd0
+        name: work
+      - hostPath:
+          path: /mnt/disks/ssd0
+        name: tmp
+    emphemeral: true # VERY important. otherwise data inside the workdir and /tmp is not cleared between builds
+```
+
 ### Runner Labels
 
 To run a workflow job on a self-hosted runner, you can use the following syntax in your workflow:
@@ -1031,6 +1241,52 @@ spec:
       group: NewGroup
 ```
 
+GitHub supports custom visilibity in a Runner Group to make it available to a specific set of repositories only. By default if no GitHub
+authentication is included in the webhook server ARC will be assumed that all runner groups to be usable in all repositories.
+Currently, GitHub do not include the repository runner group membership information in the workflow_job event (or any webhook). To make the ARC "runner group aware" additional GitHub API calls are needed to find out what runner groups are visible to the webhook's repository. This behaviour will impact your rate-limit budget and so the option needs to be explicitly configured by the end user.
+
+This option will be enabled when proper GitHub authentication options (token, app or basic auth) is provided in the webhook server and `useRunnerGroupsVisibility` is set to true, e.g.
+
+```yaml
+githubWebhookServer:
+  enabled: false
+  replicaCount: 1
+  useRunnerGroupsVisibility: true
+```
+
+### Runner Entrypoint Features
+
+> Environment variable values must all be strings
+
+The entrypoint script is aware of a few environment variables for configuring features:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeployment
+spec:
+  template:
+    spec:
+      env:
+        # Issues a sleep command at the start of the entrypoint
+        - name: STARTUP_DELAY_IN_SECONDS
+          value: "2"
+        # Disables the wait for the docker daemon to be available check
+        - name: DISABLE_WAIT_FOR_DOCKER
+          value: "true"
+        # Disables automatic runner updates
+        - name: DISABLE_RUNNER_UPDATE
+          value: "true"
+        # Configure runner with --ephemeral instead of --once flag
+        # WARNING | THIS ENV VAR IS DEPRECATED AND WILL BE REMOVED
+        # IN A FUTURE VERSION OF ARC. IN 0.22.0 ARC SETS --ephemeral VIA 
+        # THE CONTROLLER SETTING THIS ENV VAR ON POD CREATION.
+        # THIS ENV VAR WILL BE REMOVED, SEE ISSUE #1196 FOR DETAILS
+        - name: RUNNER_FEATURE_FLAG_EPHEMERAL
+          value: "true"
+```
+
 ### Using IRSA (IAM Roles for Service Accounts) in EKS
 
 > This feature requires controller version => [v0.15.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.15.0)
@@ -1056,154 +1312,19 @@ spec:
         fsGroup: 1000
 ```
 
+### Persistent Runners
 
-### Use with Istio
+Every runner managed by ARC is "ephemeral" by default. The life of an ephemeral runner managed by ARC looks like this- ARC creates a runner pod for the runner. As it's an ephemeral runner, the `--ephemeral` flag is passed to the `actions/runner` agent that runs within the `runner` container of the runner pod.
 
-Istio 1.7.0 or greater has `holdApplicationUntilProxyStarts` added in https://github.com/istio/istio/pull/24737, which enables you to delay the `runner` container startup until the injected `istio-proxy` container finish starting. Try using it if you need to use Istio. Otherwise the runner is unlikely to work, because it fails to call any GitHub API to register itself due to `istio-proxy` being not up and running yet.
+`--ephemeral` is an `actions/runner` feature that instructs the runner to stop and de-register itself after the first job run.
 
-Note that there's no official Istio integration in actions-runner-controller. It should work, but it isn't covered by our acceptance test (a contribution to resolve this is welcomed). In addition to that, none of the actions-runner-controller maintainers use Istio daily. If you need more information, or have any issues using it, refer to the following links:
+Once the ephemeral runner has completed running a workflow job, it stops with a status code of 0, hence the runner pod is marked as completed, removed by ARC.
 
-- https://github.com/actions-runner-controller/actions-runner-controller/issues/591
-- https://github.com/actions-runner-controller/actions-runner-controller/pull/592
-- https://github.com/istio/istio/issues/11130
+As it's removed after a workflow job run, the runner pod is never reused across multiple GitHub Actions workflow jobs, providing you a clean environment per each workflow job.
 
-### Stateful Runners
+Although not recommended, it's possible to disable passing `--ephemeral` flag by explicitly setting `ephemeral: false` in the `RunnerDeployment` or `RunnerSet` spec. When disabled, your runner becomes "persistent". A persistent runner does not stop after workflow job ends, and in this mode `actions/runner` is known to clean only runner's work dir after each job. That means your runner's environment, including various actions cache, docker images stored in the `dind` and layer cache, is retained across multiple workflow job runs.
 
-> This feature requires controller version => [v0.20.0](https://github.com/actions-runner-controller/actions-runner-controller/releases/tag/v0.20.0)
-
-`actions-runner-controller` supports `RunnerSet` API that let you deploy stateful runners. A stateful runner is designed to be able to store some data persists across GitHub Actions workflow and job runs. You might find it useful, for example, to speed up your docker builds by persisting the docker layer cache.
-
-A basic `RunnerSet` would look like this:
-
-```yaml
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerSet
-metadata:
-  name: example
-spec:
-  ephemeral: false
-  replicas: 2
-  repository: mumoshu/actions-runner-controller-ci
-  # Other mandatory fields from StatefulSet
-  selector:
-    matchLabels:
-      app: example
-  serviceName: example
-  template:
-    metadata:
-      labels:
-        app: example
-```
-
-As it is based on `StatefulSet`, `selector` and `template.medatada.labels` needs to be defined and have the exact same set of labels. `serviceName` must be set to some non-empty string as it is also required by `StatefulSet`.
-
-Runner-related fields like `ephemeral`, `repository`, `organization`, `enterprise`, and so on should be written directly under `spec`.
-
-Fields like `volumeClaimTemplates` that originates from `StatefulSet` should also be written directly under `spec`.
-
-Pod-related fields like security contexts and volumes are written under `spec.template.spec` like `StatefulSet`.
-
-Similarly, container-related fields like resource requests and limits, container image names and tags, security context, and so on are written under `spec.template.spec.containers`. There are two reserved container `name`, `runner` and `docker`. The former is for the container that runs [actions runner](https://github.com/actions/runner) and the latter is for the container that runs a dockerd.
-
-For a more complex example, see the below:
-
-```yaml
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerSet
-metadata:
-  name: example
-spec:
-  # NOTE: RunnerSet supports non-ephemeral runners only today
-  ephemeral: false
-  replicas: 2
-  repository: mumoshu/actions-runner-controller-ci
-  dockerdWithinRunnerContainer: true
-  template:
-    spec:
-      securityContext:
-        #All level/role/type/user values will vary based on your SELinux policies.
-        #See https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux_atomic_host/7/html/container_security_guide/docker_selinux_security_policy for information about SELinux with containers
-        seLinuxOptions: 
-          level: "s0"
-          role: "system_r"
-          type: "super_t"
-          user: "system_u"
-      containers:
-      - name: runner
-        env: []
-        resources:
-          limits:
-            cpu: "4.0"
-            memory: "8Gi"
-          requests:
-            cpu: "2.0"
-            memory: "4Gi"
-      - name: docker
-        resources:
-          limits:
-            cpu: "4.0"
-            memory: "8Gi"
-          requests:
-            cpu: "2.0"
-            memory: "4Gi"
-```
-
-You can also read the design and usage documentation written in the original pull request that introduced `RunnerSet` for more information.
-
-https://github.com/actions-runner-controller/actions-runner-controller/pull/629
-
-Under the hood, `RunnerSet` relies on Kubernetes's `StatefulSet` and Mutating Webhook. A statefulset is used to create a number of pods that has stable names and dynamically provisioned persistent volumes, so that each statefulset-managed pod gets the same persistent volume even after restarting. A mutating webhook is used to dynamically inject a runner's "registration token" which is used to call GitHub's "Create Runner" API.
-
-We envision that `RunnerSet` will eventually replace `RunnerDeployment`, as `RunnerSet` provides a more standard API that is easy to learn and use because it is based on `StatefulSet`, and it has a support for `volumeClaimTemplates` which is crucial to manage dynamically provisioned persistent volumes.
-
-**Limitations**
-
-* For autoscaling the `RunnerSet` kind only supports pull driven scaling or the `workflow_job` event for webhook driven scaling.
-* For autoscaling the `RunnerSet` kind doesn't support the [registration-only runner](#autoscaling-tofrom-0)
-* A known down-side of relying on `StatefulSet` is that it misses a support for `maxUnavailable`. A `StatefulSet` basically works like `maxUnavailable: 1` in `Deployment`, which means that it can take down only one pod concurrently while doing a rolling-update of pods. Kubernetes 1.22 doesn't support customizing it yet so probably it takes more releases to arrive. See https://github.com/kubernetes/kubernetes/issues/68397 for more information.
-
-### Ephemeral Runners
-
-Both `RunnerDeployment` and `RunnerSet` has ability to configure `ephemeral: true` in the spec.
-
-When it is configured, it passes a `--once` flag to every runner.
-
-`--once` is an experimental `actions/runner` feature that instructs the runner to stop after the first job run. But it is a known race issue that may fetch a job even when it's being terminated. If a runner fetched a job while terminating, the job is very likely to fail because the terminating runner doesn't wait for the job to complete. This is tracked in #466.
-
-> The below feature depends on an unreleased GitHub feature
-
-GitHub seems to be adding an another flag called `--ephemeral` that is race-free. The pull request to add it to `actions/runner` can be found at https://github.com/actions/runner/pull/660.
-
-`actions-runner-controller` has a feature flag backend by an environment variable to enable using `--ephemeral` instead of `--once`. The environment variable is `RUNNER_FEATURE_FLAG_EPHEMERAL`. You can se it to `true` on runner containers in your runner pods to enable the feature.
-
-> At the time of writing this, you need to wait until GitHub rolls out the server-side feature for `--ephemeral`, AND you need to include your own `actions/runner` binary built from https://github.com/actions/runner/pull/660 into the runner container image to test this feature.
->
-> Please see comments in [`runner/Dockerfile`](/runner/Dockerfile) for more information about how to build a custom image using your own `actions/runner` binary.
-
-For example, a `RunnerSet` config with the flag enabled looks like:
-
-```yaml
-kind: RunnerSet
-metadata:
-  name: example-runnerset
-spec:
-  # ...
-  template:
-    metadata:
-      labels:
-        app: example-runnerset
-    spec:
-      containers:
-      - name: runner
-        imagePullPolicy: IfNotPresent
-        env:
-        - name: RUNNER_FEATURE_FLAG_EPHEMERAL
-          value: "true"
-```
-
-Note that once https://github.com/actions/runner/pull/660 becomes generally available on GitHub, you no longer need to build a custom runner image to use this feature. Just set `RUNNER_FEATURE_FLAG_EPHEMERAL` and it should use `--ephemeral`.
-
-In the future, `--once` might get removed in `actions/runner`. `actions-runner-controller` will make `--ephemeral` the default option for `ephemeral: true` runners until the legacy flag is removed.
+Persistent runners are available as an option for some edge cases however they are not preferred as they can create challenges around providing a deterministic and secure environment.
 
 ### Software Installed in the Runner Image
 
@@ -1213,7 +1334,7 @@ The project supports being deployed on the various cloud Kubernetes platforms (e
 **Bundled Software**<br />
 The GitHub hosted runners include a large amount of pre-installed software packages. GitHub maintain a list in README files at <https://github.com/actions/virtual-environments/tree/main/images/linux>
 
-This solution maintains a few runner images with `latest` aligning with GitHub's Ubuntu version. Older images are maintained whilst GitHub also provides them as an option. These images do not contain all of the software installed on the GitHub runners. It contains the following subset of packages from the GitHub runners:
+This solution maintains a few runner images with `latest` aligning with GitHub's Ubuntu version, these images do not contain all of the software installed on the GitHub runners. The images contain the following subset of packages from the GitHub runners:
 
 - Basic CLI packages
 - git
@@ -1244,67 +1365,38 @@ spec:
   image: YOUR_CUSTOM_DOCKER_IMAGE
 ```
 
-### Common Errors
+### Using without cert-manager
 
-#### invalid header field value
+Assuming you are installing in the default namespace, ensure your certificate has SANs:
 
-```json
-2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	
-{
-  "controller": "runner",
-  "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9",
-  "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"
-}
+* `webhook-service.actions-runner-system.svc`
+* `webhook-service.actions-runner-system.svc.cluster.local`
+
+It is possible to use a self-signed certificate by following a guide like
+[this one](https://mariadb.com/docs/security/encryption/in-transit/create-self-signed-certificates-keys-openssl/)
+using `openssl`.
+
+Install your certificate as a TLS secret:
+
+```shell
+$ kubectl create secret tls webhook-server-cert \
+  -n actions-runner-system \
+  --cert=path/to/cert/file \
+  --key=path/to/key/file
 ```
 
-**Solution**
+Set the Helm chart values as follows:
 
-Your base64'ed PAT token has a new line at the end, it needs to be created without a `\n` added, either:
-* `echo -n $TOKEN | base64`
-* Create the secret as described in the docs using the shell and documented flags
-
-#### Runner coming up before network available
-
-If you're running your action runners on a service mesh like Istio, you might
-have problems with runner configuration accompanied by logs like:
-
-```
-....
-runner Starting Runner listener with startup type: service
-runner Started listener process
-runner An error occurred: Not configured
-runner Runner listener exited with error code 2
-runner Runner listener exit with retryable error, re-launch runner in 5 seconds.
-....
+```shell
+$ CA_BUNDLE=$(cat path/to/ca.pem | base64)
+$ helm --upgrade install actions-runner-controller/actions-runner-controller \
+  certManagerEnabled=false \
+  admissionWebHooks.caBundle=${CA_BUNDLE}
 ```
 
-This is because the `istio-proxy` has not completed configuring itself when the
-configuration script tries to communicate with the network.
+# Troubleshooting
 
-**Solution**<br />
-
-> Added originally to help users with older istio instances.
-> Newer Istio instances can use Istio's `holdApplicationUntilProxyStarts` attribute ([istio/istio#11130](https://github.com/istio/istio/issues/11130)) to avoid having to delay starting up the runner.
-> Please read the discussion in [#592](https://github.com/actions-runner-controller/actions-runner-controller/pull/592) for more information.
-
-_Note: Prior to the runner version v2.279.0, the environment variable referenced below was called `STARTUP_DELAY`._
-
-You can add a delay to the runner's entrypoint script by setting the `STARTUP_DELAY_IN_SECONDS` environment
-variable for the runner pod. This will cause the script to sleep X seconds, this works with any runner kind.
-
-*Example `RunnerDeployment` with a 2 second startup delay:*
-```yaml
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerDeployment
-metadata:
-  name: example-runnerdeployment-with-sleep
-spec:
-  template:
-    spec:
-      env:
-        - name: STARTUP_DELAY_IN_SECONDS
-          value: "2" # Remember! env var values must be strings.
-```
+See [troubleshooting guide](TROUBLESHOOTING.md) for solutions to various problems people have ran into consistently.
 
 # Contributing
 

TROUBLESHOOTING.md

@@ -0,0 +1,100 @@
+# Troubleshooting
+
+* [Invalid header field value](#invalid-header-field-value)
+* [Runner coming up before network available](#runner-coming-up-before-network-available)
+* [Deployment fails on GKE due to webhooks](#deployment-fails-on-gke-due-to-webhooks)
+
+## Invalid header field value
+
+**Problem**
+
+```json
+2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	
+{
+  "controller": "runner",
+  "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9",
+  "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"
+}
+```
+
+**Solution**
+
+Your base64'ed PAT token has a new line at the end, it needs to be created without a `\n` added, either:
+* `echo -n $TOKEN | base64`
+* Create the secret as described in the docs using the shell and documented flags
+
+## Runner coming up before network available
+
+**Problem**
+
+If you're running your action runners on a service mesh like Istio, you might
+have problems with runner configuration accompanied by logs like:
+
+```
+....
+runner Starting Runner listener with startup type: service
+runner Started listener process
+runner An error occurred: Not configured
+runner Runner listener exited with error code 2
+runner Runner listener exit with retryable error, re-launch runner in 5 seconds.
+....
+```
+
+This is because the `istio-proxy` has not completed configuring itself when the
+configuration script tries to communicate with the network.
+
+More broadly, there are many other circumstances where the runner pod coming up first can cause issues.
+
+**Solution**<br />
+
+> Added originally to help users with older istio instances.
+> Newer Istio instances can use Istio's `holdApplicationUntilProxyStarts` attribute ([istio/istio#11130](https://github.com/istio/istio/issues/11130)) to avoid having to delay starting up the runner.
+> Please read the discussion in [#592](https://github.com/actions-runner-controller/actions-runner-controller/pull/592) for more information.
+
+You can add a delay to the runner's entrypoint script by setting the `STARTUP_DELAY_IN_SECONDS` environment variable for the runner pod. This will cause the script to sleep X seconds, this works with any runner kind.
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeployment-with-sleep
+spec:
+  template:
+    spec:
+      env:
+        # This runner's entrypoint script will have a 5 seconds delay 
+        # as a first action within the entrypoint script
+        - name: STARTUP_DELAY_IN_SECONDS
+          value: "5"
+```
+
+## Deployment fails on GKE due to webhooks
+
+**Problem**
+
+Due to GKEs firewall settings you may run into the following errors when trying to deploy runners on a private GKE cluster:
+
+```
+Internal error occurred: failed calling webhook "mutate.runner.actions.summerwind.dev": 
+Post https://webhook-service.actions-runner-system.svc:443/mutate-actions-summerwind-dev-v1alpha1-runner?timeout=10s: 
+context deadline exceeded
+```
+
+**Solution**<br />
+
+To fix this, you need to set up a firewall rule to allow the master node to connect to the webhook port.
+The exact way to do this may wary, but the following script should point you in the right direction:
+
+```
+# 1) Retrieve the network tag automatically given to the worker nodes
+# NOTE: this only works if you have only one cluster in your GCP project. You will have to manually inspect the result of this command to find the tag for the cluster you want to target
+WORKER_NODES_TAG=$(gcloud compute instances list --format='text(tags.items[0])' --filter='metadata.kubelet-config:*' | grep tags | awk '{print $2}' | sort | uniq)
+
+# 2) Take note of the VPC network in which you deployed your cluster
+# NOTE this only works if you have only one network in which you deploy your clusters
+NETWORK=$(gcloud compute instances list --format='text(networkInterfaces[0].network)' --filter='metadata.kubelet-config:*' | grep networks | awk -F'/' '{print $NF}' | sort | uniq)
+
+# 3) Get the master source ip block
+SOURCE=$(gcloud container clusters describe <cluster-name> --region <region> | grep masterIpv4CidrBlock| cut -d ':' -f 2 | tr -d ' ')
+gcloud compute firewall-rules create k8s-cert-manager --source-ranges $SOURCE --target-tags $WORKER_NODES_TAG  --allow TCP:9443 --network $NETWORK
+```

acceptance/deploy.sh

@@ -6,6 +6,8 @@ tpe=${ACCEPTANCE_TEST_SECRET_TYPE}
 
 VALUES_FILE=${VALUES_FILE:-$(dirname $0)/values.yaml}
 
+kubectl delete secret -n actions-runner-system controller-manager || :
+
 if [ "${tpe}" == "token" ]; then
   if ! kubectl get secret controller-manager -n actions-runner-system >/dev/null; then
     kubectl create secret generic controller-manager \
@@ -16,16 +18,29 @@ elif [ "${tpe}" == "app" ]; then
   kubectl create secret generic controller-manager \
     -n actions-runner-system \
     --from-literal=github_app_id=${APP_ID:?must not be empty} \
-    --from-literal=github_app_installation_id=${INSTALLATION_ID:?must not be empty} \
-    --from-file=github_app_private_key=${PRIVATE_KEY_FILE_PATH:?must not be empty}
+    --from-literal=github_app_installation_id=${APP_INSTALLATION_ID:?must not be empty} \
+    --from-file=github_app_private_key=${APP_PRIVATE_KEY_FILE:?must not be empty}
 else
   echo "ACCEPTANCE_TEST_SECRET_TYPE must be set to either \"token\" or \"app\"" 1>&2
   exit 1
 fi
 
+if [ -n "${WEBHOOK_GITHUB_TOKEN}" ]; then
+  kubectl -n actions-runner-system delete secret \
+      github-webhook-server || :
+  kubectl -n actions-runner-system create secret generic \
+      github-webhook-server \
+      --from-literal=github_token=${WEBHOOK_GITHUB_TOKEN:?WEBHOOK_GITHUB_TOKEN must not be empty}
+else
+  echo 'Skipped deploying secret "github-webhook-server". Set WEBHOOK_GITHUB_TOKEN to deploy.' 1>&2
+fi
+
 tool=${ACCEPTANCE_TEST_DEPLOYMENT_TOOL}
 
+TEST_ID=${TEST_ID:-default}
+
 if [ "${tool}" == "helm" ]; then
+  set -v
   helm upgrade --install actions-runner-controller \
     charts/actions-runner-controller \
     -n actions-runner-system \
@@ -34,42 +49,83 @@ if [ "${tool}" == "helm" ]; then
     --set authSecret.create=false \
     --set image.repository=${NAME} \
     --set image.tag=${VERSION} \
+    --set podAnnotations.test-id=${TEST_ID} \
+    --set githubWebhookServer.podAnnotations.test-id=${TEST_ID} \
     -f ${VALUES_FILE}
-  kubectl apply -f charts/actions-runner-controller/crds
-  kubectl -n actions-runner-system wait deploy/actions-runner-controller --for condition=available --timeout 60s
+  set +v
+  # To prevent `CustomResourceDefinition.apiextensions.k8s.io "runners.actions.summerwind.dev" is invalid: metadata.annotations: Too long: must have at most 262144 bytes`
+  # errors
+  kubectl create -f charts/actions-runner-controller/crds || kubectl replace -f charts/actions-runner-controller/crds
+  # This wait fails due to timeout when it's already in crashloopback and this update doesn't change the image tag.
+  # That's why we add `|| :`. With that we prevent stopping the script in case of timeout and
+  # proceed to delete (possibly in crashloopback and/or running with outdated image) pods so that they are recreated by K8s.
+  kubectl -n actions-runner-system wait deploy/actions-runner-controller --for condition=available --timeout 60s || :
 else
   kubectl apply \
     -n actions-runner-system \
     -f release/actions-runner-controller.yaml
-  kubectl -n actions-runner-system wait deploy/controller-manager --for condition=available --timeout 120s
+  kubectl -n actions-runner-system wait deploy/controller-manager --for condition=available --timeout 120s || :
 fi
 
+# Restart all ARC pods
+kubectl -n actions-runner-system delete po -l app.kubernetes.io/name=actions-runner-controller
+
+echo Waiting for all ARC pods to be up and running after restart
+
+kubectl -n actions-runner-system wait deploy/actions-runner-controller --for condition=available --timeout 120s
+
 # Adhocly wait for some time until actions-runner-controller's admission webhook gets ready
 sleep 20
 
 RUNNER_LABEL=${RUNNER_LABEL:-self-hosted}
 
 if [ -n "${TEST_REPO}" ]; then
-  if [ -n "USE_RUNNERSET" ]; then
-      cat acceptance/testdata/repo.runnerset.yaml | envsubst | kubectl apply -f -
-      cat acceptance/testdata/repo.runnerset.hra.yaml | envsubst | kubectl apply -f -
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerset envsubst | kubectl apply -f -
   else
     echo 'Deploying runnerdeployment and hra. Set USE_RUNNERSET if you want to deploy runnerset instead.'
-    cat acceptance/testdata/repo.runnerdeploy.yaml | envsubst | kubectl apply -f -
-    cat acceptance/testdata/repo.hra.yaml | envsubst | kubectl apply -f -
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_ORG= RUNNER_MIN_REPLICAS=${REPO_RUNNER_MIN_REPLICAS} NAME=repo-runnerdeploy envsubst | kubectl apply -f -
   fi
 else
   echo 'Skipped deploying runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy.'
 fi
 
 if [ -n "${TEST_ORG}" ]; then
-  cat acceptance/testdata/org.runnerdeploy.yaml | envsubst | kubectl apply -f -
-
-  if [ -n "${TEST_ORG_REPO}" ]; then
-    cat acceptance/testdata/org.hra.yaml | envsubst | kubectl apply -f -
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerset envsubst | kubectl apply -f -
   else
-    echo 'Skipped deploying organizational hra. Set TEST_ORG_REPO to "yourorg/yourrepo" to deploy.'
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} NAME=org-runnerdeploy envsubst | kubectl apply -f -
+  fi
+
+  if [ -n "${TEST_ORG_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orgroupg-runnerset envsubst | kubectl apply -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ENTERPRISE= TEST_REPO= RUNNER_MIN_REPLICAS=${ORG_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ORG_GROUP} NAME=orggroup-runnerdeploy envsubst | kubectl apply -f -
+    fi
+  else
+    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ORG_GROUP to deploy.'
   fi
 else
   echo 'Skipped deploying organizational runnerdeployment. Set TEST_ORG to deploy.'
 fi
+
+if [ -n "${TEST_ENTERPRISE}" ]; then
+  if [ "${USE_RUNNERSET}" != "false" ]; then
+    cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerset envsubst | kubectl apply -f -
+  else
+    cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} NAME=enterprise-runnerdeploy envsubst | kubectl apply -f -
+  fi
+
+  if [ -n "${TEST_ENTERPRISE_GROUP}" ]; then
+    if [ "${USE_RUNNERSET}" != "false" ]; then
+      cat acceptance/testdata/runnerset.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerset envsubst | kubectl apply -f -
+    else
+      cat acceptance/testdata/runnerdeploy.envsubst.yaml | TEST_ORG= TEST_REPO= RUNNER_MIN_REPLICAS=${ENTERPRISE_RUNNER_MIN_REPLICAS} TEST_GROUP=${TEST_ENTERPRISE_GROUP} NAME=enterprisegroup-runnerdeploy envsubst | kubectl apply -f -
+    fi
+  else
+    echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE_GROUP to deploy.'
+  fi
+else
+  echo 'Skipped deploying enterprise runnerdeployment. Set TEST_ENTERPRISE to deploy.'
+fi

acceptance/testdata/org.hra.yaml

@@ -1,36 +0,0 @@
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: HorizontalRunnerAutoscaler
-metadata:
-  name: org
-spec:
-  scaleTargetRef:
-    name: org-runnerdeploy
-  scaleUpTriggers:
-  - githubEvent:
-      checkRun:
-        types: ["created"]
-        status: "queued"
-    amount: 1
-    duration: "1m"
-  scheduledOverrides:
-  - startTime: "2021-05-11T16:05:00+09:00"
-    endTime: "2021-05-11T16:40:00+09:00"
-    minReplicas: 2
-  - startTime: "2021-05-01T00:00:00+09:00"
-    endTime: "2021-05-03T00:00:00+09:00"
-    recurrenceRule:
-      frequency: Weekly
-      untilTime: "2022-05-01T00:00:00+09:00"
-    minReplicas: 0
-  minReplicas: 0
-  maxReplicas: 5
-  # Used to test that HRA is working for org runners
-  metrics:
-  - type: PercentageRunnersBusy
-    scaleUpThreshold: '0.75'
-    scaleDownThreshold: '0.3'
-    scaleUpFactor: '2'
-    scaleDownFactor: '0.5'
-  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
-    repositoryNames:
-    - ${TEST_ORG_REPO}

acceptance/testdata/org.runnerdeploy.yaml

@@ -1,37 +0,0 @@
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerDeployment
-metadata:
-  name: org-runnerdeploy
-spec:
-#  replicas: 1
-  template:
-    spec:
-      organization: ${TEST_ORG}
-
-      #
-      # Custom runner image
-      #
-      image: ${RUNNER_NAME}:${RUNNER_TAG}
-      imagePullPolicy: IfNotPresent
-
-      #
-      # dockerd within runner container
-      #
-      ## Replace `mumoshu/actions-runner-dind:dev` with your dind image
-      #dockerdWithinRunnerContainer: true
-      #image: mumoshu/actions-runner-dind:dev
-
-      #
-      # Set the MTU used by dockerd-managed network interfaces (including docker-build-ubuntu)
-      #
-      #dockerMTU: 1450
-
-      #Runner group
-      # labels:
-      # - "mylabel 1"
-      # - "mylabel 2"
-
-      #
-      # Non-standard working directory
-      #
-      # workDir: "/"

acceptance/testdata/repo.hra.yaml

@@ -1,25 +0,0 @@
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: HorizontalRunnerAutoscaler
-metadata:
-  name: actions-runner-aos-autoscaler
-spec:
-  scaleTargetRef:
-    name: example-runnerdeploy
-  scaleUpTriggers:
-  - githubEvent:
-      checkRun:
-        types: ["created"]
-        status: "queued"
-    amount: 1
-    duration: "1m"
-  minReplicas: 0
-  maxReplicas: 5
-  metrics:
-  - type: PercentageRunnersBusy
-    scaleUpThreshold: '0.75'
-    scaleDownThreshold: '0.3'
-    scaleUpFactor: '2'
-    scaleDownFactor: '0.5'
-  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
-    repositoryNames:
-    - ${TEST_REPO}

acceptance/testdata/repo.runnerdeploy.yaml

@@ -1,37 +0,0 @@
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: RunnerDeployment
-metadata:
-  name: example-runnerdeploy
-spec:
-#  replicas: 1
-  template:
-    spec:
-      repository: ${TEST_REPO}
-
-      #
-      # Custom runner image
-      #
-      image: ${RUNNER_NAME}:${RUNNER_TAG}
-      imagePullPolicy: IfNotPresent
-
-      #
-      # dockerd within runner container
-      #
-      ## Replace `mumoshu/actions-runner-dind:dev` with your dind image
-      #dockerdWithinRunnerContainer: true
-      #image: mumoshu/actions-runner-dind:dev
-
-      #
-      # Set the MTU used by dockerd-managed network interfaces (including docker-build-ubuntu)
-      #
-      #dockerMTU: 1450
-
-      #Runner group
-      # labels:
-      # - "mylabel 1"
-      # - "mylabel 2"
-
-      #
-      # Non-standard working directory
-      #
-      # workDir: "/"

acceptance/testdata/repo.runnerset.hra.yaml

@@ -1,29 +0,0 @@
-apiVersion: actions.summerwind.dev/v1alpha1
-kind: HorizontalRunnerAutoscaler
-metadata:
-  name: example-runnerset
-spec:
-  scaleTargetRef:
-    kind: RunnerSet
-    name: example-runnerset
-  scaleUpTriggers:
-  - githubEvent:
-      checkRun:
-        types: ["created"]
-        status: "queued"
-    amount: 1
-    duration: "1m"
-  # RunnerSet doesn't support scale from/to zero yet
-  minReplicas: 1
-  maxReplicas: 5
-  # This should be less than 600(seconds, the default) for faster testing
-  scaleDownDelaySecondsAfterScaleOut: 60
-  metrics:
-  - type: PercentageRunnersBusy
-    scaleUpThreshold: '0.75'
-    scaleDownThreshold: '0.3'
-    scaleUpFactor: '2'
-    scaleDownFactor: '0.5'
-  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
-    repositoryNames:
-    - ${TEST_REPO}

acceptance/testdata/runnerdeploy.envsubst.yaml

@@ -0,0 +1,65 @@
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: ${NAME}
+spec:
+#  replicas: 1
+  template:
+    spec:
+      enterprise: ${TEST_ENTERPRISE}
+      group: ${TEST_GROUP}
+      organization: ${TEST_ORG}
+      repository: ${TEST_REPO}
+
+      #
+      # Custom runner image
+      #
+      image: ${RUNNER_NAME}:${RUNNER_TAG}
+      imagePullPolicy: IfNotPresent
+
+      ephemeral: ${TEST_EPHEMERAL}
+
+      # Whether to pass --ephemeral (true) or --once (false, deprecated)
+      env:
+      - name: RUNNER_FEATURE_FLAG_EPHEMERAL
+        value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
+
+      #
+      # dockerd within runner container
+      #
+      ## Replace `mumoshu/actions-runner-dind:dev` with your dind image
+      #dockerdWithinRunnerContainer: true
+      #image: mumoshu/actions-runner-dind:dev
+      dockerdWithinRunnerContainer: ${RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER}
+
+      #
+      # Set the MTU used by dockerd-managed network interfaces (including docker-build-ubuntu)
+      #
+      #dockerMTU: 1450
+
+      #Runner group
+      # labels:
+      # - "mylabel 1"
+      # - "mylabel 2"
+      labels:
+      - "${RUNNER_LABEL}"
+
+      #
+      # Non-standard working directory
+      #
+      # workDir: "/"
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: ${NAME}
+spec:
+  scaleTargetRef:
+    name: ${NAME}
+  scaleUpTriggers:
+  - githubEvent: {}
+    amount: 1
+    duration: "10m"
+  minReplicas: ${RUNNER_MIN_REPLICAS}
+  maxReplicas: 10
+  scaleDownDelaySecondsAfterScaleOut: ${RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT}

acceptance/testdata/runnerset.envsubst.yaml

@@ -1,17 +1,17 @@
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: RunnerSet
 metadata:
-  name: example-runnerset
+  name: ${NAME}
 spec:
   # MANDATORY because it is based on StatefulSet: Results in a below error when omitted:
   #   missing required field "selector" in dev.summerwind.actions.v1alpha1.RunnerSet.spec
   selector:
     matchLabels:
-      app: example-runnerset
+      app: ${NAME}
 
   # MANDATORY because it is based on StatefulSet: Results in a below error when omitted:
   # missing required field "serviceName" in dev.summerwind.actions.v1alpha1.RunnerSet.spec]
-  serviceName: example-runnerset
+  serviceName: ${NAME}
 
   #replicas: 1
 
@@ -20,16 +20,23 @@ spec:
   # result in queued jobs hanging forever.
   ephemeral: ${TEST_EPHEMERAL}
 
+  enterprise: ${TEST_ENTERPRISE}
+  group: ${TEST_GROUP}
+  organization: ${TEST_ORG}
   repository: ${TEST_REPO}
+
   #
   # Custom runner image
   #
   image: ${RUNNER_NAME}:${RUNNER_TAG}
+
   #
   # dockerd within runner container
   #
   ## Replace `mumoshu/actions-runner-dind:dev` with your dind image
   #dockerdWithinRunnerContainer: true
+  dockerdWithinRunnerContainer: ${RUNNER_DOCKERD_WITHIN_RUNNER_CONTAINER}
+
   #
   # Set the MTU used by dockerd-managed network interfaces (including docker-build-ubuntu)
   #
@@ -47,7 +54,7 @@ spec:
   template:
     metadata:
       labels:
-        app: example-runnerset
+        app: ${NAME}
     spec:
       containers:
       - name: runner
@@ -57,3 +64,19 @@ spec:
           value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}"
       #- name: docker
       #  #image: mumoshu/actions-runner-dind:dev
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: ${NAME}
+spec:
+  scaleTargetRef:
+    kind: RunnerSet
+    name: ${NAME}
+  scaleUpTriggers:
+  - githubEvent: {}
+    amount: 1
+    duration: "10m"
+  minReplicas: ${RUNNER_MIN_REPLICAS}
+  maxReplicas: 10
+  scaleDownDelaySecondsAfterScaleOut: ${RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT}

acceptance/values.yaml

@@ -1,12 +1,15 @@
 # Set actions-runner-controller settings for testing
-githubAPICacheDuration: 10s
+logLevel: "-4"
 githubWebhookServer:
+  logLevel: "-4"
   enabled: true
   labels: {}
   replicaCount: 1
   syncPeriod: 10m
+  useRunnerGroupsVisibility: true
   secret:
-    create: true
+    enabled: true
+    # create: true
     name: "github-webhook-server"
     ### GitHub Webhook Configuration
     #github_webhook_secret_token: ""

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -107,6 +107,9 @@ type CapacityReservation struct {
 	Name           string      `json:"name,omitempty"`
 	ExpirationTime metav1.Time `json:"expirationTime,omitempty"`
 	Replicas       int         `json:"replicas,omitempty"`
+
+	// +optional
+	EffectiveTime metav1.Time `json:"effectiveTime,omitempty"`
 }
 
 type ScaleTargetRef struct {

api/v1alpha1/runner_types.go

@@ -81,6 +81,9 @@ type RunnerPodSpec struct {
 	// +optional
 	DockerVolumeMounts []corev1.VolumeMount `json:"dockerVolumeMounts,omitempty"`
 
+	// +optional
+	DockerEnv []corev1.EnvVar `json:"dockerEnv,omitempty"`
+
 	// +optional
 	Containers []corev1.Container `json:"containers,omitempty"`
 

api/v1alpha1/runnerdeployment_types.go

@@ -31,6 +31,14 @@ type RunnerDeploymentSpec struct {
 	// +nullable
 	Replicas *int `json:"replicas,omitempty"`
 
+	// EffectiveTime is the time the upstream controller requested to sync Replicas.
+	// It is usually populated by the webhook-based autoscaler via HRA.
+	// The value is inherited to RunnerRepicaSet(s) and used to prevent ephemeral runners from unnecessarily recreated.
+	//
+	// +optional
+	// +nullable
+	EffectiveTime *metav1.Time `json:"effectiveTime"`
+
 	// +optional
 	// +nullable
 	Selector *metav1.LabelSelector `json:"selector"`

api/v1alpha1/runnerdeployment_webhook.go

@@ -26,7 +26,7 @@ import (
 )
 
 // log is for logging in this package.
-var runenrDeploymentLog = logf.Log.WithName("runnerdeployment-resource")
+var runnerDeploymentLog = logf.Log.WithName("runnerdeployment-resource")
 
 func (r *RunnerDeployment) SetupWebhookWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).
@@ -49,13 +49,13 @@ var _ webhook.Validator = &RunnerDeployment{}
 
 // ValidateCreate implements webhook.Validator so a webhook will be registered for the type
 func (r *RunnerDeployment) ValidateCreate() error {
-	runenrDeploymentLog.Info("validate resource to be created", "name", r.Name)
+	runnerDeploymentLog.Info("validate resource to be created", "name", r.Name)
 	return r.Validate()
 }
 
 // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
 func (r *RunnerDeployment) ValidateUpdate(old runtime.Object) error {
-	runenrDeploymentLog.Info("validate resource to be updated", "name", r.Name)
+	runnerDeploymentLog.Info("validate resource to be updated", "name", r.Name)
 	return r.Validate()
 }
 

api/v1alpha1/runnerreplicaset_types.go

@@ -26,6 +26,15 @@ type RunnerReplicaSetSpec struct {
 	// +nullable
 	Replicas *int `json:"replicas,omitempty"`
 
+	// EffectiveTime is the time the upstream controller requested to sync Replicas.
+	// It is usually populated by the webhook-based autoscaler via HRA and RunnerDeployment.
+	// The value is used to prevent runnerreplicaset controller from unnecessarily recreating ephemeral runners
+	// based on potentially outdated Replicas value.
+	//
+	// +optional
+	// +nullable
+	EffectiveTime *metav1.Time `json:"effectiveTime"`
+
 	// +optional
 	// +nullable
 	Selector *metav1.LabelSelector `json:"selector"`

api/v1alpha1/runnerset_types.go

@@ -25,6 +25,14 @@ import (
 type RunnerSetSpec struct {
 	RunnerConfig `json:",inline"`
 
+	// EffectiveTime is the time the upstream controller requested to sync Replicas.
+	// It is usually populated by the webhook-based autoscaler via HRA.
+	// It is used to prevent ephemeral runners from unnecessarily recreated.
+	//
+	// +optional
+	// +nullable
+	EffectiveTime *metav1.Time `json:"effectiveTime,omitempty"`
+
 	appsv1.StatefulSetSpec `json:",inline"`
 }
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -47,6 +47,7 @@ func (in *CacheEntry) DeepCopy() *CacheEntry {
 func (in *CapacityReservation) DeepCopyInto(out *CapacityReservation) {
 	*out = *in
 	in.ExpirationTime.DeepCopyInto(&out.ExpirationTime)
+	in.EffectiveTime.DeepCopyInto(&out.EffectiveTime)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservation.
@@ -498,6 +499,10 @@ func (in *RunnerDeploymentSpec) DeepCopyInto(out *RunnerDeploymentSpec) {
 		*out = new(int)
 		**out = **in
 	}
+	if in.EffectiveTime != nil {
+		in, out := &in.EffectiveTime, &out.EffectiveTime
+		*out = (*in).DeepCopy()
+	}
 	if in.Selector != nil {
 		in, out := &in.Selector, &out.Selector
 		*out = new(metav1.LabelSelector)
@@ -599,6 +604,13 @@ func (in *RunnerPodSpec) DeepCopyInto(out *RunnerPodSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.DockerEnv != nil {
+		in, out := &in.DockerEnv, &out.DockerEnv
+		*out = make([]v1.EnvVar, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.Containers != nil {
 		in, out := &in.Containers, &out.Containers
 		*out = make([]v1.Container, len(*in))
@@ -805,6 +817,10 @@ func (in *RunnerReplicaSetSpec) DeepCopyInto(out *RunnerReplicaSetSpec) {
 		*out = new(int)
 		**out = **in
 	}
+	if in.EffectiveTime != nil {
+		in, out := &in.EffectiveTime, &out.EffectiveTime
+		*out = (*in).DeepCopy()
+	}
 	if in.Selector != nil {
 		in, out := &in.Selector, &out.Selector
 		*out = new(metav1.LabelSelector)
@@ -916,6 +932,10 @@ func (in *RunnerSetList) DeepCopyObject() runtime.Object {
 func (in *RunnerSetSpec) DeepCopyInto(out *RunnerSetSpec) {
 	*out = *in
 	in.RunnerConfig.DeepCopyInto(&out.RunnerConfig)
+	if in.EffectiveTime != nil {
+		in, out := &in.EffectiveTime, &out.EffectiveTime
+		*out = (*in).DeepCopy()
+	}
 	in.StatefulSetSpec.DeepCopyInto(&out.StatefulSetSpec)
 }
 

charts/actions-runner-controller/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.15.0
+version: 0.17.1
 
 # Used as the default manager tag value when no tag property is provided in the values.yaml
-appVersion: 0.20.3
+appVersion: 0.22.1
 
 home: https://github.com/actions-runner-controller/actions-runner-controller
 

charts/actions-runner-controller/README.md

@@ -15,9 +15,13 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `syncPeriod`                                             | Set the period in which the controler reconciles the desired runners count                                                 | 10m                                                                  |
 | `enableLeaderElection`                                   | Enable election configuration                                                                                              | true                                                                 |
 | `leaderElectionId`                                       | Set the election ID for the controller group                                                                               |                                                                      |
-| `githubAPICacheDuration`                                 | Set the cache period for API calls                                                                                         |                                                                      |
 | `githubEnterpriseServerURL`                              | Set the URL for a self-hosted GitHub Enterprise Server                                                                     |                                                                      |
+| `githubURL`                                              | Override GitHub URL to be used for GitHub API calls                                                                        |                                                                      |
+| `githubUploadURL`                                        | Override GitHub Upload URL to be used for GitHub API calls                                                                 |                                                                      |
+| `runnerGithubURL`                                        | Override GitHub URL to be used by runners during registration                                                              |                                                                      |
 | `logLevel`                                               | Set the log level of the controller container                                                                              |                                                                      |
+| `additionalVolumes`                                      | Set additional volumes to add to the manager container                                                                     |                                                                      |
+| `additionalVolumeMounts`                                 | Set additional volume mounts to add to the manager container                                                               |                                                                      |
 | `authSecret.create`                                      | Deploy the controller auth secret                                                                                          | false                                                                |
 | `authSecret.name`                                        | Set the name of the auth secret                                                                                            | controller-manager                                                   |
 | `authSecret.annotations`                                 | Set annotations for the auth Secret                                                                                        |                                                                      |
@@ -25,10 +29,14 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `authSecret.github_app_installation_id`                  | The ID of your GitHub App installation. **This can't be set at the same time as `authSecret.github_token`**                |                                                                      |
 | `authSecret.github_app_private_key`                      | The multiline string of your GitHub App's private key. **This can't be set at the same time as `authSecret.github_token`** |                                                                      |
 | `authSecret.github_token`                                | Your chosen GitHub PAT token. **This can't be set at the same time as the `authSecret.github_app_*`**                      |                                                                      |
+| `authSecret.github_basicauth_username`                     | Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                      |
+| `authSecret.github_basicauth_password`                     | Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API                 |                                                                      |
 | `dockerRegistryMirror`                                   | The default Docker Registry Mirror used by runners.                                                                        |                                                                      |
+| `hostNetwork`                                            | The "hostNetwork" of the controller container                                                                              | false                                                                |
 | `image.repository`                                       | The "repository/image" of the controller container                                                                         | summerwind/actions-runner-controller                                 |
 | `image.tag`                                              | The tag of the controller container                                                                                        |                                                                      |
 | `image.actionsRunnerRepositoryAndTag`                    | The "repository/image" of the actions runner container                                                                     | summerwind/actions-runner:latest                                     |
+| `image.actionsRunnerImagePullSecrets`                    | Optional image pull secrets to be included in the runner pod's ImagePullSecrets                                            |                                                                      |
 | `image.dindSidecarRepositoryAndTag`                      | The "repository/image" of the dind sidecar container                                                                       | docker:dind                                                          |
 | `image.pullPolicy`                                       | The pull policy of the controller image                                                                                    | IfNotPresent                                                         |
 | `metrics.serviceMonitor`                                 | Deploy serviceMonitor kind for for use with prometheus-operator CRDs                                                       | false                                                                |
@@ -39,9 +47,9 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `metrics.proxy.image.tag`                                | The tag of the kube-proxy image to use when pulling the container                                                          | v0.10.0                                                              |
 | `metrics.serviceMonitorLabels`                           | Set labels to apply to ServiceMonitor resources                                                                            |                                                                      |
 | `imagePullSecrets`                                       | Specifies the secret to be used when pulling the controller pod containers                                                 |                                                                      |
-| `fullNameOverride`                                       | Override the full resource names	                                                                                        |                                                                      |
+| `fullnameOverride`                                       | Override the full resource names	                                                                                        |                                                                      |
 | `nameOverride`                                           | Override the resource name prefix	                                                                                        |                                                                      |
-| `serviceAccont.annotations`                              | Set annotations to the service account                                                                                     |                                                                      |
+| `serviceAccount.annotations`                              | Set annotations to the service account                                                                                     |                                                                      |
 | `serviceAccount.create`                                  | Deploy the controller pod under a service account                                                                          | true                                                                 |
 | `podAnnotations`                                         | Set annotations for the controller pod                                                                                     |                                                                      |
 | `podLabels`                                              | Set labels for the controller pod                                                                                          |                                                                      |
@@ -49,30 +57,34 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `securityContext`                                        | Set the security context for each container in the controller pod                                                          |                                                                      |
 | `podSecurityContext`                                     | Set the security context to controller pod                                                                                 |                                                                      |
 | `service.annotations`                                    | Set annotations for the provisioned webhook service resource                                                               |                                                                      |
-| `service.port`                                           | Set controller service type                                                                                                |                                                                      |
-| `service.type`                                           | Set controller service ports                                                                                               |                                                                      |
+| `service.port`                                           | Set controller service ports                                                                                                |                                                                      |
+| `service.type`                                           | Set controller service type                                                                                               |                                                                      |
 | `topologySpreadConstraints`                              | Set the controller pod topologySpreadConstraints                                                                           |                                                                      |
 | `nodeSelector`                                           | Set the controller pod nodeSelector                                                                                        |                                                                      |
 | `resources`                                              | Set the controller pod resources                                                                                           |                                                                      |
-| `affinity`                                               | Set the controller pod affinity rules                                                                                      |
-| `podDisruptionBudget.enabled`                                               | Enables a PDB to ensure HA of controller pods                                                                                     |      false                                                                    |
-| `podDisruptionBudget.minAvailable`                                               | Minimum number of pods that must be available after eviction                                                                                     |                                                                          |
-| `podDisruptionBudget.maxUnavailable`                                               | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                                                                     |                                                                          |
+| `affinity`                                               | Set the controller pod affinity rules                                                                                      |                                                                      |
+| `podDisruptionBudget.enabled`                            | Enables a PDB to ensure HA of controller pods                                                                              |      false                                                           |
+| `podDisruptionBudget.minAvailable`                       | Minimum number of pods that must be available after eviction                                                               |                                                                      |
+| `podDisruptionBudget.maxUnavailable`                     | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                      |
 | `tolerations`                                            | Set the controller pod tolerations                                                                                         |                                                                      |
 | `env`                                                    | Set environment variables for the controller container                                                                     |                                                                      |
 | `priorityClassName`                                      | Set the controller pod priorityClassName                                                                                   |                                                                      |
 | `scope.watchNamespace`                                   | Tells the controller and the github webhook server which namespace to watch if `scope.singleNamespace` is true             | `Release.Namespace` (the default namespace of the helm chart).       |
 | `scope.singleNamespace`                                  | Limit the controller to watch a single namespace                                                                           | false                                                                |
+| `certManagerEnabled`                                     | Enable cert-manager. If disabled you must set admissionWebHooks.caBundle and create TLS secrets manually                   | true                                                                 |
+| `admissionWebHooks.caBundle`                             | Base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate                                  |                                                                      |
 | `githubWebhookServer.logLevel`                           | Set the log level of the githubWebhookServer container                                                                     |                                                                      |
 | `githubWebhookServer.replicaCount`                       | Set the number of webhook server pods                                                                                      | 1                                                                    |
+| `githubWebhookServer.useRunnerGroupsVisibility`          | Enable supporting runner groups with custom visibility. This will incur in extra API calls and may blow up your budget. Currently, you also need to set `githubWebhookServer.secret.enabled` to enable this feature. | false                                                                |
 | `githubWebhookServer.syncPeriod`                         | Set the period in which the controller reconciles the resources                                                            | 10m                                                                  |
 | `githubWebhookServer.enabled`                            | Deploy the webhook server pod                                                                                              | false                                                                |
+| `githubWebhookServer.secret.enabled`                      | Passes the webhook hook secret to the github-webhook-server                                                                             | false                                                                |
 | `githubWebhookServer.secret.create`                      | Deploy the webhook hook secret                                                                                             | false                                                                |
 | `githubWebhookServer.secret.name`                        | Set the name of the webhook hook secret                                                                                    | github-webhook-server                                                |
 | `githubWebhookServer.secret.github_webhook_secret_token` | Set the webhook secret token value                                                                                         |                                                                      |
 | `githubWebhookServer.imagePullSecrets`                   | Specifies the secret to be used when pulling the githubWebhookServer pod containers                                        |                                                                      |
-| `githubWebhookServer.nameOveride`                        | Override the resource name prefix	                                                                                        |                                                                      |
-| `githubWebhookServer.fullNameOveride`                    | Override the full resource names	                                                                                        |                                                                      |
+| `githubWebhookServer.nameOverride`                        | Override the resource name prefix	                                                                                        |                                                                      |
+| `githubWebhookServer.fullnameOverride`                    | Override the full resource names	                                                                                        |                                                                      |
 | `githubWebhookServer.serviceAccount.create`              | Deploy the githubWebhookServer under a service account                                                                     | true                                                                 |
 | `githubWebhookServer.serviceAccount.annotations`         | Set annotations for the service account                                                                                    |                                                                      |
 | `githubWebhookServer.serviceAccount.name`                | Set the service account name                                                                                               |                                                                      |
@@ -92,6 +104,7 @@ All additional docs are kept in the `docs/` folder, this README is solely for do
 | `githubWebhookServer.ingress.annotations`                | Set annotations for the ingress kind                                                                                       |                                                                      |
 | `githubWebhookServer.ingress.hosts`                      | Set hosts configuration for ingress                                                                                        | `[{"host": "chart-example.local", "paths": []}]`                     |
 | `githubWebhookServer.ingress.tls`                        | Set tls configuration for ingress                                                                                          |                                                                      |
-| `githubWebhookServer.podDisruptionBudget.enabled`                                               | Enables a PDB to ensure HA of githubwebhook pods                                                                                     |      false                                                                    |
-| `githubWebhookServer.podDisruptionBudget.minAvailable`                                               | Minimum number of pods that must be available after eviction                                                                                     |                                                                          |
-| `githubWebhookServer.podDisruptionBudget.maxUnavailable`                                               | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                                                                     |                                                                          |
+| `githubWebhookServer.ingress.ingressClassName`           | Set ingress class name                                                                                                     |                                                                      |
+| `githubWebhookServer.podDisruptionBudget.enabled`        | Enables a PDB to ensure HA of githubwebhook pods                                                                           |      false                                                           |
+| `githubWebhookServer.podDisruptionBudget.minAvailable`   | Minimum number of pods that must be available after eviction                                                               |                                                                      |
+| `githubWebhookServer.podDisruptionBudget.maxUnavailable` | Maximum number of pods that can be unavailable after eviction. Kubernetes 1.7+ required.                                   |                                                                      |

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.6.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: horizontalrunnerautoscalers.actions.summerwind.dev
 spec:
@@ -49,6 +49,9 @@ spec:
                   items:
                     description: CapacityReservation specifies the number of replicas temporarily added to the scale target until ExpirationTime.
                     properties:
+                      effectiveTime:
+                        format: date-time
+                        type: string
                       expirationTime:
                         format: date-time
                         type: string

charts/actions-runner-controller/docs/UPGRADING.md

@@ -21,17 +21,20 @@ Due to the above you can't just do a `helm upgrade` to release the latest versio
 1. Upgrade CRDs
 
 ```shell
- # REMEMBER TO UPDATE THE CHART_VERSION TO RELEVANT CHART VERISON!!!!
-CHART_VERSION=0.14.0
+# REMEMBER TO UPDATE THE CHART_VERSION TO RELEVANT CHART VERISON!!!!
+CHART_VERSION=0.17.0
 
 curl -L https://github.com/actions-runner-controller/actions-runner-controller/releases/download/actions-runner-controller-${CHART_VERSION}/actions-runner-controller-${CHART_VERSION}.tgz | tar zxv --strip 1 actions-runner-controller/crds
 
-kubectl apply -f crds/
+kubectl replace -f crds/
 ```
 
 2. Upgrade the Helm release
 
 ```shell
+# helm repo [command]
+helm repo update
+
 # helm upgrade [RELEASE] [CHART] [flags]
 helm upgrade actions-runner-controller \
   actions-runner-controller/actions-runner-controller \

charts/actions-runner-controller/templates/_helpers.tpl

@@ -68,6 +68,10 @@ Create the name of the service account to use
 {{- default (include "actions-runner-controller.fullname" .) .Values.authSecret.name -}}
 {{- end }}
 
+{{- define "actions-runner-controller.githubWebhookServerSecretName" -}}
+{{- default (include "actions-runner-controller.fullname" .) .Values.githubWebhookServer.secret.name -}}
+{{- end }}
+
 {{- define "actions-runner-controller.leaderElectionRoleName" -}}
 {{- include "actions-runner-controller.fullname" . }}-leader-election
 {{- end }}

charts/actions-runner-controller/templates/certificate.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.certManagerEnabled }}
 # The following manifests contain a self-signed issuer CR and a certificate CR.
 # More document can be found at https://docs.cert-manager.io
 # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
@@ -22,3 +23,4 @@ spec:
     kind: Issuer
     name: {{ include "actions-runner-controller.selfsignedIssuerName" . }}
   secretName: {{ include "actions-runner-controller.servingCertName" . }}
+{{- end }}

charts/actions-runner-controller/templates/deployment.yaml

@@ -14,6 +14,7 @@ spec:
     metadata:
       {{- with .Values.podAnnotations }}
       annotations:
+        kubectl.kubernetes.io/default-logs-container: "manager"
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
@@ -46,6 +47,9 @@ spec:
         - "--sync-period={{ .Values.syncPeriod }}"
         - "--docker-image={{ .Values.image.dindSidecarRepositoryAndTag }}"
         - "--runner-image={{ .Values.image.actionsRunnerRepositoryAndTag }}"
+        {{- range .Values.image.actionsRunnerImagePullSecrets }}
+        - "--runner-image-pull-secret={{ . }}"
+        {{- end }}
         {{- if .Values.dockerRegistryMirror }}
         - "--docker-registry-mirror={{ .Values.dockerRegistryMirror }}"
         {{- end }}
@@ -58,6 +62,9 @@ spec:
         {{- if .Values.logLevel }}
         - "--log-level={{ .Values.logLevel }}"
         {{- end }}
+        {{- if .Values.runnerGithubURL  }}
+        - "--runner-github-url={{ .Values.runnerGithubURL }}"
+        {{- end }}
         command:
         - "/manager"
         env:
@@ -65,6 +72,15 @@ spec:
         - name: GITHUB_ENTERPRISE_URL
           value: {{ .Values.githubEnterpriseServerURL }}
         {{- end }}
+        {{- if .Values.githubURL  }}
+        - name: GITHUB_URL
+          value: {{ .Values.githubURL }}
+        {{- end }}
+        {{- if .Values.githubUploadURL  }}
+        - name: GITHUB_UPLOAD_URL
+          value: {{ .Values.githubUploadURL }}
+        {{- end }}
+        {{- if .Values.authSecret.enabled }}
         - name: GITHUB_TOKEN
           valueFrom:
             secretKeyRef:
@@ -84,7 +100,22 @@ spec:
               name: {{ include "actions-runner-controller.secretName" . }}
               optional: true
         - name: GITHUB_APP_PRIVATE_KEY
-          value: /etc/actions-runner-controller/github_app_private_key
+          valueFrom:
+            secretKeyRef:
+              key: github_app_private_key
+              name: {{ include "actions-runner-controller.secretName" . }}
+              optional: true
+        {{- if .Values.authSecret.github_basicauth_username }}
+        - name: GITHUB_BASICAUTH_USERNAME
+          value: {{ .Values.authSecret.github_basicauth_username }}
+        {{- end }}
+        - name: GITHUB_BASICAUTH_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: github_basicauth_password
+              name: {{ include "actions-runner-controller.secretName" . }}
+              optional: true
+        {{- end }}
         {{- range $key, $val := .Values.env }}
         - name: {{ $key }}
           value: {{ $val | quote }}
@@ -106,14 +137,19 @@ spec:
         securityContext:
           {{- toYaml .Values.securityContext | nindent 12 }}
         volumeMounts:
+        {{- if .Values.authSecret.enabled }}
         - mountPath: "/etc/actions-runner-controller"
           name: secret
           readOnly: true
+        {{- end }}
         - mountPath: /tmp
           name: tmp
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
+        {{- if .Values.additionalVolumeMounts }}
+          {{- toYaml .Values.additionalVolumeMounts | nindent 8 }} 
+        {{- end }}
       {{- if .Values.metrics.proxy.enabled }}
       - args:
         - "--secure-listen-address=0.0.0.0:{{ .Values.metrics.port }}"
@@ -133,15 +169,20 @@ spec:
       {{- end }}
       terminationGracePeriodSeconds: 10
       volumes:
+      {{- if .Values.authSecret.enabled }}
       - name: secret
         secret:
           secretName: {{ include "actions-runner-controller.secretName" . }}
+      {{- end }}
       - name: cert
         secret:
           defaultMode: 420
           secretName: {{ include "actions-runner-controller.servingCertName" . }}
       - name: tmp
         emptyDir: {}
+      {{- if .Values.additionalVolumes }}
+        {{- toYaml .Values.additionalVolumes | nindent 6}}
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -158,3 +199,6 @@ spec:
       topologySpreadConstraints:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.hostNetwork }}
+      hostNetwork: {{ .Values.hostNetwork }}
+      {{- end }}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -15,6 +15,7 @@ spec:
     metadata:
       {{- with .Values.githubWebhookServer.podAnnotations }}
       annotations:
+        kubectl.kubernetes.io/default-logs-container: "github-webhook-server"
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
@@ -45,6 +46,9 @@ spec:
         {{- if .Values.scope.singleNamespace }}
         - "--watch-namespace={{ default .Release.Namespace .Values.scope.watchNamespace }}"
         {{- end }}
+        {{- if .Values.runnerGithubURL  }}
+        - "--runner-github-url={{ .Values.runnerGithubURL }}"
+        {{- end }}
         command:
         - "/github-webhook-server"
         env:
@@ -54,6 +58,54 @@ spec:
               key: github_webhook_secret_token
               name: {{ include "actions-runner-controller-github-webhook-server.secretName" . }}
               optional: true
+        {{- if .Values.githubEnterpriseServerURL  }}
+        - name: GITHUB_ENTERPRISE_URL
+          value: {{ .Values.githubEnterpriseServerURL }}
+        {{- end }}
+        {{- if .Values.githubURL  }}
+        - name: GITHUB_URL
+          value: {{ .Values.githubURL }}
+        {{- end }}
+        {{- if .Values.githubUploadURL  }}
+        - name: GITHUB_UPLOAD_URL
+          value: {{ .Values.githubUploadURL }}
+        {{- end }}
+        {{- if and .Values.githubWebhookServer.useRunnerGroupsVisibility .Values.githubWebhookServer.secret.enabled }}
+        - name: GITHUB_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: github_token
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        - name: GITHUB_APP_ID
+          valueFrom:
+            secretKeyRef:
+              key: github_app_id
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        - name: GITHUB_APP_INSTALLATION_ID
+          valueFrom:
+            secretKeyRef:
+              key: github_app_installation_id
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        - name: GITHUB_APP_PRIVATE_KEY
+          valueFrom:
+            secretKeyRef:
+              key: github_app_private_key
+              name: {{ include "actions-runner-controller.githubWebhookServerSecretName" . }}
+              optional: true
+        {{- if .Values.authSecret.github_basicauth_username }}
+        - name: GITHUB_BASICAUTH_USERNAME
+          value: {{ .Values.authSecret.github_basicauth_username }}
+        {{- end }}
+        - name: GITHUB_BASICAUTH_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: github_basicauth_password
+              name: {{ include "actions-runner-controller.secretName" . }}
+              optional: true
+        {{- end }}
         {{- range $key, $val := .Values.githubWebhookServer.env }}
         - name: {{ $key }}
           value: {{ $val | quote }}

charts/actions-runner-controller/templates/githubwebhook.ingress.yaml

@@ -1,9 +1,11 @@
 {{- if .Values.githubWebhookServer.ingress.enabled -}}
 {{- $fullName := include "actions-runner-controller-github-webhook-server.fullname" . -}}
 {{- $svcPort := (index .Values.githubWebhookServer.service.ports 0).port -}}
-{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+apiVersion: networking.k8s.io/v1
+{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
 apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
+{{- else if .Capabilities.APIVersions.Has "extensions/v1beta1" }}
 apiVersion: extensions/v1beta1
 {{- end }}
 kind: Ingress
@@ -26,6 +28,9 @@ spec:
       secretName: {{ .secretName }}
     {{- end }}
   {{- end }}
+  {{- with .Values.githubWebhookServer.ingress.ingressClassName }}
+  ingressClassName: {{ . }}
+  {{- end }}
   rules:
     {{- range .Values.githubWebhookServer.ingress.hosts }}
     - host: {{ .host | quote }}
@@ -33,9 +38,19 @@ spec:
         paths:
           {{- range .paths }}
           - path: {{ .path }}
+            {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+            pathType: {{ .pathType }}
+            {{- end }}
             backend:
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              service:
+               name: {{ $fullName }}
+               port:
+                 number: {{ $svcPort }}
+              {{- else }}
               serviceName: {{ $fullName }}
               servicePort: {{ $svcPort }}
+              {{- end }}
           {{- end }}
     {{- end }}
   {{- end }}

charts/actions-runner-controller/templates/manager_secrets.yaml

@@ -26,4 +26,7 @@ data:
 {{- if .Values.authSecret.github_token }}
   github_token: {{ .Values.authSecret.github_token | toString | b64enc }}
 {{- end }}
+{{- if .Values.authSecret.github_basicauth_password }}
+  github_basicauth_password: {{ .Values.authSecret.github_basicauth_password | toString | b64enc }}
+{{- end }}
 {{- end }}

charts/actions-runner-controller/templates/webhook_configs.yaml

@@ -5,12 +5,22 @@ kind: MutatingWebhookConfiguration
 metadata:
   creationTimestamp: null
   name: {{ include "actions-runner-controller.fullname" . }}-mutating-webhook-configuration
+  {{- if .Values.certManagerEnabled }}
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "actions-runner-controller.servingCertName" . }}
+  {{- end }}
 webhooks:
 - admissionReviewVersions:
   - v1beta1
+  {{- if .Values.scope.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      name: {{ default .Release.Namespace .Values.scope.watchNamespace }}
+  {{- end }}
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ quote .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -30,7 +40,15 @@ webhooks:
   sideEffects: None
 - admissionReviewVersions:
   - v1beta1
+  {{- if .Values.scope.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      name: {{ default .Release.Namespace .Values.scope.watchNamespace }}
+  {{- end }}
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -50,7 +68,15 @@ webhooks:
   sideEffects: None
 - admissionReviewVersions:
   - v1beta1
+  {{- if .Values.scope.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      name: {{ default .Release.Namespace .Values.scope.watchNamespace }}
+  {{- end }}
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -70,7 +96,15 @@ webhooks:
   sideEffects: None
 - admissionReviewVersions:
   - v1beta1
+  {{- if .Values.scope.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      name: {{ default .Release.Namespace .Values.scope.watchNamespace }}
+  {{- end }}
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -96,12 +130,22 @@ kind: ValidatingWebhookConfiguration
 metadata:
   creationTimestamp: null
   name: {{ include "actions-runner-controller.fullname" . }}-validating-webhook-configuration
+  {{- if .Values.certManagerEnabled }}
   annotations:
     cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "actions-runner-controller.servingCertName" . }}
+  {{- end }}
 webhooks:
 - admissionReviewVersions:
   - v1beta1
+  {{- if .Values.scope.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      name: {{ default .Release.Namespace .Values.scope.watchNamespace }}
+  {{- end }}
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -121,7 +165,15 @@ webhooks:
   sideEffects: None
 - admissionReviewVersions:
   - v1beta1
+  {{- if .Values.scope.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      name: {{ default .Release.Namespace .Values.scope.watchNamespace }}
+  {{- end }}
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
@@ -141,7 +193,15 @@ webhooks:
   sideEffects: None
 - admissionReviewVersions:
   - v1beta1
+  {{- if .Values.scope.singleNamespace }}
+  namespaceSelector:
+    matchLabels:
+      name: {{ default .Release.Namespace .Values.scope.watchNamespace }}
+  {{- end }}
   clientConfig:
+    {{- if .Values.admissionWebHooks.caBundle }}
+    caBundle: {{ .Values.admissionWebHooks.caBundle }}
+    {{- end }}
     service:
       name: {{ include "actions-runner-controller.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}

charts/actions-runner-controller/values.yaml

@@ -13,6 +13,7 @@ enableLeaderElection: true
 # Must be unique if more than one controller installed onto the same namespace.
 #leaderElectionId: "actions-runner-controller"
 
+# DEPRECATED: This has been removed as unnecessary in #1192
 # The controller tries its best not to repeat the duplicate GitHub API call
 # within this duration.
 # Defaults to syncPeriod - 10s.
@@ -21,9 +22,21 @@ enableLeaderElection: true
 # The URL of your GitHub Enterprise server, if you're using one.
 #githubEnterpriseServerURL: https://github.example.com
 
+# Override GitHub URLs in case of using proxy APIs
+#githubURL: ""
+#githubUploadURL: ""
+#runnerGithubURL: ""
+
 # Only 1 authentication method can be deployed at a time
 # Uncomment the configuration you are applying and fill in the details
+#
+# If authSecret.enabled=true these values are inherited to actions-runner-controller's controller-manager container's env.
+#
+# Do set authSecret.enabled=false and set env if you want full control over
+# the GitHub authn related envvars of the container.
+# See https://github.com/actions-runner-controller/actions-runner-controller/pull/937 for more details.
 authSecret:
+  enabled: true
   create: false
   name: "controller-manager"
   annotations: {}
@@ -34,6 +47,9 @@ authSecret:
   #github_app_private_key: |
   ### GitHub PAT Configuration
   #github_token: ""
+  ### Basic auth for github API proxy
+  #github_basicauth_username: ""
+  #github_basicauth_password: ""
 
 dockerRegistryMirror: ""
 image:
@@ -41,6 +57,9 @@ image:
   actionsRunnerRepositoryAndTag: "summerwind/actions-runner:latest"
   dindSidecarRepositoryAndTag: "docker:dind"
   pullPolicy: IfNotPresent
+  # The default image-pull secrets name for self-hosted runner container.
+  # It's added to spec.ImagePullSecrets of self-hosted runner pods.
+  actionsRunnerImagePullSecrets: []
 
 imagePullSecrets: []
 nameOverride: ""
@@ -88,7 +107,7 @@ metrics:
     enabled: true
     image:
       repository: quay.io/brancz/kube-rbac-proxy
-      tag: v0.10.0
+      tag: v0.11.0
 
 resources:
   {}
@@ -126,6 +145,14 @@ env:
   # https_proxy: "proxy.com:8080"
   # no_proxy: ""
 
+## specify additional volumes to mount in the manager container, this can be used
+## to specify additional storage of material or to inject files from ConfigMaps
+## into the running container
+additionalVolumes: []
+
+## specify where the additional volumes are mounted in the manager container
+additionalVolumeMounts: []
+
 scope:
   # If true, the controller will only watch custom resources in a single namespace
   singleNamespace: false
@@ -133,11 +160,23 @@ scope:
   # The default value is "", which means the namespace of the controller
   watchNamespace: ""
 
+certManagerEnabled: true
+
+admissionWebHooks:
+  {}
+  #caBundle: "Ci0tLS0tQk...<base64-encoded PEM bundle containing the CA that signed the webhook's serving certificate>...tLS0K"
+
+# There may be alternatives to setting `hostNetwork: true`, see
+# https://github.com/actions-runner-controller/actions-runner-controller/issues/1005#issuecomment-993097155
+#hostNetwork: true
+
 githubWebhookServer:
   enabled: false
   replicaCount: 1
   syncPeriod: 10m
+  useRunnerGroupsVisibility: false
   secret:
+    enabled: false
     create: false
     name: "github-webhook-server"
     ### GitHub Webhook Configuration
@@ -174,13 +213,15 @@ githubWebhookServer:
         #nodePort: someFixedPortForUseWithTerraformCdkCfnEtc
   ingress:
     enabled: false
-    annotations:
-      {}
+    ingressClassName: ""
+    annotations: {}
       # kubernetes.io/ingress.class: nginx
       # kubernetes.io/tls-acme: "true"
     hosts:
       - host: chart-example.local
         paths: []
+        #  - path: /*
+        #    pathType: ImplementationSpecific
     tls: []
     #  - secretName: chart-example-tls
     #    hosts:
@@ -190,4 +231,4 @@ githubWebhookServer:
   podDisruptionBudget:
     enabled: false
     # minAvailable: 1
-    # maxUnavailable: 3
+    # maxUnavailable: 3

cmd/githubwebhookserver/main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"flag"
+	"fmt"
 	"net/http"
 	"os"
 	"sync"
@@ -27,14 +28,15 @@ import (
 
 	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/controllers"
-	zaplib "go.uber.org/zap"
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/actions-runner-controller/actions-runner-controller/logging"
+	"github.com/kelseyhightower/envconfig"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/exec"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -44,10 +46,7 @@ var (
 )
 
 const (
-	logLevelDebug = "debug"
-	logLevelInfo  = "info"
-	logLevelWarn  = "warn"
-	logLevelError = "error"
+	webhookSecretTokenEnvName = "GITHUB_WEBHOOK_SECRET_TOKEN"
 )
 
 func init() {
@@ -65,16 +64,26 @@ func main() {
 		metricsAddr string
 
 		// The secret token of the GitHub Webhook. See https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks
-		webhookSecretToken string
+		webhookSecretToken    string
+		webhookSecretTokenEnv string
 
 		watchNamespace string
 
 		enableLeaderElection bool
 		syncPeriod           time.Duration
 		logLevel             string
+
+		ghClient *github.Client
 	)
 
-	webhookSecretToken = os.Getenv("GITHUB_WEBHOOK_SECRET_TOKEN")
+	var c github.Config
+	err = envconfig.Process("github", &c)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: processing environment variables: %v\n", err)
+		os.Exit(1)
+	}
+
+	webhookSecretTokenEnv = os.Getenv(webhookSecretTokenEnvName)
 
 	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
@@ -82,11 +91,27 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
-	flag.StringVar(&logLevel, "log-level", logLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
+	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
+	flag.StringVar(&webhookSecretToken, "github-webhook-secret-token", "", "The personal access token of GitHub.")
+	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
+	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
+	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
+	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
+	flag.StringVar(&c.URL, "github-url", c.URL, "GitHub URL to be used for GitHub API calls")
+	flag.StringVar(&c.UploadURL, "github-upload-url", c.UploadURL, "GitHub Upload URL to be used for GitHub API calls")
+	flag.StringVar(&c.BasicauthUsername, "github-basicauth-username", c.BasicauthUsername, "Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
+	flag.StringVar(&c.BasicauthPassword, "github-basicauth-password", c.BasicauthPassword, "Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
+	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
+
 	flag.Parse()
 
+	if webhookSecretToken == "" && webhookSecretTokenEnv != "" {
+		setupLog.Info(fmt.Sprintf("Using the value from %s for -github-webhook-secret-token", webhookSecretTokenEnvName))
+		webhookSecretToken = webhookSecretTokenEnv
+	}
+
 	if webhookSecretToken == "" {
-		setupLog.Info("-webhook-secret-token is missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks")
+		setupLog.Info(fmt.Sprintf("-github-webhook-secret-token and %s are missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks and specify it via the flag or the envvar", webhookSecretTokenEnvName))
 	}
 
 	if watchNamespace == "" {
@@ -95,24 +120,28 @@ func main() {
 		setupLog.Info("-watch-namespace is %q. Only HorizontalRunnerAutoscalers in %q are watched, cached, and considered as scale targets.")
 	}
 
-	logger := zap.New(func(o *zap.Options) {
-		switch logLevel {
-		case logLevelDebug:
-			o.Development = true
-		case logLevelInfo:
-			lvl := zaplib.NewAtomicLevelAt(zaplib.InfoLevel)
-			o.Level = &lvl
-		case logLevelWarn:
-			lvl := zaplib.NewAtomicLevelAt(zaplib.WarnLevel)
-			o.Level = &lvl
-		case logLevelError:
-			lvl := zaplib.NewAtomicLevelAt(zaplib.ErrorLevel)
-			o.Level = &lvl
-		}
-	})
+	logger := logging.NewLogger(logLevel)
 
 	ctrl.SetLogger(logger)
 
+	// In order to support runner groups with custom visibility (selected repositories), we need to perform some GitHub API calls.
+	// Let the user define if they want to opt-in supporting this option by providing the proper GitHub authentication parameters
+	// Without an opt-in, runner groups with custom visibility won't be supported to save API calls
+	// That is, all runner groups managed by ARC are assumed to be visible to any repositories,
+	// which is wrong when you have one or more non-default runner groups in your organization or enterprise.
+	if len(c.Token) > 0 || (c.AppID > 0 && c.AppInstallationID > 0 && c.AppPrivateKey != "") || (len(c.BasicauthUsername) > 0 && len(c.BasicauthPassword) > 0) {
+		c.Log = &logger
+
+		ghClient, err = c.NewClient()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "Error: Client creation failed.", err)
+			setupLog.Error(err, "unable to create controller", "controller", "Runner")
+			os.Exit(1)
+		}
+	} else {
+		setupLog.Info("GitHub client is not initialized. Runner groups with custom visibility are not supported. If needed, please provide GitHub authentication. This will incur in extra GitHub API calls")
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		SyncPeriod:         &syncPeriod,
@@ -127,16 +156,18 @@ func main() {
 	}
 
 	hraGitHubWebhook := &controllers.HorizontalRunnerAutoscalerGitHubWebhook{
+		Name:           "webhookbasedautoscaler",
 		Client:         mgr.GetClient(),
-		Log:            ctrl.Log.WithName("controllers").WithName("Runner"),
+		Log:            ctrl.Log.WithName("controllers").WithName("webhookbasedautoscaler"),
 		Recorder:       nil,
 		Scheme:         mgr.GetScheme(),
 		SecretKeyBytes: []byte(webhookSecretToken),
 		Namespace:      watchNamespace,
+		GitHubClient:   ghClient,
 	}
 
 	if err = hraGitHubWebhook.SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "Runner")
+		setupLog.Error(err, "unable to create controller", "controller", "webhookbasedautoscaler")
 		os.Exit(1)
 	}
 

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.6.0
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: horizontalrunnerautoscalers.actions.summerwind.dev
 spec:
@@ -49,6 +49,9 @@ spec:
                   items:
                     description: CapacityReservation specifies the number of replicas temporarily added to the scale target until ExpirationTime.
                     properties:
+                      effectiveTime:
+                        format: date-time
+                        type: string
                       expirationTime:
                         format: date-time
                         type: string

controllers/autoscaling.go

@@ -7,9 +7,9 @@ import (
 	"math"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/google/go-github/v39/github"
 )
 
 const (
@@ -19,47 +19,6 @@ const (
 	defaultScaleDownFactor    = 0.7
 )
 
-func getValueAvailableAt(now time.Time, from, to *time.Time, reservedValue int) *int {
-	if to != nil && now.After(*to) {
-		return nil
-	}
-
-	if from != nil && now.Before(*from) {
-		return nil
-	}
-
-	return &reservedValue
-}
-
-func (r *HorizontalRunnerAutoscalerReconciler) fetchSuggestedReplicasFromCache(hra v1alpha1.HorizontalRunnerAutoscaler) *int {
-	var entry *v1alpha1.CacheEntry
-
-	for i := range hra.Status.CacheEntries {
-		ent := hra.Status.CacheEntries[i]
-
-		if ent.Key != v1alpha1.CacheEntryKeyDesiredReplicas {
-			continue
-		}
-
-		if !time.Now().Before(ent.ExpirationTime.Time) {
-			continue
-		}
-
-		entry = &ent
-
-		break
-	}
-
-	if entry != nil {
-		v := getValueAvailableAt(time.Now(), nil, &entry.ExpirationTime.Time, entry.Value)
-		if v != nil {
-			return v
-		}
-	}
-
-	return nil
-}
-
 func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	if hra.Spec.MinReplicas == nil {
 		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
@@ -164,14 +123,24 @@ func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgr
 			fallback_cb()
 			return
 		}
-		jobs, _, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, nil)
-		if err != nil {
-			r.Log.Error(err, "Error listing workflow jobs")
-			fallback_cb()
-		} else if len(jobs.Jobs) == 0 {
+		opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}}
+		var allJobs []*github.WorkflowJob
+		for {
+			jobs, resp, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
+			if err != nil {
+				r.Log.Error(err, "Error listing workflow jobs")
+				return //err
+			}
+			allJobs = append(allJobs, jobs.Jobs...)
+			if resp.NextPage == 0 {
+				break
+			}
+			opt.Page = resp.NextPage
+		}
+		if len(allJobs) == 0 {
 			fallback_cb()
 		} else {
-			for _, job := range jobs.Jobs {
+			for _, job := range allJobs {
 				switch job.GetStatus() {
 				case "completed":
 					// We add a case for `completed` so it is not counted in `unknown`.

controllers/autoscaling_test.go

@@ -234,7 +234,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, _, _, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)
@@ -502,7 +502,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 			st := h.scaleTargetFromRD(context.Background(), rd)
 
-			got, _, _, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
+			got, err := h.computeReplicasWithCache(log, metav1Now.Time, st, hra, minReplicas)
 			if err != nil {
 				if tc.err == "" {
 					t.Fatalf("unexpected error: expected none, got %v", err)

controllers/constants.go

@@ -0,0 +1,66 @@
+package controllers
+
+import "time"
+
+const (
+	LabelKeyRunnerSetName = "runnerset-name"
+)
+
+const (
+	// This names requires at least one slash to work.
+	// See https://github.com/google/knative-gcp/issues/378
+	runnerPodFinalizerName = "actions.summerwind.dev/runner-pod"
+
+	annotationKeyPrefix = "actions-runner/"
+
+	AnnotationKeyLastRegistrationCheckTime = "actions-runner-controller/last-registration-check-time"
+
+	// AnnotationKeyUnregistrationCompleteTimestamp is the annotation that is added onto the pod once the previously started unregistration process has been completed.
+	AnnotationKeyUnregistrationCompleteTimestamp = annotationKeyPrefix + "unregistration-complete-timestamp"
+
+	// AnnotationKeyRunnerCompletionWaitStartTimestamp is the annotation that is added onto the pod when
+	// ARC decided to wait until the pod to complete by itself, without the need for ARC to unregister the corresponding runner.
+	AnnotationKeyRunnerCompletionWaitStartTimestamp = annotationKeyPrefix + "runner-completion-wait-start-timestamp"
+
+	// unregistarionStartTimestamp is the annotation that contains the time that the requested unregistration process has been started
+	AnnotationKeyUnregistrationStartTimestamp = annotationKeyPrefix + "unregistration-start-timestamp"
+
+	// AnnotationKeyUnregistrationRequestTimestamp is the annotation that contains the time that the unregistration has been requested.
+	// This doesn't immediately start the unregistration. Instead, ARC will first check if the runner has already been registered.
+	// If not, ARC will hold on until the registration to complete first, and only after that it starts the unregistration process.
+	// This is crucial to avoid a race between ARC marking the runner pod for deletion while the actions-runner registers itself to GitHub, leaving the assigned job
+	// hang like forever.
+	AnnotationKeyUnregistrationRequestTimestamp = annotationKeyPrefix + "unregistration-request-timestamp"
+
+	AnnotationKeyRunnerID = annotationKeyPrefix + "id"
+
+	// This can be any value but a larger value can make an unregistration timeout longer than configured in practice.
+	DefaultUnregistrationRetryDelay = time.Minute
+
+	// RetryDelayOnCreateRegistrationError is the delay between retry attempts for runner registration token creation.
+	// Usually, a retry in this case happens when e.g. your PAT has no access to certain scope of runners, like you're using repository admin's token
+	// for creating a broader scoped runner token, like organizationa or enterprise runner token.
+	// Such permission issue will never fixed automatically, so we don't need to retry so often, hence this value.
+	RetryDelayOnCreateRegistrationError = 3 * time.Minute
+
+	// registrationTimeout is the duration until a pod times out after it becomes Ready and Running.
+	// A pod that is timed out can be terminated if needed.
+	registrationTimeout = 10 * time.Minute
+
+	defaultRegistrationCheckInterval = time.Minute
+
+	// DefaultRunnerPodRecreationDelayAfterWebhookScale is the delay until syncing the runners with the desired replicas
+	// after a webhook-based scale up.
+	// This is used to prevent ARC from recreating completed runner pods that are deleted soon without being used at all.
+	// In other words, this is used as a timer to wait for the completed runner to emit the next `workflow_job` webhook event to decrease the desired replicas.
+	// So if we set 30 seconds for this, you are basically saying that you would assume GitHub and your installation of ARC to
+	// emit and propagate a workflow_job completion event down to the RunnerSet or RunnerReplicaSet, vha ARC's github webhook server and HRA, in approximately 30 seconds.
+	// In case it actually took more than DefaultRunnerPodRecreationDelayAfterWebhookScale for the workflow_job completion event to arrive,
+	// ARC will recreate the completed runner(s), assuming something went wrong in either GitHub, your K8s cluster, or ARC, so ARC needs to resync anyway.
+	//
+	// See https://github.com/actions-runner-controller/actions-runner-controller/pull/1180
+	DefaultRunnerPodRecreationDelayAfterWebhookScale = 10 * time.Minute
+
+	EnvVarRunnerName  = "RUNNER_NAME"
+	EnvVarRunnerToken = "RUNNER_TOKEN"
+)

controllers/horizontal_runner_autoscaler_webhook.go

@@ -30,19 +30,22 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/go-logr/logr"
-	gogithub "github.com/google/go-github/v37/github"
+	gogithub "github.com/google/go-github/v39/github"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/actions-runner-controller/actions-runner-controller/simulator"
 )
 
 const (
 	scaleTargetKey = "scaleTarget"
 
 	keyPrefixEnterprise = "enterprises/"
+	keyRunnerGroup      = "/group/"
 )
 
 // HorizontalRunnerAutoscalerGitHubWebhook autoscales a HorizontalRunnerAutoscaler and the RunnerDeployment on each
@@ -57,6 +60,9 @@ type HorizontalRunnerAutoscalerGitHubWebhook struct {
 	// the administrator is generated and specified in GitHub Web UI.
 	SecretKeyBytes []byte
 
+	// GitHub Client to discover runner groups assigned to a repository
+	GitHubClient *github.Client
+
 	// Namespace is the namespace to watch for HorizontalRunnerAutoscaler's to be
 	// scaled on Webhook.
 	// Set to empty for letting it watch for all namespaces.
@@ -87,7 +93,7 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.Respons
 			if err != nil {
 				msg := err.Error()
 				if written, err := w.Write([]byte(msg)); err != nil {
-					autoscaler.Log.Error(err, "failed writing http error response", "msg", msg, "written", written)
+					autoscaler.Log.V(1).Error(err, "failed writing http error response", "msg", msg, "written", written)
 				}
 			}
 		}
@@ -284,7 +290,7 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.Respons
 	}
 
 	if target == nil {
-		log.Info(
+		log.V(1).Info(
 			"Scale target not found. If this is unexpected, ensure that there is exactly one repository-wide or organizational runner deployment that matches this webhook event",
 		)
 
@@ -345,9 +351,7 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) findHRAsByKey(ctx con
 			return nil, err
 		}
 
-		for _, d := range hraList.Items {
-			hras = append(hras, d)
-		}
+		hras = append(hras, hraList.Items...)
 	}
 
 	return hras, nil
@@ -436,63 +440,30 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleTarget(ctx co
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTarget(ctx context.Context, log logr.Logger, repo, owner, ownerType, enterprise string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) {
-	repositoryRunnerKey := owner + "/" + repo
-
-	if target, err := autoscaler.getScaleTarget(ctx, repositoryRunnerKey, f); err != nil {
-		log.Info("finding repository-wide runner", "repository", repositoryRunnerKey)
-		return nil, err
-	} else if target != nil {
-		log.Info("scale up target is repository-wide runners", "repository", repo)
-		return target, nil
+	scaleTarget := func(value string) (*ScaleTarget, error) {
+		return autoscaler.getScaleTarget(ctx, value, f)
 	}
-
-	if ownerType == "User" {
-		log.V(1).Info("no repository runner found", "organization", owner)
-
-		return nil, nil
-	}
-
-	if target, err := autoscaler.getScaleTarget(ctx, owner, f); err != nil {
-		log.Info("finding organizational runner", "organization", owner)
-		return nil, err
-	} else if target != nil {
-		log.Info("scale up target is organizational runners", "organization", owner)
-		return target, nil
-	}
-
-	if enterprise == "" {
-		log.V(1).Info("no repository runner or organizational runner found",
-			"repository", repositoryRunnerKey,
-			"organization", owner,
-		)
-
-		return nil, nil
-	}
-
-	if target, err := autoscaler.getScaleTarget(ctx, enterpriseKey(enterprise), f); err != nil {
-		log.Error(err, "finding enterprise runner", "enterprise", enterprise)
-		return nil, err
-	} else if target != nil {
-		log.Info("scale up target is enterprise runners", "enterprise", enterprise)
-		return target, nil
-	} else {
-		log.V(1).Info("no repository/organizational/enterprise runner found",
-			"repository", repositoryRunnerKey,
-			"organization", owner,
-			"enterprises", enterprise,
-		)
-	}
-
-	return nil, nil
+	return autoscaler.getScaleUpTargetWithFunction(ctx, log, repo, owner, ownerType, enterprise, scaleTarget)
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getJobScaleUpTargetForRepoOrOrg(
 	ctx context.Context, log logr.Logger, repo, owner, ownerType, enterprise string, labels []string,
 ) (*ScaleTarget, error) {
+
+	scaleTarget := func(value string) (*ScaleTarget, error) {
+		return autoscaler.getJobScaleTarget(ctx, value, labels)
+	}
+	return autoscaler.getScaleUpTargetWithFunction(ctx, log, repo, owner, ownerType, enterprise, scaleTarget)
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTargetWithFunction(
+	ctx context.Context, log logr.Logger, repo, owner, ownerType, enterprise string, scaleTarget func(value string) (*ScaleTarget, error)) (*ScaleTarget, error) {
+
 	repositoryRunnerKey := owner + "/" + repo
 
-	if target, err := autoscaler.getJobScaleTarget(ctx, repositoryRunnerKey, labels); err != nil {
-		log.Info("finding repository-wide runner", "repository", repositoryRunnerKey)
+	// Search for repository HRAs
+	if target, err := scaleTarget(repositoryRunnerKey); err != nil {
+		log.Error(err, "finding repository-wide runner", "repository", repositoryRunnerKey)
 		return nil, err
 	} else if target != nil {
 		log.Info("job scale up target is repository-wide runners", "repository", repo)
@@ -500,42 +471,180 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getJobScaleUpTargetFo
 	}
 
 	if ownerType == "User" {
-		log.V(1).Info("no repository runner found", "organization", owner)
-
+		log.V(1).Info("user repositories not supported", "owner", owner)
 		return nil, nil
 	}
 
-	if target, err := autoscaler.getJobScaleTarget(ctx, owner, labels); err != nil {
-		log.Info("finding organizational runner", "organization", owner)
+	// Find the potential runner groups first to avoid spending API queries needless. Once/if GitHub improves an
+	// API to find related/linked runner groups from a specific repository this logic could be removed
+	managedRunnerGroups, err := autoscaler.getManagedRunnerGroupsFromHRAs(ctx, enterprise, owner)
+	if err != nil {
+		log.Error(err, "finding potential organization/enterprise runner groups from HRAs", "organization", owner)
 		return nil, err
-	} else if target != nil {
-		log.Info("job scale up target is organizational runners", "organization", owner)
-		return target, nil
 	}
-
-	if enterprise == "" {
-		log.V(1).Info("no repository runner or organizational runner found",
-			"repository", repositoryRunnerKey,
-			"organization", owner,
-		)
-		return nil, nil
-	}
-
-	if target, err := autoscaler.getJobScaleTarget(ctx, enterpriseKey(enterprise), labels); err != nil {
-		log.Error(err, "finding enterprise runner", "enterprise", enterprise)
-		return nil, err
-	} else if target != nil {
-		log.Info("scale up target is enterprise runners", "enterprise", enterprise)
-		return target, nil
-	} else {
+	if managedRunnerGroups.IsEmpty() {
 		log.V(1).Info("no repository/organizational/enterprise runner found",
 			"repository", repositoryRunnerKey,
 			"organization", owner,
 			"enterprises", enterprise,
 		)
+	} else {
+		log.V(1).Info("Found some runner groups are managed by ARC", "groups", managedRunnerGroups)
 	}
 
-	return nil, nil
+	var visibleGroups *simulator.VisibleRunnerGroups
+	if autoscaler.GitHubClient != nil {
+		simu := &simulator.Simulator{
+			Client: autoscaler.GitHubClient,
+		}
+		// Get available organization runner groups and enterprise runner groups for a repository
+		// These are the sum of runner groups with repository access = All repositories and runner groups
+		// where owner/repo has access to as well. The list will include default runner group also if it has access to
+		visibleGroups, err = simu.GetRunnerGroupsVisibleToRepository(ctx, owner, repositoryRunnerKey, managedRunnerGroups)
+		log.V(1).Info("Searching in runner groups", "groups", visibleGroups)
+		if err != nil {
+			log.Error(err, "Unable to find runner groups from repository", "organization", owner, "repository", repo)
+			return nil, fmt.Errorf("error while finding visible runner groups: %v", err)
+		}
+	} else {
+		// For backwards compatibility if GitHub authentication is not configured, we assume all runner groups have
+		// visibility=all to honor the previous implementation, therefore any available enterprise/organization runner
+		// is a potential target for scaling. This will also avoid doing extra API calls caused by
+		// GitHubClient.GetRunnerGroupsVisibleToRepository in case users are not using custom visibility on their runner
+		// groups or they are using only default runner groups
+		visibleGroups = managedRunnerGroups
+	}
+
+	scaleTargetKey := func(rg simulator.RunnerGroup) string {
+		switch rg.Kind {
+		case simulator.Default:
+			switch rg.Scope {
+			case simulator.Organization:
+				return owner
+			case simulator.Enterprise:
+				return enterpriseKey(enterprise)
+			}
+		case simulator.Custom:
+			switch rg.Scope {
+			case simulator.Organization:
+				return organizationalRunnerGroupKey(owner, rg.Name)
+			case simulator.Enterprise:
+				return enterpriseRunnerGroupKey(enterprise, rg.Name)
+			}
+		}
+		return ""
+	}
+
+	log.V(1).Info("groups", "groups", visibleGroups)
+
+	var t *ScaleTarget
+
+	traverseErr := visibleGroups.Traverse(func(rg simulator.RunnerGroup) (bool, error) {
+		key := scaleTargetKey(rg)
+
+		target, err := scaleTarget(key)
+
+		if err != nil {
+			log.Error(err, "finding runner group", "enterprise", enterprise, "organization", owner, "repository", repo, "key", key)
+			return false, err
+		} else if target == nil {
+			return false, nil
+		}
+
+		t = target
+		log.V(1).Info("job scale up target found", "enterprise", enterprise, "organization", owner, "repository", repo, "key", key)
+
+		return true, nil
+	})
+
+	if traverseErr != nil {
+		return nil, err
+	}
+
+	if t == nil {
+		log.V(1).Info("no repository/organizational/enterprise runner found",
+			"repository", repositoryRunnerKey,
+			"organization", owner,
+			"enterprise", enterprise,
+		)
+	}
+
+	return t, nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getManagedRunnerGroupsFromHRAs(ctx context.Context, enterprise, org string) (*simulator.VisibleRunnerGroups, error) {
+	groups := simulator.NewVisibleRunnerGroups()
+	ns := autoscaler.Namespace
+
+	var defaultListOpts []client.ListOption
+	if ns != "" {
+		defaultListOpts = append(defaultListOpts, client.InNamespace(ns))
+	}
+
+	opts := append([]client.ListOption{}, defaultListOpts...)
+	if autoscaler.Namespace != "" {
+		opts = append(opts, client.InNamespace(autoscaler.Namespace))
+	}
+
+	var hraList v1alpha1.HorizontalRunnerAutoscalerList
+	if err := autoscaler.List(ctx, &hraList, opts...); err != nil {
+		return groups, err
+	}
+
+	for _, hra := range hraList.Items {
+		var o, e, g string
+
+		kind := hra.Spec.ScaleTargetRef.Kind
+		switch kind {
+		case "RunnerSet":
+			var rs v1alpha1.RunnerSet
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rs); err != nil {
+				return groups, err
+			}
+			o, e, g = rs.Spec.Organization, rs.Spec.Enterprise, rs.Spec.Group
+		case "RunnerDeployment", "":
+			var rd v1alpha1.RunnerDeployment
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil {
+				return groups, err
+			}
+			o, e, g = rd.Spec.Template.Spec.Organization, rd.Spec.Template.Spec.Enterprise, rd.Spec.Template.Spec.Group
+		default:
+			return nil, fmt.Errorf("unsupported scale target kind: %v", kind)
+		}
+
+		if g != "" && e == "" && o == "" {
+			autoscaler.Log.V(1).Info(
+				"invalid runner group config in scale target: spec.group must be set along with either spec.enterprise or spec.organization",
+				"scaleTargetKind", kind,
+				"group", g,
+				"enterprise", e,
+				"organization", o,
+			)
+
+			continue
+		}
+
+		if e != enterprise && o != org {
+			autoscaler.Log.V(1).Info(
+				"Skipped scale target irrelevant to event",
+				"eventOrganization", org,
+				"eventEnterprise", enterprise,
+				"scaleTargetKind", kind,
+				"scaleTargetGroup", g,
+				"scaleTargetEnterprise", e,
+				"scaleTargetOrganization", o,
+			)
+
+			continue
+		}
+
+		rg := simulator.NewRunnerGroupFromProperties(e, o, g)
+
+		if err := groups.Add(rg); err != nil {
+			return groups, fmt.Errorf("failed adding visible group from HRA %s/%s: %w", hra.Namespace, hra.Name, err)
+		}
+	}
+	return groups, nil
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getJobScaleTarget(ctx context.Context, name string, labels []string) (*ScaleTarget, error) {
@@ -580,13 +689,17 @@ HRA:
 				return nil, err
 			}
 
-			if len(labels) == 1 && labels[0] == "self-hosted" {
-				return &ScaleTarget{HorizontalRunnerAutoscaler: hra, ScaleUpTrigger: v1alpha1.ScaleUpTrigger{Duration: duration}}, nil
-			}
-
 			// Ensure that the RunnerSet-managed runners have all the labels requested by the workflow_job.
 			for _, l := range labels {
 				var matched bool
+
+				// ignore "self-hosted" label as all instance here are self-hosted
+				if l == "self-hosted" {
+					continue
+				}
+
+				// TODO labels related to OS and architecture needs to be explicitly declared or the current implementation will not be able to find them.
+
 				for _, l2 := range rs.Spec.Labels {
 					if l == l2 {
 						matched = true
@@ -607,13 +720,17 @@ HRA:
 				return nil, err
 			}
 
-			if len(labels) == 1 && labels[0] == "self-hosted" {
-				return &ScaleTarget{HorizontalRunnerAutoscaler: hra, ScaleUpTrigger: v1alpha1.ScaleUpTrigger{Duration: duration}}, nil
-			}
-
 			// Ensure that the RunnerDeployment-managed runners have all the labels requested by the workflow_job.
 			for _, l := range labels {
 				var matched bool
+
+				// ignore "self-hosted" label as all instance here are self-hosted
+				if l == "self-hosted" {
+					continue
+				}
+
+				// TODO labels related to OS and architecture needs to be explicitly declared or the current implementation will not be able to find them.
+
 				for _, l2 := range rd.Spec.Template.Spec.Labels {
 					if l == l2 {
 						matched = true
@@ -651,8 +768,10 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScale(ctx context.
 	capacityReservations := getValidCapacityReservations(copy)
 
 	if amount > 0 {
+		now := time.Now()
 		copy.Spec.CapacityReservations = append(capacityReservations, v1alpha1.CapacityReservation{
-			ExpirationTime: metav1.Time{Time: time.Now().Add(target.ScaleUpTrigger.Duration.Duration)},
+			EffectiveTime:  metav1.Time{Time: now},
+			ExpirationTime: metav1.Time{Time: now.Add(target.ScaleUpTrigger.Duration.Duration)},
 			Replicas:       amount,
 		})
 	} else if amount < 0 {
@@ -671,10 +790,16 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScale(ctx context.
 		copy.Spec.CapacityReservations = reservations
 	}
 
-	autoscaler.Log.Info(
-		"Patching hra for capacityReservations update",
-		"before", target.HorizontalRunnerAutoscaler.Spec.CapacityReservations,
-		"after", copy.Spec.CapacityReservations,
+	before := len(target.HorizontalRunnerAutoscaler.Spec.CapacityReservations)
+	expired := before - len(capacityReservations)
+	after := len(copy.Spec.CapacityReservations)
+
+	autoscaler.Log.V(1).Info(
+		fmt.Sprintf("Patching hra %s for capacityReservations update", target.HorizontalRunnerAutoscaler.Name),
+		"before", before,
+		"expired", expired,
+		"amount", amount,
+		"after", after,
 	)
 
 	if err := autoscaler.Client.Patch(ctx, copy, client.MergeFrom(&target.HorizontalRunnerAutoscaler)); err != nil {
@@ -710,37 +835,62 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr
 		hra := rawObj.(*v1alpha1.HorizontalRunnerAutoscaler)
 
 		if hra.Spec.ScaleTargetRef.Name == "" {
+			autoscaler.Log.V(1).Info(fmt.Sprintf("scale target ref name not set for hra %s", hra.Name))
 			return nil
 		}
 
 		switch hra.Spec.ScaleTargetRef.Kind {
 		case "", "RunnerDeployment":
 			var rd v1alpha1.RunnerDeployment
-
 			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil {
+				autoscaler.Log.V(1).Info(fmt.Sprintf("RunnerDeployment not found with scale target ref name %s for hra %s", hra.Spec.ScaleTargetRef.Name, hra.Name))
 				return nil
 			}
 
-			keys := []string{rd.Spec.Template.Spec.Repository, rd.Spec.Template.Spec.Organization}
-
-			if enterprise := rd.Spec.Template.Spec.Enterprise; enterprise != "" {
-				keys = append(keys, enterpriseKey(enterprise))
+			keys := []string{}
+			if rd.Spec.Template.Spec.Repository != "" {
+				keys = append(keys, rd.Spec.Template.Spec.Repository) // Repository runners
 			}
-
+			if rd.Spec.Template.Spec.Organization != "" {
+				if group := rd.Spec.Template.Spec.Group; group != "" {
+					keys = append(keys, organizationalRunnerGroupKey(rd.Spec.Template.Spec.Organization, rd.Spec.Template.Spec.Group)) // Organization runner groups
+				} else {
+					keys = append(keys, rd.Spec.Template.Spec.Organization) // Organization runners
+				}
+			}
+			if enterprise := rd.Spec.Template.Spec.Enterprise; enterprise != "" {
+				if group := rd.Spec.Template.Spec.Group; group != "" {
+					keys = append(keys, enterpriseRunnerGroupKey(enterprise, rd.Spec.Template.Spec.Group)) // Enterprise runner groups
+				} else {
+					keys = append(keys, enterpriseKey(enterprise)) // Enterprise runners
+				}
+			}
+			autoscaler.Log.V(2).Info(fmt.Sprintf("HRA keys indexed for HRA %s: %v", hra.Name, keys))
 			return keys
 		case "RunnerSet":
 			var rs v1alpha1.RunnerSet
-
 			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rs); err != nil {
+				autoscaler.Log.V(1).Info(fmt.Sprintf("RunnerSet not found with scale target ref name %s for hra %s", hra.Spec.ScaleTargetRef.Name, hra.Name))
 				return nil
 			}
 
-			keys := []string{rs.Spec.Repository, rs.Spec.Organization}
-
-			if enterprise := rs.Spec.Enterprise; enterprise != "" {
-				keys = append(keys, enterpriseKey(enterprise))
+			keys := []string{}
+			if rs.Spec.Repository != "" {
+				keys = append(keys, rs.Spec.Repository) // Repository runners
 			}
-
+			if rs.Spec.Organization != "" {
+				keys = append(keys, rs.Spec.Organization) // Organization runners
+				if group := rs.Spec.Group; group != "" {
+					keys = append(keys, organizationalRunnerGroupKey(rs.Spec.Organization, rs.Spec.Group)) // Organization runner groups
+				}
+			}
+			if enterprise := rs.Spec.Enterprise; enterprise != "" {
+				keys = append(keys, enterpriseKey(enterprise)) // Enterprise runners
+				if group := rs.Spec.Group; group != "" {
+					keys = append(keys, enterpriseRunnerGroupKey(enterprise, rs.Spec.Group)) // Enterprise runner groups
+				}
+			}
+			autoscaler.Log.V(2).Info(fmt.Sprintf("HRA keys indexed for HRA %s: %v", hra.Name, keys))
 			return keys
 		}
 
@@ -758,3 +908,11 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr
 func enterpriseKey(name string) string {
 	return keyPrefixEnterprise + name
 }
+
+func organizationalRunnerGroupKey(owner, group string) string {
+	return owner + keyRunnerGroup + group
+}
+
+func enterpriseRunnerGroupKey(enterprise, group string) string {
+	return keyPrefixEnterprise + enterprise + keyRunnerGroup + group
+}

controllers/horizontal_runner_autoscaler_webhook_on_check_run.go

@@ -3,7 +3,7 @@ package controllers
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/pkg/actionsglob"
-	"github.com/google/go-github/v37/github"
+	"github.com/google/go-github/v39/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchCheckRunEvent(event *github.CheckRunEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v37/github"
+	"github.com/google/go-github/v39/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPullRequestEvent(event *github.PullRequestEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_on_push.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-github/v37/github"
+	"github.com/google/go-github/v39/github"
 )
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event *github.PushEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {

controllers/horizontal_runner_autoscaler_webhook_test.go

@@ -15,7 +15,7 @@ import (
 
 	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v37/github"
+	"github.com/google/go-github/v39/github"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -253,6 +253,145 @@ func TestWebhookWorkflowJob(t *testing.T) {
 	})
 }
 
+func TestWebhookWorkflowJobWithSelfHostedLabel(t *testing.T) {
+	setupTest := func() github.WorkflowJobEvent {
+		f, err := os.Open("testdata/org_webhook_workflow_job_with_self_hosted_label_payload.json")
+		if err != nil {
+			t.Fatalf("could not open the fixture: %s", err)
+		}
+		defer f.Close()
+		var e github.WorkflowJobEvent
+		if err := json.NewDecoder(f).Decode(&e); err != nil {
+			t.Fatalf("invalid json: %s", err)
+		}
+
+		return e
+	}
+	t.Run("Successful", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"label1"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"scaled test-name by 1",
+			initObjs,
+		)
+	})
+	t.Run("WrongLabels", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"bad-label"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"no horizontalrunnerautoscaler to scale for this github event",
+			initObjs,
+		)
+	})
+	// This test verifies that the old way of matching labels doesn't work anymore
+	t.Run("OldLabels", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"label1": "label1",
+						},
+					},
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"bad-label"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"no horizontalrunnerautoscaler to scale for this github event",
+			initObjs,
+		)
+	})
+}
+
 func TestGetRequest(t *testing.T) {
 	hra := HorizontalRunnerAutoscalerGitHubWebhook{}
 	request, _ := http.NewRequest(http.MethodGet, "/", nil)
@@ -306,12 +445,14 @@ func TestGetValidCapacityReservations(t *testing.T) {
 func installTestLogger(webhook *HorizontalRunnerAutoscalerGitHubWebhook) *bytes.Buffer {
 	logs := &bytes.Buffer{}
 
-	log := testLogger{
+	sink := &testLogSink{
 		name:   "testlog",
 		writer: logs,
 	}
 
-	webhook.Log = &log
+	log := logr.New(sink)
+
+	webhook.Log = log
 
 	return logs
 }
@@ -398,18 +539,22 @@ func sendWebhook(server *httptest.Server, eventType string, event interface{}) (
 	return http.DefaultClient.Do(req)
 }
 
-// testLogger is a sample logr.Logger that logs in-memory.
+// testLogSink is a sample logr.Logger that logs in-memory.
 // It's only for testing log outputs.
-type testLogger struct {
+type testLogSink struct {
 	name      string
 	keyValues map[string]interface{}
 
 	writer io.Writer
 }
 
-var _ logr.Logger = &testLogger{}
+var _ logr.LogSink = &testLogSink{}
 
-func (l *testLogger) Info(msg string, kvs ...interface{}) {
+func (l *testLogSink) Init(_ logr.RuntimeInfo) {
+
+}
+
+func (l *testLogSink) Info(_ int, msg string, kvs ...interface{}) {
 	fmt.Fprintf(l.writer, "%s] %s\t", l.name, msg)
 	for k, v := range l.keyValues {
 		fmt.Fprintf(l.writer, "%s=%+v ", k, v)
@@ -420,28 +565,24 @@ func (l *testLogger) Info(msg string, kvs ...interface{}) {
 	fmt.Fprintf(l.writer, "\n")
 }
 
-func (_ *testLogger) Enabled() bool {
+func (_ *testLogSink) Enabled(level int) bool {
 	return true
 }
 
-func (l *testLogger) Error(err error, msg string, kvs ...interface{}) {
+func (l *testLogSink) Error(err error, msg string, kvs ...interface{}) {
 	kvs = append(kvs, "error", err)
-	l.Info(msg, kvs...)
+	l.Info(0, msg, kvs...)
 }
 
-func (l *testLogger) V(_ int) logr.InfoLogger {
-	return l
-}
-
-func (l *testLogger) WithName(name string) logr.Logger {
-	return &testLogger{
+func (l *testLogSink) WithName(name string) logr.LogSink {
+	return &testLogSink{
 		name:      l.name + "." + name,
 		keyValues: l.keyValues,
 		writer:    l.writer,
 	}
 }
 
-func (l *testLogger) WithValues(kvs ...interface{}) logr.Logger {
+func (l *testLogSink) WithValues(kvs ...interface{}) logr.LogSink {
 	newMap := make(map[string]interface{}, len(l.keyValues)+len(kvs)/2)
 	for k, v := range l.keyValues {
 		newMap[k] = v
@@ -449,7 +590,7 @@ func (l *testLogger) WithValues(kvs ...interface{}) logr.Logger {
 	for i := 0; i < len(kvs); i += 2 {
 		newMap[kvs[i].(string)] = kvs[i+1]
 	}
-	return &testLogger{
+	return &testLogSink{
 		name:      l.name,
 		keyValues: newMap,
 		writer:    l.writer,

controllers/horizontalrunnerautoscaler_controller.go

@@ -25,10 +25,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/go-logr/logr"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 
-	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -99,11 +99,33 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(ctx context.Context, re
 		return r.reconcile(ctx, req, log, hra, st, func(newDesiredReplicas int) error {
 			currentDesiredReplicas := getIntOrDefault(rd.Spec.Replicas, defaultReplicas)
 
+			ephemeral := rd.Spec.Template.Spec.Ephemeral == nil || *rd.Spec.Template.Spec.Ephemeral
+
+			var effectiveTime *time.Time
+
+			for _, r := range hra.Spec.CapacityReservations {
+				t := r.EffectiveTime
+				if effectiveTime == nil || effectiveTime.Before(t.Time) {
+					effectiveTime = &t.Time
+				}
+			}
+
 			// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
 			if currentDesiredReplicas != newDesiredReplicas {
 				copy := rd.DeepCopy()
 				copy.Spec.Replicas = &newDesiredReplicas
 
+				if ephemeral && effectiveTime != nil {
+					copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+				}
+
+				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rd)); err != nil {
+					return fmt.Errorf("patching runnerdeployment to have %d replicas: %w", newDesiredReplicas, err)
+				}
+			} else if ephemeral && effectiveTime != nil {
+				copy := rd.DeepCopy()
+				copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+
 				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rd)); err != nil {
 					return fmt.Errorf("patching runnerdeployment to have %d replicas: %w", newDesiredReplicas, err)
 				}
@@ -180,15 +202,38 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(ctx context.Context, re
 			}
 			currentDesiredReplicas := getIntOrDefault(replicas, defaultReplicas)
 
+			ephemeral := rs.Spec.Ephemeral == nil || *rs.Spec.Ephemeral
+
+			var effectiveTime *time.Time
+
+			for _, r := range hra.Spec.CapacityReservations {
+				t := r.EffectiveTime
+				if effectiveTime == nil || effectiveTime.Before(t.Time) {
+					effectiveTime = &t.Time
+				}
+			}
+
 			if currentDesiredReplicas != newDesiredReplicas {
 				copy := rs.DeepCopy()
 				v := int32(newDesiredReplicas)
 				copy.Spec.Replicas = &v
 
+				if ephemeral && effectiveTime != nil {
+					copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+				}
+
+				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rs)); err != nil {
+					return fmt.Errorf("patching runnerset to have %d replicas: %w", newDesiredReplicas, err)
+				}
+			} else if ephemeral && effectiveTime != nil {
+				copy := rs.DeepCopy()
+				copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+
 				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rs)); err != nil {
 					return fmt.Errorf("patching runnerset to have %d replicas: %w", newDesiredReplicas, err)
 				}
 			}
+
 			return nil
 		})
 	}
@@ -262,7 +307,7 @@ func (r *HorizontalRunnerAutoscalerReconciler) reconcile(ctx context.Context, re
 		return ctrl.Result{}, err
 	}
 
-	newDesiredReplicas, computedReplicas, computedReplicasFromCache, err := r.computeReplicasWithCache(log, now, st, hra, minReplicas)
+	newDesiredReplicas, err := r.computeReplicasWithCache(log, now, st, hra, minReplicas)
 	if err != nil {
 		r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
 
@@ -287,24 +332,6 @@ func (r *HorizontalRunnerAutoscalerReconciler) reconcile(ctx context.Context, re
 		updated.Status.DesiredReplicas = &newDesiredReplicas
 	}
 
-	if computedReplicasFromCache == nil {
-		cacheEntries := getValidCacheEntries(updated, now)
-
-		var cacheDuration time.Duration
-
-		if r.CacheDuration > 0 {
-			cacheDuration = r.CacheDuration
-		} else {
-			cacheDuration = 10 * time.Minute
-		}
-
-		updated.Status.CacheEntries = append(cacheEntries, v1alpha1.CacheEntry{
-			Key:            v1alpha1.CacheEntryKeyDesiredReplicas,
-			Value:          computedReplicas,
-			ExpirationTime: metav1.Time{Time: time.Now().Add(cacheDuration)},
-		})
-	}
-
 	var overridesSummary string
 
 	if (active != nil && upcoming == nil) || (active != nil && upcoming != nil && active.Period.EndTime.Before(upcoming.Period.StartTime)) {
@@ -339,18 +366,6 @@ func (r *HorizontalRunnerAutoscalerReconciler) reconcile(ctx context.Context, re
 	return ctrl.Result{}, nil
 }
 
-func getValidCacheEntries(hra *v1alpha1.HorizontalRunnerAutoscaler, now time.Time) []v1alpha1.CacheEntry {
-	var cacheEntries []v1alpha1.CacheEntry
-
-	for _, ent := range hra.Status.CacheEntries {
-		if ent.ExpirationTime.After(now) {
-			cacheEntries = append(cacheEntries, ent)
-		}
-	}
-
-	return cacheEntries
-}
-
 func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	name := "horizontalrunnerautoscaler-controller"
 	if r.Name != "" {
@@ -443,32 +458,18 @@ func (r *HorizontalRunnerAutoscalerReconciler) getMinReplicas(log logr.Logger, n
 	return minReplicas, active, upcoming, nil
 }
 
-func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, int, *int, error) {
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
 	var suggestedReplicas int
 
-	suggestedReplicasFromCache := r.fetchSuggestedReplicasFromCache(hra)
+	v, err := r.suggestDesiredReplicas(st, hra)
+	if err != nil {
+		return 0, err
+	}
 
-	var cached *int
-
-	if suggestedReplicasFromCache != nil {
-		cached = suggestedReplicasFromCache
-
-		if cached == nil {
-			suggestedReplicas = minReplicas
-		} else {
-			suggestedReplicas = *cached
-		}
+	if v == nil {
+		suggestedReplicas = minReplicas
 	} else {
-		v, err := r.suggestDesiredReplicas(st, hra)
-		if err != nil {
-			return 0, 0, nil, err
-		}
-
-		if v == nil {
-			suggestedReplicas = minReplicas
-		} else {
-			suggestedReplicas = *v
-		}
+		suggestedReplicas = *v
 	}
 
 	var reserved int
@@ -527,8 +528,8 @@ func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr
 		"min", minReplicas,
 	}
 
-	if cached != nil {
-		kvs = append(kvs, "cached", *cached)
+	if maxReplicas := hra.Spec.MaxReplicas; maxReplicas != nil {
+		kvs = append(kvs, "max", *maxReplicas)
 	}
 
 	if scaleDownDelayUntil != nil {
@@ -536,13 +537,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr
 		kvs = append(kvs, "scale_down_delay_until", scaleDownDelayUntil)
 	}
 
-	if maxReplicas := hra.Spec.MaxReplicas; maxReplicas != nil {
-		kvs = append(kvs, "max", *maxReplicas)
-	}
-
 	log.V(1).Info(fmt.Sprintf("Calculated desired replicas of %d", newDesiredReplicas),
 		kvs...,
 	)
 
-	return newDesiredReplicas, suggestedReplicas, suggestedReplicasFromCache, nil
+	return newDesiredReplicas, nil
 }

controllers/horizontalrunnerautoscaler_controller_test.go

@@ -1,50 +0,0 @@
-package controllers
-
-import (
-	"testing"
-	"time"
-
-	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
-	"github.com/google/go-cmp/cmp"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func TestGetValidCacheEntries(t *testing.T) {
-	now := time.Now()
-
-	hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
-		Status: actionsv1alpha1.HorizontalRunnerAutoscalerStatus{
-			CacheEntries: []actionsv1alpha1.CacheEntry{
-				{
-					Key:            "foo",
-					Value:          1,
-					ExpirationTime: metav1.Time{Time: now.Add(-time.Second)},
-				},
-				{
-					Key:            "foo",
-					Value:          2,
-					ExpirationTime: metav1.Time{Time: now},
-				},
-				{
-					Key:            "foo",
-					Value:          3,
-					ExpirationTime: metav1.Time{Time: now.Add(time.Second)},
-				},
-			},
-		},
-	}
-
-	revs := getValidCacheEntries(hra, now)
-
-	counts := map[string]int{}
-
-	for _, r := range revs {
-		counts[r.Key] += r.Value
-	}
-
-	want := map[string]int{"foo": 3}
-
-	if d := cmp.Diff(want, counts); d != "" {
-		t.Errorf("%s", d)
-	}
-}

controllers/integration_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	github2 "github.com/actions-runner-controller/actions-runner-controller/github"
-	"github.com/google/go-github/v37/github"
+	"github.com/google/go-github/v39/github"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
 
@@ -108,8 +108,9 @@ func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
 			RunnerImage:                 "example/runner:test",
 			DockerImage:                 "example/docker:test",
 			Name:                        controllerName("runner"),
-			RegistrationRecheckInterval: time.Millisecond,
-			RegistrationRecheckJitter:   time.Millisecond,
+			RegistrationRecheckInterval: time.Millisecond * 100,
+			RegistrationRecheckJitter:   time.Millisecond * 10,
+			UnregistrationRetryDelay:    1 * time.Second,
 		}
 		err = runnerController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup runner controller")
@@ -268,7 +269,6 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2)
-				ExpectHRAStatusCacheEntryLengthEventuallyEquals(ctx, ns.Name, name, 1)
 			}
 
 			{
@@ -371,7 +371,6 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
 				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3)
-				ExpectHRAStatusCacheEntryLengthEventuallyEquals(ctx, ns.Name, name, 1)
 			}
 
 			{
@@ -1077,24 +1076,172 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 			}
 		})
 
+		It("should be able to scale visible organization runner group with default labels", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+									Group:      "baz",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, rd, "test RunnerDeployment")
+
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(5),
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{},
+								Amount:      1,
+								Duration:    metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler")
+
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+			}
+
+			{
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 2 replicas on first workflow_job webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "pending", "created", []string{"self-hosted"})
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
+			}
+		})
+
+		It("should be able to scale visible organization runner group with custom labels", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+									Group:      "baz",
+									Labels:     []string{"custom-label"},
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, rd, "test RunnerDeployment")
+
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(5),
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{},
+								Amount:      1,
+								Duration:    metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler")
+
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+			}
+
+			{
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 2 replicas on first workflow_job webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "pending", "created", []string{"custom-label"})
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
+			}
+		})
+
 	})
 })
 
-func ExpectHRAStatusCacheEntryLengthEventuallyEquals(ctx context.Context, ns string, name string, value int, optionalDescriptions ...interface{}) {
-	EventuallyWithOffset(
-		1,
-		func() int {
-			var hra actionsv1alpha1.HorizontalRunnerAutoscaler
-
-			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &hra)
-
-			ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to get test HRA resource")
-
-			return len(hra.Status.CacheEntries)
-		},
-		time.Second*5, time.Millisecond*500).Should(Equal(value), optionalDescriptions...)
-}
-
 func ExpectHRADesiredReplicasEquals(ctx context.Context, ns, name string, desired int, optionalDescriptions ...interface{}) {
 	var rd actionsv1alpha1.HorizontalRunnerAutoscaler
 
@@ -1208,6 +1355,28 @@ func (env *testEnvironment) SendUserCheckRunEvent(owner, repo, status, action st
 	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
 }
 
+func (env *testEnvironment) SendWorkflowJobEvent(owner, repo, status, action string, labels []string) {
+	resp, err := sendWebhook(env.webhookServer, "workflow_job", &github.WorkflowJobEvent{
+		Org: &github.Organization{
+			Name: github.String(owner),
+		},
+		WorkflowJob: &github.WorkflowJob{
+			Labels: labels,
+		},
+		Action: github.String("queued"),
+		Repo: &github.Repository{
+			Name: github.String(repo),
+			Owner: &github.User{
+				Login: github.String(owner),
+				Type:  github.String("Organization"),
+			},
+		},
+	})
+
+	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to send check_run event")
+
+	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
+}
 func (env *testEnvironment) SyncRunnerRegistrations() {
 	var runnerList actionsv1alpha1.RunnerList
 

controllers/pod_runner_token_injector.go

@@ -59,9 +59,9 @@ func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Reque
 		return newEmptyResponse()
 	}
 
-	enterprise, okEnterprise := getEnv(runnerContainer, "RUNNER_ENTERPRISE")
-	repo, okRepo := getEnv(runnerContainer, "RUNNER_REPO")
-	org, okOrg := getEnv(runnerContainer, "RUNNER_ORG")
+	enterprise, okEnterprise := getEnv(runnerContainer, EnvVarEnterprise)
+	repo, okRepo := getEnv(runnerContainer, EnvVarRepo)
+	org, okOrg := getEnv(runnerContainer, EnvVarOrg)
 	if !okRepo || !okOrg || !okEnterprise {
 		return newEmptyResponse()
 	}

controllers/runner_controller.go

@@ -18,21 +18,19 @@ package controllers
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/hash"
-	gogithub "github.com/google/go-github/v37/github"
-	"k8s.io/apimachinery/pkg/util/wait"
-
 	"github.com/go-logr/logr"
+
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -52,9 +50,12 @@ const (
 	// This is an annotation internal to actions-runner-controller and can change in backward-incompatible ways
 	annotationKeyRegistrationOnly = "actions-runner-controller/registration-only"
 
-	EnvVarOrg        = "RUNNER_ORG"
-	EnvVarRepo       = "RUNNER_REPO"
-	EnvVarEnterprise = "RUNNER_ENTERPRISE"
+	EnvVarOrg                        = "RUNNER_ORG"
+	EnvVarRepo                       = "RUNNER_REPO"
+	EnvVarEnterprise                 = "RUNNER_ENTERPRISE"
+	EnvVarEphemeral                  = "RUNNER_EPHEMERAL"
+	EnvVarRunnerFeatureFlagEphemeral = "RUNNER_FEATURE_FLAG_EPHEMERAL"
+	EnvVarTrue                       = "true"
 )
 
 // RunnerReconciler reconciles a Runner object
@@ -65,11 +66,14 @@ type RunnerReconciler struct {
 	Scheme                      *runtime.Scheme
 	GitHubClient                *github.Client
 	RunnerImage                 string
+	RunnerImagePullSecrets      []string
 	DockerImage                 string
 	DockerRegistryMirror        string
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
+
+	UnregistrationRetryDelay time.Duration
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
@@ -87,12 +91,6 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	err := runner.Validate()
-	if err != nil {
-		log.Info("Failed to validate runner spec", "error", err.Error())
-		return ctrl.Result{}, nil
-	}
-
 	if runner.ObjectMeta.DeletionTimestamp.IsZero() {
 		finalizers, added := addFinalizer(runner.ObjectMeta.Finalizers, finalizerName)
 
@@ -108,428 +106,142 @@ func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 			return ctrl.Result{}, nil
 		}
 	} else {
-		finalizers, removed := removeFinalizer(runner.ObjectMeta.Finalizers, finalizerName)
-
-		if removed {
-			if len(runner.Status.Registration.Token) > 0 {
-				ok, err := r.unregisterRunner(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
-				if err != nil {
-					if errors.Is(err, &gogithub.RateLimitError{}) {
-						// We log the underlying error when we failed calling GitHub API to list or unregisters,
-						// or the runner is still busy.
-						log.Error(
-							err,
-							fmt.Sprintf(
-								"Failed to unregister runner due to GitHub API rate limits. Delaying retry for %s to avoid excessive GitHub API calls",
-								retryDelayOnGitHubAPIRateLimitError,
-							),
-						)
-
-						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
-					}
-
-					return ctrl.Result{}, err
-				}
-
-				if !ok {
-					log.V(1).Info("Runner no longer exists on GitHub")
-				}
-			} else {
-				log.V(1).Info("Runner was never registered on GitHub")
-			}
-
-			newRunner := runner.DeepCopy()
-			newRunner.ObjectMeta.Finalizers = finalizers
-
-			if err := r.Patch(ctx, newRunner, client.MergeFrom(&runner)); err != nil {
-				log.Error(err, "Failed to update runner for finalizer removal")
-				return ctrl.Result{}, err
-			}
-
-			log.Info("Removed runner from GitHub", "repository", runner.Spec.Repository, "organization", runner.Spec.Organization)
-		}
-
-		return ctrl.Result{}, nil
-	}
-
-	registrationOnly := metav1.HasAnnotation(runner.ObjectMeta, annotationKeyRegistrationOnly)
-	if registrationOnly && runner.Status.Phase != "" {
-		// At this point we are sure that the registration-only runner has successfully configured and
-		// is of `offline` status, because we set runner.Status.Phase to that of the runner pod only after
-		// successful registration.
-
 		var pod corev1.Pod
 		if err := r.Get(ctx, req.NamespacedName, &pod); err != nil {
 			if !kerrors.IsNotFound(err) {
-				log.Info(fmt.Sprintf("Retrying soon as we failed to get registration-only runner pod: %v", err))
-
-				return ctrl.Result{Requeue: true}, nil
-			}
-		} else if err := r.Delete(ctx, &pod); err != nil {
-			if !kerrors.IsNotFound(err) {
-				log.Info(fmt.Sprintf("Retrying soon as we failed to delete registration-only runner pod: %v", err))
-
+				log.Info(fmt.Sprintf("Retrying soon as we failed to get runner pod: %v", err))
 				return ctrl.Result{Requeue: true}, nil
 			}
 		}
 
-		log.Info("Successfully deleted registration-only runner pod to free node and cluster resource")
-
-		// Return here to not recreate the deleted pod, because recreating it is the waste of cluster and node resource,
-		// and also defeats the original purpose of scale-from/to-zero we're trying to implement by using the registration-only runner.
-		return ctrl.Result{}, nil
+		// Request to remove a runner. DeletionTimestamp was set in the runner - we need to unregister runner
+		return r.processRunnerDeletion(runner, ctx, log, &pod)
 	}
 
 	var pod corev1.Pod
 	if err := r.Get(ctx, req.NamespacedName, &pod); err != nil {
 		if !kerrors.IsNotFound(err) {
+			// An error ocurred
 			return ctrl.Result{}, err
 		}
+		return r.processRunnerCreation(ctx, runner, log)
+	}
 
-		if updated, err := r.updateRegistrationToken(ctx, runner); err != nil {
-			return ctrl.Result{}, err
-		} else if updated {
-			return ctrl.Result{Requeue: true}, nil
-		}
+	phase := string(pod.Status.Phase)
+	if phase == "" {
+		phase = "Created"
+	}
 
-		newPod, err := r.newPod(runner)
-		if err != nil {
-			log.Error(err, "Could not create pod")
-			return ctrl.Result{}, err
-		}
-
-		if err := r.Create(ctx, &newPod); err != nil {
-			if kerrors.IsAlreadyExists(err) {
-				// Gracefully handle pod-already-exists errors due to informer cache delay.
-				// Without this we got a few errors like the below on new runner pod:
-				// 2021-03-16T00:23:10.116Z        ERROR   controller-runtime.controller   Reconciler error      {"controller": "runner-controller", "request": "default/example-runnerdeploy-b2g2g-j4mcp", "error": "pods \"example-runnerdeploy-b2g2g-j4mcp\" already exists"}
-				log.Info(
-					"Failed to create pod due to AlreadyExists error. Probably this pod has been already created in previous reconcilation but is still not in the informer cache. Will retry on pod created. If it doesn't repeat, there's no problem",
-				)
-
-				return ctrl.Result{}, nil
-			}
-
-			log.Error(err, "Failed to create pod resource")
-
-			return ctrl.Result{}, err
-		}
-
-		r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodCreated", fmt.Sprintf("Created pod '%s'", newPod.Name))
-		log.Info("Created runner pod", "repository", runner.Spec.Repository)
-	} else {
-		if !pod.ObjectMeta.DeletionTimestamp.IsZero() {
-			deletionTimeout := 1 * time.Minute
-			currentTime := time.Now()
-			deletionDidTimeout := currentTime.Sub(pod.DeletionTimestamp.Add(deletionTimeout)) > 0
-
-			if deletionDidTimeout {
-				log.Info(
-					fmt.Sprintf("Failed to delete pod within %s. ", deletionTimeout)+
-						"This is typically the case when a Kubernetes node became unreachable "+
-						"and the kube controller started evicting nodes. Forcefully deleting the pod to not get stuck.",
-					"podDeletionTimestamp", pod.DeletionTimestamp,
-					"currentTime", currentTime,
-					"configuredDeletionTimeout", deletionTimeout,
-				)
-
-				var force int64 = 0
-				// forcefully delete runner as we would otherwise get stuck if the node stays unreachable
-				if err := r.Delete(ctx, &pod, &client.DeleteOptions{GracePeriodSeconds: &force}); err != nil {
-					// probably
-					if !kerrors.IsNotFound(err) {
-						log.Error(err, "Failed to forcefully delete pod resource ...")
-						return ctrl.Result{}, err
-					}
-					// forceful deletion finally succeeded
-					return ctrl.Result{Requeue: true}, nil
-				}
-
-				r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodDeleted", fmt.Sprintf("Forcefully deleted pod '%s'", pod.Name))
-				log.Info("Forcefully deleted runner pod", "repository", runner.Spec.Repository)
-				// give kube manager a little time to forcefully delete the stuck pod
-				return ctrl.Result{RequeueAfter: 3 * time.Second}, err
-			} else {
-				return ctrl.Result{}, err
-			}
-		}
-
-		// If pod has ended up succeeded we need to restart it
-		// Happens e.g. when dind is in runner and run completes
-		stopped := pod.Status.Phase == corev1.PodSucceeded
-
-		if !stopped {
-			if pod.Status.Phase == corev1.PodRunning {
-				for _, status := range pod.Status.ContainerStatuses {
-					if status.Name != containerName {
-						continue
-					}
-
-					if status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 {
-						stopped = true
-					}
-				}
-			}
-		}
-
-		restart := stopped
-
-		if registrationOnly && stopped {
-			restart = false
-
-			log.Info(
-				"Observed that registration-only runner for scaling-from-zero has successfully stopped. " +
-					"Unlike other pods, this one will be recreated only when runner spec changes.",
+	if runner.Status.Phase != phase {
+		if pod.Status.Phase == corev1.PodRunning {
+			// Seeing this message, you can expect the runner to become `Running` soon.
+			log.V(1).Info(
+				"Runner appears to have been registered and running.",
+				"podCreationTimestamp", pod.CreationTimestamp,
 			)
 		}
 
-		if updated, err := r.updateRegistrationToken(ctx, runner); err != nil {
-			return ctrl.Result{}, err
-		} else if updated {
-			return ctrl.Result{Requeue: true}, nil
-		}
+		updated := runner.DeepCopy()
+		updated.Status.Phase = phase
+		updated.Status.Reason = pod.Status.Reason
+		updated.Status.Message = pod.Status.Message
 
-		newPod, err := r.newPod(runner)
-		if err != nil {
-			log.Error(err, "Could not create pod")
+		if err := r.Status().Patch(ctx, updated, client.MergeFrom(&runner)); err != nil {
+			log.Error(err, "Failed to update runner status for Phase/Reason/Message")
 			return ctrl.Result{}, err
 		}
-
-		if registrationOnly {
-			newPod.Spec.Containers[0].Env = append(
-				newPod.Spec.Containers[0].Env,
-				corev1.EnvVar{
-					Name:  "RUNNER_REGISTRATION_ONLY",
-					Value: "true",
-				},
-			)
-		}
-
-		var registrationRecheckDelay time.Duration
-
-		// all checks done below only decide whether a restart is needed
-		// if a restart was already decided before, there is no need for the checks
-		// saving API calls and scary log messages
-		if !restart {
-			registrationCheckInterval := time.Minute
-			if r.RegistrationRecheckInterval > 0 {
-				registrationCheckInterval = r.RegistrationRecheckInterval
-			}
-
-			// We want to call ListRunners GitHub Actions API only once per runner per minute.
-			// This if block, in conjunction with:
-			//   return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil
-			// achieves that.
-			if lastCheckTime := runner.Status.LastRegistrationCheckTime; lastCheckTime != nil {
-				nextCheckTime := lastCheckTime.Add(registrationCheckInterval)
-				now := time.Now()
-
-				// Requeue scheduled by RequeueAfter can happen a bit earlier (like dozens of milliseconds)
-				// so to avoid excessive, in-effective retry, we heuristically ignore the remaining delay in case it is
-				// shorter than 1s
-				requeueAfter := nextCheckTime.Sub(now) - time.Second
-				if requeueAfter > 0 {
-					log.Info(
-						fmt.Sprintf("Skipped registration check because it's deferred until %s. Retrying in %s at latest", nextCheckTime, requeueAfter),
-						"lastRegistrationCheckTime", lastCheckTime,
-						"registrationCheckInterval", registrationCheckInterval,
-					)
-
-					// Without RequeueAfter, the controller may not retry on scheduled. Instead, it must wait until the
-					// next sync period passes, which can be too much later than nextCheckTime.
-					//
-					// We need to requeue on this reconcilation even though we have already scheduled the initial
-					// requeue previously with `return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil`.
-					// Apparently, the workqueue used by controller-runtime seems to deduplicate and resets the delay on
-					// other requeues- so the initial scheduled requeue may have been reset due to requeue on
-					// spec/status change.
-					return ctrl.Result{RequeueAfter: requeueAfter}, nil
-				}
-			}
-
-			notFound := false
-			offline := false
-
-			runnerBusy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
-
-			currentTime := time.Now()
-
-			if err != nil {
-				var notFoundException *github.RunnerNotFound
-				var offlineException *github.RunnerOffline
-				if errors.As(err, &notFoundException) {
-					notFound = true
-				} else if errors.As(err, &offlineException) {
-					offline = true
-				} else {
-					var e *gogithub.RateLimitError
-					if errors.As(err, &e) {
-						// We log the underlying error when we failed calling GitHub API to list or unregisters,
-						// or the runner is still busy.
-						log.Error(
-							err,
-							fmt.Sprintf(
-								"Failed to check if runner is busy due to Github API rate limit. Retrying in %s to avoid excessive GitHub API calls",
-								retryDelayOnGitHubAPIRateLimitError,
-							),
-						)
-
-						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
-					}
-
-					return ctrl.Result{}, err
-				}
-			}
-
-			// See the `newPod` function called above for more information
-			// about when this hash changes.
-			curHash := pod.Labels[LabelKeyPodTemplateHash]
-			newHash := newPod.Labels[LabelKeyPodTemplateHash]
-
-			if !runnerBusy && curHash != newHash {
-				restart = true
-			}
-
-			registrationTimeout := 10 * time.Minute
-			durationAfterRegistrationTimeout := currentTime.Sub(pod.CreationTimestamp.Add(registrationTimeout))
-			registrationDidTimeout := durationAfterRegistrationTimeout > 0
-
-			if notFound {
-				if registrationDidTimeout {
-					log.Info(
-						"Runner failed to register itself to GitHub in timely manner. "+
-							"Recreating the pod to see if it resolves the issue. "+
-							"CAUTION: If you see this a lot, you should investigate the root cause. "+
-							"See https://github.com/actions-runner-controller/actions-runner-controller/issues/288",
-						"podCreationTimestamp", pod.CreationTimestamp,
-						"currentTime", currentTime,
-						"configuredRegistrationTimeout", registrationTimeout,
-					)
-
-					restart = true
-				} else {
-					log.V(1).Info(
-						"Runner pod exists but we failed to check if runner is busy. Apparently it still needs more time.",
-						"runnerName", runner.Name,
-					)
-				}
-			} else if offline {
-				if registrationOnly {
-					log.Info(
-						"Observed that registration-only runner for scaling-from-zero has successfully been registered.",
-						"podCreationTimestamp", pod.CreationTimestamp,
-						"currentTime", currentTime,
-						"configuredRegistrationTimeout", registrationTimeout,
-					)
-				} else if registrationDidTimeout {
-					log.Info(
-						"Already existing GitHub runner still appears offline . "+
-							"Recreating the pod to see if it resolves the issue. "+
-							"CAUTION: If you see this a lot, you should investigate the root cause. ",
-						"podCreationTimestamp", pod.CreationTimestamp,
-						"currentTime", currentTime,
-						"configuredRegistrationTimeout", registrationTimeout,
-					)
-
-					restart = true
-				} else {
-					log.V(1).Info(
-						"Runner pod exists but the GitHub runner appears to be still offline. Waiting for runner to get online ...",
-						"runnerName", runner.Name,
-					)
-				}
-			}
-
-			if (notFound || (offline && !registrationOnly)) && !registrationDidTimeout {
-				registrationRecheckJitter := 10 * time.Second
-				if r.RegistrationRecheckJitter > 0 {
-					registrationRecheckJitter = r.RegistrationRecheckJitter
-				}
-
-				registrationRecheckDelay = registrationCheckInterval + wait.Jitter(registrationRecheckJitter, 0.1)
-			}
-		}
-
-		// Don't do anything if there's no need to restart the runner
-		if !restart {
-			// This guard enables us to update runner.Status.Phase to `Running` only after
-			// the runner is registered to GitHub.
-			if registrationRecheckDelay > 0 {
-				log.V(1).Info(fmt.Sprintf("Rechecking the runner registration in %s", registrationRecheckDelay))
-
-				updated := runner.DeepCopy()
-				updated.Status.LastRegistrationCheckTime = &metav1.Time{Time: time.Now()}
-
-				if err := r.Status().Patch(ctx, updated, client.MergeFrom(&runner)); err != nil {
-					log.Error(err, "Failed to update runner status for LastRegistrationCheckTime")
-					return ctrl.Result{}, err
-				}
-
-				return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil
-			}
-
-			if runner.Status.Phase != string(pod.Status.Phase) {
-				if pod.Status.Phase == corev1.PodRunning {
-					// Seeing this message, you can expect the runner to become `Running` soon.
-					log.Info(
-						"Runner appears to have registered and running.",
-						"podCreationTimestamp", pod.CreationTimestamp,
-					)
-				}
-
-				updated := runner.DeepCopy()
-				updated.Status.Phase = string(pod.Status.Phase)
-				updated.Status.Reason = pod.Status.Reason
-				updated.Status.Message = pod.Status.Message
-
-				if err := r.Status().Patch(ctx, updated, client.MergeFrom(&runner)); err != nil {
-					log.Error(err, "Failed to update runner status for Phase/Reason/Message")
-					return ctrl.Result{}, err
-				}
-			}
-
-			return ctrl.Result{}, nil
-		}
-
-		// Delete current pod if recreation is needed
-		if err := r.Delete(ctx, &pod); err != nil {
-			log.Error(err, "Failed to delete pod resource")
-			return ctrl.Result{}, err
-		}
-
-		r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodDeleted", fmt.Sprintf("Deleted pod '%s'", newPod.Name))
-		log.Info("Deleted runner pod", "repository", runner.Spec.Repository)
 	}
 
 	return ctrl.Result{}, nil
 }
 
-func (r *RunnerReconciler) unregisterRunner(ctx context.Context, enterprise, org, repo, name string) (bool, error) {
-	runners, err := r.GitHubClient.ListRunners(ctx, enterprise, org, repo)
-	if err != nil {
-		return false, err
-	}
+func runnerContainerExitCode(pod *corev1.Pod) *int32 {
+	for _, status := range pod.Status.ContainerStatuses {
+		if status.Name != containerName {
+			continue
+		}
 
-	id := int64(0)
-	for _, runner := range runners {
-		if runner.GetName() == name {
-			if runner.GetBusy() {
-				return false, fmt.Errorf("runner is busy")
-			}
-			id = runner.GetID()
-			break
+		if status.State.Terminated != nil {
+			return &status.State.Terminated.ExitCode
 		}
 	}
 
-	if id == int64(0) {
-		return false, nil
+	return nil
+}
+
+func runnerPodOrContainerIsStopped(pod *corev1.Pod) bool {
+	// If pod has ended up succeeded we need to restart it
+	// Happens e.g. when dind is in runner and run completes
+	stopped := pod.Status.Phase == corev1.PodSucceeded
+
+	if !stopped {
+		if pod.Status.Phase == corev1.PodRunning {
+			for _, status := range pod.Status.ContainerStatuses {
+				if status.Name != containerName {
+					continue
+				}
+
+				if status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 {
+					stopped = true
+				}
+			}
+		}
 	}
 
-	if err := r.GitHubClient.RemoveRunner(ctx, enterprise, org, repo, id); err != nil {
-		return false, err
+	return stopped
+}
+
+func (r *RunnerReconciler) processRunnerDeletion(runner v1alpha1.Runner, ctx context.Context, log logr.Logger, pod *corev1.Pod) (reconcile.Result, error) {
+	finalizers, removed := removeFinalizer(runner.ObjectMeta.Finalizers, finalizerName)
+
+	if removed {
+		newRunner := runner.DeepCopy()
+		newRunner.ObjectMeta.Finalizers = finalizers
+
+		if err := r.Patch(ctx, newRunner, client.MergeFrom(&runner)); err != nil {
+			log.Error(err, "Unable to remove finalizer")
+			return ctrl.Result{}, err
+		}
+
+		log.Info("Removed finalizer")
 	}
 
-	return true, nil
+	return ctrl.Result{}, nil
+}
+
+func (r *RunnerReconciler) processRunnerCreation(ctx context.Context, runner v1alpha1.Runner, log logr.Logger) (reconcile.Result, error) {
+	if updated, err := r.updateRegistrationToken(ctx, runner); err != nil {
+		return ctrl.Result{RequeueAfter: RetryDelayOnCreateRegistrationError}, nil
+	} else if updated {
+		return ctrl.Result{Requeue: true}, nil
+	}
+
+	newPod, err := r.newPod(runner)
+	if err != nil {
+		log.Error(err, "Could not create pod")
+		return ctrl.Result{}, err
+	}
+
+	if err := r.Create(ctx, &newPod); err != nil {
+		if kerrors.IsAlreadyExists(err) {
+			// Gracefully handle pod-already-exists errors due to informer cache delay.
+			// Without this we got a few errors like the below on new runner pod:
+			// 2021-03-16T00:23:10.116Z        ERROR   controller-runtime.controller   Reconciler error      {"controller": "runner-controller", "request": "default/example-runnerdeploy-b2g2g-j4mcp", "error": "pods \"example-runnerdeploy-b2g2g-j4mcp\" already exists"}
+			log.Info(
+				"Failed to create pod due to AlreadyExists error. Probably this pod has been already created in previous reconcilation but is still not in the informer cache. Will retry on pod created. If it doesn't repeat, there's no problem",
+			)
+			return ctrl.Result{}, nil
+		}
+
+		log.Error(err, "Failed to create pod resource")
+
+		return ctrl.Result{}, err
+	}
+
+	r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodCreated", fmt.Sprintf("Created pod '%s'", newPod.Name))
+	log.Info("Created runner pod", "repository", runner.Spec.Repository)
+
+	return ctrl.Result{}, nil
 }
 
 func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v1alpha1.Runner) (bool, error) {
@@ -541,6 +253,10 @@ func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v
 
 	rt, err := r.GitHubClient.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 	if err != nil {
+		// An error can be a permanent, permission issue like the below:
+		//    POST https://api.github.com/enterprises/YOUR_ENTERPRISE/actions/runners/registration-token: 403 Resource not accessible by integration []
+		// In such case retrying in seconds might not make much sense.
+
 		r.Recorder.Event(&runner, corev1.EventTypeWarning, "FailedUpdateRegistrationToken", "Updating registration token failed")
 		log.Error(err, "Failed to get new registration token")
 		return false, err
@@ -599,6 +315,11 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		runner.ObjectMeta.Annotations,
 		runner.Spec,
 		r.GitHubClient.GithubBaseURL,
+		// Token change should trigger replacement.
+		// We need to include this explicitly here because
+		// runner.Spec does not contain the possibly updated token stored in the
+		// runner status yet.
+		runner.Status.Registration.Token,
 	)
 
 	objectMeta := metav1.ObjectMeta{
@@ -624,6 +345,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 				Name:         "docker",
 				VolumeMounts: runner.Spec.DockerVolumeMounts,
 				Resources:    runner.Spec.DockerdContainerResources,
+				Env:          runner.Spec.DockerEnv,
 			})
 		}
 	} else {
@@ -635,7 +357,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 
 	registrationOnly := metav1.HasAnnotation(runner.ObjectMeta, annotationKeyRegistrationOnly)
 
-	pod, err := newRunnerPod(template, runner.Spec.RunnerConfig, r.RunnerImage, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL, registrationOnly)
+	pod, err := newRunnerPod(runner.Name, template, runner.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubClient.GithubBaseURL, registrationOnly)
 	if err != nil {
 		return pod, err
 	}
@@ -644,10 +366,29 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	runnerSpec := runner.Spec
 
 	if len(runnerSpec.VolumeMounts) != 0 {
+		// if operater provides a work volume mount, use that
+		isPresent, _ := workVolumeMountPresent(runnerSpec.VolumeMounts)
+		if isPresent {
+			// remove work volume since it will be provided from runnerSpec.Volumes
+			// if we don't remove it here we would get a duplicate key error, i.e. two volumes named work
+			_, index := workVolumeMountPresent(pod.Spec.Containers[0].VolumeMounts)
+			pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts[:index], pod.Spec.Containers[0].VolumeMounts[index+1:]...)
+		}
+
 		pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, runnerSpec.VolumeMounts...)
 	}
 
 	if len(runnerSpec.Volumes) != 0 {
+		// if operator provides a work volume. use that
+		isPresent, _ := workVolumePresent(runnerSpec.Volumes)
+		if isPresent {
+			_, index := workVolumePresent(pod.Spec.Volumes)
+
+			// remove work volume since it will be provided from runnerSpec.Volumes
+			// if we don't remove it here we would get a duplicate key error, i.e. two volumes named work
+			pod.Spec.Volumes = append(pod.Spec.Volumes[:index], pod.Spec.Volumes[index+1:]...)
+		}
+
 		pod.Spec.Volumes = append(pod.Spec.Volumes, runnerSpec.Volumes...)
 	}
 	if len(runnerSpec.InitContainers) != 0 {
@@ -715,25 +456,18 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 func mutatePod(pod *corev1.Pod, token string) *corev1.Pod {
 	updated := pod.DeepCopy()
 
-	for i := range pod.Spec.Containers {
-		if pod.Spec.Containers[i].Name == "runner" {
-			updated.Spec.Containers[i].Env = append(updated.Spec.Containers[i].Env,
-				corev1.EnvVar{
-					Name:  "RUNNER_NAME",
-					Value: pod.ObjectMeta.Name,
-				},
-				corev1.EnvVar{
-					Name:  "RUNNER_TOKEN",
-					Value: token,
-				},
-			)
-		}
+	if getRunnerEnv(pod, EnvVarRunnerName) == "" {
+		setRunnerEnv(updated, EnvVarRunnerName, pod.ObjectMeta.Name)
+	}
+
+	if getRunnerEnv(pod, EnvVarRunnerToken) == "" {
+		setRunnerEnv(updated, EnvVarRunnerToken, token)
 	}
 
 	return updated
 }
 
-func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, registrationOnly bool) (corev1.Pod, error) {
+func newRunnerPod(runnerName string, template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, defaultRunnerImage string, defaultRunnerImagePullSecrets []string, defaultDockerImage, defaultDockerRegistryMirror string, githubBaseURL string, registrationOnly bool) (corev1.Pod, error) {
 	var (
 		privileged                bool = true
 		dockerdInRunner           bool = runnerSpec.DockerdWithinRunnerContainer != nil && *runnerSpec.DockerdWithinRunnerContainer
@@ -742,6 +476,12 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 		dockerdInRunnerPrivileged bool = dockerdInRunner
 	)
 
+	template = *template.DeepCopy()
+
+	// This label selector is used by default when rd.Spec.Selector is empty.
+	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerName)
+	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyPodMutation, LabelValuePodMutation)
+
 	workDir := runnerSpec.WorkDir
 	if workDir == "" {
 		workDir = "/runner/_work"
@@ -754,6 +494,8 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 		dockerRegistryMirror = *runnerSpec.DockerRegistryMirror
 	}
 
+	// Be aware some of the environment variables are used
+	// in the runner entrypoint script
 	env := []corev1.EnvVar{
 		{
 			Name:  EnvVarOrg,
@@ -775,6 +517,10 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 			Name:  "RUNNER_GROUP",
 			Value: runnerSpec.Group,
 		},
+		{
+			Name:  "DOCKER_ENABLED",
+			Value: fmt.Sprintf("%v", dockerEnabled || dockerdInRunner),
+		},
 		{
 			Name:  "DOCKERD_IN_RUNNER",
 			Value: fmt.Sprintf("%v", dockerdInRunner),
@@ -788,7 +534,7 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 			Value: workDir,
 		},
 		{
-			Name:  "RUNNER_EPHEMERAL",
+			Name:  EnvVarEphemeral,
 			Value: fmt.Sprintf("%v", ephemeral),
 		},
 	}
@@ -876,6 +622,15 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 		}...)
 	}
 
+	if len(pod.Spec.ImagePullSecrets) == 0 && len(defaultRunnerImagePullSecrets) > 0 {
+		// runner spec didn't provide custom values and default image pull secrets are provided
+		for _, imagePullSecret := range defaultRunnerImagePullSecrets {
+			pod.Spec.ImagePullSecrets = append(pod.Spec.ImagePullSecrets, corev1.LocalObjectReference{
+				Name: imagePullSecret,
+			})
+		}
+	}
+
 	if dockerRegistryMirror != "" && dockerdInRunner {
 		runnerContainer.Env = append(runnerContainer.Env, []corev1.EnvVar{
 			{
@@ -948,6 +703,7 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 				},
 			},
 		)
+
 		runnerContainer.VolumeMounts = append(runnerContainer.VolumeMounts,
 			corev1.VolumeMount{
 				Name:      "work",
@@ -959,6 +715,7 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 				ReadOnly:  true,
 			},
 		)
+
 		runnerContainer.Env = append(runnerContainer.Env, []corev1.EnvVar{
 			{
 				Name:  "DOCKER_HOST",
@@ -977,10 +734,6 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 		// Determine the volume mounts assigned to the docker sidecar. In case extra mounts are included in the RunnerSpec, append them to the standard
 		// set of mounts. See https://github.com/actions-runner-controller/actions-runner-controller/issues/435 for context.
 		dockerVolumeMounts := []corev1.VolumeMount{
-			{
-				Name:      "work",
-				MountPath: workDir,
-			},
 			{
 				Name:      runnerVolumeName,
 				MountPath: runnerVolumeMountPath,
@@ -991,6 +744,14 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 			},
 		}
 
+		mountPresent, _ := workVolumeMountPresent(dockerdContainer.VolumeMounts)
+		if !mountPresent {
+			dockerVolumeMounts = append(dockerVolumeMounts, corev1.VolumeMount{
+				Name:      "work",
+				MountPath: workDir,
+			})
+		}
+
 		if dockerdContainer.Image == "" {
 			dockerdContainer.Image = defaultDockerImage
 		}
@@ -1049,6 +810,12 @@ func newRunnerPod(template corev1.Pod, runnerSpec v1alpha1.RunnerConfig, default
 		}
 	}
 
+	// TODO Remove this once we remove RUNNER_FEATURE_FLAG_EPHEMERAL from runner's entrypoint.sh
+	// and make --ephemeral the default option.
+	if getRunnerEnv(pod, EnvVarRunnerFeatureFlagEphemeral) == "" {
+		setRunnerEnv(pod, EnvVarRunnerFeatureFlagEphemeral, EnvVarTrue)
+	}
+
 	return *pod, nil
 }
 
@@ -1096,3 +863,21 @@ func removeFinalizer(finalizers []string, finalizerName string) ([]string, bool)
 
 	return result, removed
 }
+
+func workVolumePresent(items []corev1.Volume) (bool, int) {
+	for index, item := range items {
+		if item.Name == "work" {
+			return true, index
+		}
+	}
+	return false, 0
+}
+
+func workVolumeMountPresent(items []corev1.VolumeMount) (bool, int) {
+	for index, item := range items {
+		if item.Name == "work" {
+			return true, index
+		}
+	}
+	return false, 0
+}

controllers/runner_graceful_stop.go

@@ -0,0 +1,396 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/go-logr/logr"
+	gogithub "github.com/google/go-github/v39/github"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// tickRunnerGracefulStop reconciles the runner and the runner pod in a way so that
+// we can delete the runner pod without disrupting a workflow job.
+//
+// This function returns a non-nil pointer to corev1.Pod as the first return value
+// if the runner is considered to have gracefully stopped, hence it's pod is safe for deletion.
+//
+// It's a "tick" operation so a graceful stop can take multiple calls to complete.
+// This function is designed to complete a lengthy graceful stop process in a unblocking way.
+// When it wants to be retried later, the function returns a non-nil *ctrl.Result as the second return value, may or may not populating the error in the second return value.
+// The caller is expected to return the returned ctrl.Result and error to postpone the current reconcilation loop and trigger a scheduled retry.
+func tickRunnerGracefulStop(ctx context.Context, retryDelay time.Duration, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*corev1.Pod, *ctrl.Result, error) {
+	pod, err := annotatePodOnce(ctx, c, log, pod, AnnotationKeyUnregistrationStartTimestamp, time.Now().Format(time.RFC3339))
+	if err != nil {
+		return nil, &ctrl.Result{}, err
+	}
+
+	if res, err := ensureRunnerUnregistration(ctx, retryDelay, log, ghClient, c, enterprise, organization, repository, runner, pod); res != nil {
+		return nil, res, err
+	}
+
+	pod, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyUnregistrationCompleteTimestamp, time.Now().Format(time.RFC3339))
+	if err != nil {
+		return nil, &ctrl.Result{}, err
+	}
+
+	return pod, nil, nil
+}
+
+// annotatePodOnce annotates the pod if it wasn't.
+// Returns the provided pod as-is if it was already annotated.
+// Returns the updated pod if the pod was missing the annotation and the update to add the annotation succeeded.
+func annotatePodOnce(ctx context.Context, c client.Client, log logr.Logger, pod *corev1.Pod, k, v string) (*corev1.Pod, error) {
+	if pod == nil {
+		return nil, nil
+	}
+
+	if _, ok := getAnnotation(pod, k); ok {
+		return pod, nil
+	}
+
+	updated := pod.DeepCopy()
+	setAnnotation(&updated.ObjectMeta, k, v)
+	if err := c.Patch(ctx, updated, client.MergeFrom(pod)); err != nil {
+		log.Error(err, fmt.Sprintf("Failed to patch pod to have %s annotation", k))
+		return nil, err
+	}
+
+	log.V(2).Info("Annotated pod", "key", k, "value", v)
+
+	return updated, nil
+}
+
+// If the first return value is nil, it's safe to delete the runner pod.
+func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*ctrl.Result, error) {
+	var runnerID *int64
+
+	if id, ok := getAnnotation(pod, AnnotationKeyRunnerID); ok {
+		v, err := strconv.ParseInt(id, 10, 64)
+		if err != nil {
+			return &ctrl.Result{}, err
+		}
+
+		runnerID = &v
+	}
+
+	if runnerID == nil {
+		runner, err := getRunner(ctx, ghClient, enterprise, organization, repository, runner)
+		if err != nil {
+			return &ctrl.Result{}, err
+		}
+
+		if runner != nil && runner.ID != nil {
+			runnerID = runner.ID
+		}
+	}
+
+	code := runnerContainerExitCode(pod)
+
+	if pod != nil && pod.Annotations[AnnotationKeyUnregistrationCompleteTimestamp] != "" {
+		// If it's already unregistered in the previous reconcilation loop,
+		// you can safely assume that it won't get registered again so it's safe to delete the runner pod.
+		log.Info("Runner pod is marked as already unregistered.")
+	} else if runnerID == nil {
+		log.Info(
+			"Unregistration started before runner ID is assigned. " +
+				"Perhaps the runner pod was terminated by anyone other than ARC? Was it OOM killed? " +
+				"Marking unregistration as completed anyway because there's nothing ARC can do.",
+		)
+	} else if pod != nil && runnerPodOrContainerIsStopped(pod) {
+		// If it's an ephemeral runner with the actions/runner container exited with 0,
+		// we can safely assume that it has unregistered itself from GitHub Actions
+		// so it's natural that RemoveRunner fails due to 404.
+
+		// If pod has ended up succeeded we need to restart it
+		// Happens e.g. when dind is in runner and run completes
+		log.Info("Runner pod has been stopped with a successful status.")
+	} else if pod != nil && pod.Annotations[AnnotationKeyRunnerCompletionWaitStartTimestamp] != "" {
+		log.Info("Runner pod is annotated to wait for completion")
+
+		return &ctrl.Result{RequeueAfter: retryDelay}, nil
+	} else if ok, err := unregisterRunner(ctx, ghClient, enterprise, organization, repository, runner, *runnerID); err != nil {
+		if errors.Is(err, &gogithub.RateLimitError{}) {
+			// We log the underlying error when we failed calling GitHub API to list or unregisters,
+			// or the runner is still busy.
+			log.Error(
+				err,
+				fmt.Sprintf(
+					"Failed to unregister runner due to GitHub API rate limits. Delaying retry for %s to avoid excessive GitHub API calls",
+					retryDelayOnGitHubAPIRateLimitError,
+				),
+			)
+
+			return &ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+		}
+
+		log.V(1).Info("Failed to unregister runner before deleting the pod.", "error", err)
+
+		var runnerBusy bool
+
+		errRes := &gogithub.ErrorResponse{}
+		if errors.As(err, &errRes) {
+			if errRes.Response.StatusCode == 403 {
+				log.Error(err, "Unable to unregister due to permission error. "+
+					"Perhaps you've changed the permissions of PAT or GitHub App, or you updated authentication method of ARC in a wrong way? "+
+					"ARC considers it as already unregistered and continue removing the pod. "+
+					"You may need to remove the runner on GitHub UI.")
+
+				return nil, nil
+			}
+
+			runner, _ := getRunner(ctx, ghClient, enterprise, organization, repository, runner)
+
+			var runnerID int64
+
+			if runner != nil && runner.ID != nil {
+				runnerID = *runner.ID
+			}
+
+			runnerBusy = errRes.Response.StatusCode == 422
+
+			if runnerBusy && code != nil {
+				log.V(2).Info("Runner container has already stopped but the unregistration attempt failed. "+
+					"This can happen when the runner container crashed due to an unhandled error, OOM, etc. "+
+					"ARC terminates the pod anyway. You'd probably need to manually delete the runner later by calling the GitHub API",
+					"runnerExitCode", *code,
+					"runnerID", runnerID,
+				)
+
+				return nil, nil
+			}
+		}
+
+		if runnerBusy {
+			// We want to prevent spamming the deletion attemps but returning ctrl.Result with RequeueAfter doesn't
+			// work as the reconcilation can happen earlier due to pod status update.
+			// For ephemeral runners, we can expect it to stop and unregister itself on completion.
+			// So we can just wait for the completion without actively retrying unregistration.
+			ephemeral := getRunnerEnv(pod, EnvVarEphemeral)
+			if ephemeral == "true" {
+				pod, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerCompletionWaitStartTimestamp, time.Now().Format(time.RFC3339))
+				if err != nil {
+					return &ctrl.Result{}, err
+				}
+
+				return &ctrl.Result{}, nil
+			}
+
+			log.V(2).Info("Retrying runner unregistration because the static runner is still busy")
+			// Otherwise we may end up spamming 422 errors,
+			// each call consuming GitHub API rate limit
+			// https://github.com/actions-runner-controller/actions-runner-controller/pull/1167#issuecomment-1064213271
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		return &ctrl.Result{}, err
+	} else if ok {
+		log.Info("Runner has just been unregistered.")
+	} else if pod == nil {
+		// `r.unregisterRunner()` will returns `false, nil` if the runner is not found on GitHub.
+		// However, that doesn't always mean the pod can be safely removed.
+		//
+		// If the pod does not exist for the runner,
+		// it may be due to that the runner pod has never been created.
+		// In that case we can safely assume that the runner will never be registered.
+
+		log.Info("Runner was not found on GitHub and the runner pod was not found on Kuberntes.")
+	} else if ts := pod.Annotations[AnnotationKeyUnregistrationStartTimestamp]; ts != "" {
+		log.Info("Runner unregistration is in-progress. It can take forever to complete if if it's a static runner constantly running jobs."+
+			" It can also take very long time if it's an ephemeral runner that is running a log-running job.", "error", err)
+
+		return &ctrl.Result{RequeueAfter: retryDelay}, nil
+	} else {
+		// A runner and a runner pod that is created by this version of ARC should match
+		// any of the above branches.
+		//
+		// But we leave this match all branch for potential backward-compatibility.
+		// The caller is expected to take appropriate actions, like annotating the pod as started the unregistration process,
+		// and retry later.
+		log.V(1).Info("Runner unregistration is being retried later.")
+
+		return &ctrl.Result{RequeueAfter: retryDelay}, nil
+	}
+
+	return nil, nil
+}
+
+func ensureRunnerPodRegistered(ctx context.Context, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*corev1.Pod, *ctrl.Result, error) {
+	_, hasRunnerID := getAnnotation(pod, AnnotationKeyRunnerID)
+	if runnerPodOrContainerIsStopped(pod) || hasRunnerID {
+		return pod, nil, nil
+	}
+
+	r, err := getRunner(ctx, ghClient, enterprise, organization, repository, runner)
+	if err != nil {
+		return nil, &ctrl.Result{RequeueAfter: 10 * time.Second}, err
+	}
+
+	if r == nil || r.ID == nil {
+		return nil, &ctrl.Result{RequeueAfter: 10 * time.Second}, err
+	}
+
+	id := *r.ID
+
+	updated, err := annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerID, fmt.Sprintf("%d", id))
+	if err != nil {
+		return nil, &ctrl.Result{RequeueAfter: 10 * time.Second}, err
+	}
+
+	return updated, nil, nil
+}
+
+func getAnnotation(obj client.Object, key string) (string, bool) {
+	if obj.GetAnnotations() == nil {
+		return "", false
+	}
+
+	v, ok := obj.GetAnnotations()[key]
+
+	return v, ok
+}
+
+func setAnnotation(meta *metav1.ObjectMeta, key, value string) {
+	if meta.Annotations == nil {
+		meta.Annotations = map[string]string{}
+	}
+
+	meta.Annotations[key] = value
+}
+
+func podConditionTransitionTime(pod *corev1.Pod, tpe corev1.PodConditionType, v corev1.ConditionStatus) *metav1.Time {
+	for _, c := range pod.Status.Conditions {
+		if c.Type == tpe && c.Status == v {
+			return &c.LastTransitionTime
+		}
+	}
+
+	return nil
+}
+
+func podConditionTransitionTimeAfter(pod *corev1.Pod, tpe corev1.PodConditionType, d time.Duration) bool {
+	c := podConditionTransitionTime(pod, tpe, corev1.ConditionTrue)
+	if c == nil {
+		return false
+	}
+
+	return c.Add(d).Before(time.Now())
+}
+
+func podRunnerID(pod *corev1.Pod) string {
+	id, _ := getAnnotation(pod, AnnotationKeyRunnerID)
+	return id
+}
+
+func getRunnerEnv(pod *corev1.Pod, key string) string {
+	for _, c := range pod.Spec.Containers {
+		if c.Name == containerName {
+			for _, e := range c.Env {
+				if e.Name == key {
+					return e.Value
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func setRunnerEnv(pod *corev1.Pod, key, value string) {
+	for i := range pod.Spec.Containers {
+		c := pod.Spec.Containers[i]
+		if c.Name == containerName {
+			for j, env := range c.Env {
+				if env.Name == key {
+					pod.Spec.Containers[i].Env[j].Value = value
+					return
+				}
+			}
+			pod.Spec.Containers[i].Env = append(c.Env, corev1.EnvVar{Name: key, Value: value})
+		}
+	}
+}
+
+// unregisterRunner unregisters the runner from GitHub Actions by name.
+//
+// This function returns:
+//
+// Case 1. (true, nil) when it has successfully unregistered the runner.
+// Case 2. (false, nil) when (2-1.) the runner has been already unregistered OR (2-2.) the runner will never be created OR (2-3.) the runner is not created yet and it is about to be registered(hence we couldn't see it's existence from GitHub Actions API yet)
+// Case 3. (false, err) when it postponed unregistration due to the runner being busy, or it tried to unregister the runner but failed due to
+//   an error returned by GitHub API.
+//
+// When the returned values is "Case 2. (false, nil)", the caller must handle the three possible sub-cases appropriately.
+// In other words, all those three sub-cases cannot be distinguished by this function alone.
+//
+// - Case "2-1." can happen when e.g. ARC has successfully unregistered in a previous reconcilation loop or it was an ephemeral runner that finished it's job run(an ephemeral runner is designed to stop after a job run).
+//   You'd need to maintain the runner state(i.e. if it's already unregistered or not) somewhere,
+//   so that you can either not call this function at all if the runner state says it's already unregistered, or determine that it's case "2-1." when you got (false, nil).
+//
+// - Case "2-2." can happen when e.g. the runner registration token was somehow broken so that `config.sh` within the runner container was never meant to succeed.
+//   Waiting and retrying forever on this case is not a solution, because `config.sh` won't succeed with a wrong token hence the runner gets stuck in this state forever.
+//   There isn't a perfect solution to this, but a practical workaround would be implement a "grace period" in the caller side.
+//
+// - Case "2-3." can happen when e.g. ARC recreated an ephemral runner pod in a previous reconcilation loop and then it was requested to delete the runner before the runner comes up.
+//   If handled inappropriately, this can cause a race condition betweeen a deletion of the runner pod and GitHub scheduling a workflow job onto the runner.
+//
+// Once successfully detected case "2-1." or "2-2.", you can safely delete the runner pod because you know that the runner won't come back
+// as long as you recreate the runner pod.
+//
+// If it was "2-3.", you need a workaround to avoid the race condition.
+//
+// You shall introduce a "grace period" mechanism, similar or equal to that is required for "Case 2-2.", so that you ever
+// start the runner pod deletion only after it's more and more likely that the runner pod is not coming up.
+//
+// Beware though, you need extra care to set an appropriate grace period depending on your environment.
+// There isn't a single right grace period that works for everyone.
+// The longer the grace period is, the earlier a cluster resource shortage can occur due to throttoled runner pod deletions,
+// while the shorter the grace period is, the more likely you may encounter the race issue.
+func unregisterRunner(ctx context.Context, client *github.Client, enterprise, org, repo, name string, id int64) (bool, error) {
+	// For the record, historically ARC did not try to call RemoveRunner on a busy runner, but it's no longer true.
+	// The reason ARC did so was to let a runner running a job to not stop prematurely.
+	//
+	// However, we learned that RemoveRunner already has an ability to prevent stopping a busy runner,
+	// so ARC doesn't need to do anything special for a graceful runner stop.
+	// It can just call RemoveRunner, and if it returned 200 you're guaranteed that the runner will not automatically come back and
+	// the runner pod is safe for deletion.
+	//
+	// Trying to remove a busy runner can result in errors like the following:
+	//    failed to remove runner: DELETE https://api.github.com/repos/actions-runner-controller/mumoshu-actions-test/actions/runners/47: 422 Bad request - Runner \"example-runnerset-0\" is still running a job\" []
+	//
+	// # NOTES
+	//
+	// - It can be "status=offline" at the same time but that's another story.
+	// - After https://github.com/actions-runner-controller/actions-runner-controller/pull/1127, ListRunners responses that are used to
+	//   determine if the runner is busy can be more outdated than before, as those responeses are now cached for 60 seconds.
+	// - Note that 60 seconds is controlled by the Cache-Control response header provided by GitHub so we don't have a strict control on it but we assume it won't
+	//   change from 60 seconds.
+	//
+	// TODO: Probably we can just remove the runner by ID without seeing if the runner is busy, by treating it as busy when a remove-runner call failed with 422?
+	if err := client.RemoveRunner(ctx, enterprise, org, repo, id); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+func getRunner(ctx context.Context, client *github.Client, enterprise, org, repo, name string) (*gogithub.Runner, error) {
+	runners, err := client.ListRunners(ctx, enterprise, org, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner, nil
+		}
+	}
+
+	return nil, nil
+}

controllers/runner_pod_controller.go

@@ -22,10 +22,8 @@ import (
 	"fmt"
 	"time"
 
-	gogithub "github.com/google/go-github/v37/github"
-	"k8s.io/apimachinery/pkg/util/wait"
-
 	"github.com/go-logr/logr"
+
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
@@ -47,16 +45,10 @@ type RunnerPodReconciler struct {
 	Name                        string
 	RegistrationRecheckInterval time.Duration
 	RegistrationRecheckJitter   time.Duration
+
+	UnregistrationRetryDelay time.Duration
 }
 
-const (
-	// This names requires at least one slash to work.
-	// See https://github.com/google/knative-gcp/issues/378
-	runnerPodFinalizerName = "actions.summerwind.dev/runner-pod"
-
-	AnnotationKeyLastRegistrationCheckTime = "actions-runner-controller/last-registration-check-time"
-)
-
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
@@ -73,9 +65,19 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
+	var envvars []corev1.EnvVar
+	for _, container := range runnerPod.Spec.Containers {
+		if container.Name == "runner" {
+			envvars = container.Env
+		}
+	}
+
+	if len(envvars) == 0 {
+		return ctrl.Result{}, errors.New("Could not determine env vars for runner Pod")
+	}
+
 	var enterprise, org, repo string
 
-	envvars := runnerPod.Spec.Containers[0].Env
 	for _, e := range envvars {
 		switch e.Name {
 		case EnvVarEnterprise:
@@ -99,44 +101,36 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 				return ctrl.Result{}, err
 			}
 
+			log.V(2).Info("Added finalizer")
+
 			return ctrl.Result{}, nil
 		}
 	} else {
+		log.V(2).Info("Seen deletion-timestamp is already set")
+
 		finalizers, removed := removeFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
 
 		if removed {
-			ok, err := r.unregisterRunner(ctx, enterprise, org, repo, runnerPod.Name)
-			if err != nil {
-				if errors.Is(err, &gogithub.RateLimitError{}) {
-					// We log the underlying error when we failed calling GitHub API to list or unregisters,
-					// or the runner is still busy.
-					log.Error(
-						err,
-						fmt.Sprintf(
-							"Failed to unregister runner due to GitHub API rate limits. Delaying retry for %s to avoid excessive GitHub API calls",
-							retryDelayOnGitHubAPIRateLimitError,
-						),
-					)
-
-					return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
-				}
-
-				return ctrl.Result{}, err
+			// In a standard scenario, the upstream controller, like runnerset-controller, ensures this runner to be gracefully stopped before the deletion timestamp is set.
+			// But for the case that the user manually deleted it for whatever reason,
+			// we have to ensure it to gracefully stop now.
+			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+			if res != nil {
+				return *res, err
 			}
 
-			if !ok {
-				log.V(1).Info("Runner no longer exists on GitHub")
-			}
+			patchedPod := updatedPod.DeepCopy()
+			patchedPod.ObjectMeta.Finalizers = finalizers
 
-			newRunner := runnerPod.DeepCopy()
-			newRunner.ObjectMeta.Finalizers = finalizers
-
-			if err := r.Patch(ctx, newRunner, client.MergeFrom(&runnerPod)); err != nil {
+			// We commit the removal of the finalizer so that Kuberenetes notices it and delete the pod resource from the cluster.
+			if err := r.Patch(ctx, patchedPod, client.MergeFrom(&runnerPod)); err != nil {
 				log.Error(err, "Failed to update runner for finalizer removal")
 				return ctrl.Result{}, err
 			}
 
-			log.Info("Removed runner from GitHub", "repository", repo, "organization", org)
+			log.V(2).Info("Removed finalizer")
+
+			return ctrl.Result{}, nil
 		}
 
 		deletionTimeout := 1 * time.Minute
@@ -174,246 +168,45 @@ func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, nil
 	}
 
-	// If pod has ended up succeeded we need to restart it
-	// Happens e.g. when dind is in runner and run completes
-	stopped := runnerPod.Status.Phase == corev1.PodSucceeded
-
-	if !stopped {
-		if runnerPod.Status.Phase == corev1.PodRunning {
-			for _, status := range runnerPod.Status.ContainerStatuses {
-				if status.Name != containerName {
-					continue
-				}
-
-				if status.State.Terminated != nil && status.State.Terminated.ExitCode == 0 {
-					stopped = true
-				}
-			}
-		}
+	po, res, err := ensureRunnerPodRegistered(ctx, log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+	if res != nil {
+		return *res, err
 	}
 
-	restart := stopped
+	runnerPod = *po
 
-	var registrationRecheckDelay time.Duration
+	if _, unregistrationRequested := getAnnotation(&runnerPod, AnnotationKeyUnregistrationRequestTimestamp); unregistrationRequested {
+		log.V(2).Info("Progressing unregistration because unregistration-request timestamp is set")
 
-	// all checks done below only decide whether a restart is needed
-	// if a restart was already decided before, there is no need for the checks
-	// saving API calls and scary log messages
-	if !restart {
-		registrationCheckInterval := time.Minute
-		if r.RegistrationRecheckInterval > 0 {
-			registrationCheckInterval = r.RegistrationRecheckInterval
+		// At this point we're sure that DeletionTimestamp is not set yet, but the unregistration process is triggered by an upstream controller like runnerset-controller.
+		//
+		// In a standard scenario, ARC starts the unregistration process before marking the pod for deletion at all,
+		// so that it isn't subject to terminationGracePeriod and can safely take hours to finish it's work.
+		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, r.GitHubClient, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+		if res != nil {
+			return *res, err
 		}
 
-		lastCheckTimeStr := runnerPod.Annotations[AnnotationKeyLastRegistrationCheckTime]
-
-		var lastCheckTime *time.Time
-
-		if lastCheckTimeStr != "" {
-			t, err := time.Parse(time.RFC3339, lastCheckTimeStr)
-			if err != nil {
-				log.Error(err, "failed to parase last check time %q", lastCheckTimeStr)
-				return ctrl.Result{}, nil
-			}
-
-			lastCheckTime = &t
-		}
-
-		// We want to call ListRunners GitHub Actions API only once per runner per minute.
-		// This if block, in conjunction with:
-		//   return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil
-		// achieves that.
-		if lastCheckTime != nil {
-			nextCheckTime := lastCheckTime.Add(registrationCheckInterval)
-			now := time.Now()
-
-			// Requeue scheduled by RequeueAfter can happen a bit earlier (like dozens of milliseconds)
-			// so to avoid excessive, in-effective retry, we heuristically ignore the remaining delay in case it is
-			// shorter than 1s
-			requeueAfter := nextCheckTime.Sub(now) - time.Second
-			if requeueAfter > 0 {
-				log.Info(
-					fmt.Sprintf("Skipped registration check because it's deferred until %s. Retrying in %s at latest", nextCheckTime, requeueAfter),
-					"lastRegistrationCheckTime", lastCheckTime,
-					"registrationCheckInterval", registrationCheckInterval,
-				)
-
-				// Without RequeueAfter, the controller may not retry on scheduled. Instead, it must wait until the
-				// next sync period passes, which can be too much later than nextCheckTime.
-				//
-				// We need to requeue on this reconcilation even though we have already scheduled the initial
-				// requeue previously with `return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil`.
-				// Apparently, the workqueue used by controller-runtime seems to deduplicate and resets the delay on
-				// other requeues- so the initial scheduled requeue may have been reset due to requeue on
-				// spec/status change.
-				return ctrl.Result{RequeueAfter: requeueAfter}, nil
-			}
-		}
-
-		notFound := false
-		offline := false
-
-		_, err := r.GitHubClient.IsRunnerBusy(ctx, enterprise, org, repo, runnerPod.Name)
-
-		currentTime := time.Now()
-
-		if err != nil {
-			var notFoundException *github.RunnerNotFound
-			var offlineException *github.RunnerOffline
-			if errors.As(err, &notFoundException) {
-				notFound = true
-			} else if errors.As(err, &offlineException) {
-				offline = true
-			} else {
-				var e *gogithub.RateLimitError
-				if errors.As(err, &e) {
-					// We log the underlying error when we failed calling GitHub API to list or unregisters,
-					// or the runner is still busy.
-					log.Error(
-						err,
-						fmt.Sprintf(
-							"Failed to check if runner is busy due to Github API rate limit. Retrying in %s to avoid excessive GitHub API calls",
-							retryDelayOnGitHubAPIRateLimitError,
-						),
-					)
-
-					return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
-				}
-
-				return ctrl.Result{}, err
-			}
-		}
-
-		registrationTimeout := 10 * time.Minute
-		durationAfterRegistrationTimeout := currentTime.Sub(runnerPod.CreationTimestamp.Add(registrationTimeout))
-		registrationDidTimeout := durationAfterRegistrationTimeout > 0
-
-		if notFound {
-			if registrationDidTimeout {
-				log.Info(
-					"Runner failed to register itself to GitHub in timely manner. "+
-						"Recreating the pod to see if it resolves the issue. "+
-						"CAUTION: If you see this a lot, you should investigate the root cause. "+
-						"See https://github.com/actions-runner-controller/actions-runner-controller/issues/288",
-					"podCreationTimestamp", runnerPod.CreationTimestamp,
-					"currentTime", currentTime,
-					"configuredRegistrationTimeout", registrationTimeout,
-				)
-
-				restart = true
-			} else {
-				log.V(1).Info(
-					"Runner pod exists but we failed to check if runner is busy. Apparently it still needs more time.",
-					"runnerName", runnerPod.Name,
-				)
-			}
-		} else if offline {
-			if registrationDidTimeout {
-				log.Info(
-					"Already existing GitHub runner still appears offline . "+
-						"Recreating the pod to see if it resolves the issue. "+
-						"CAUTION: If you see this a lot, you should investigate the root cause. ",
-					"podCreationTimestamp", runnerPod.CreationTimestamp,
-					"currentTime", currentTime,
-					"configuredRegistrationTimeout", registrationTimeout,
-				)
-
-				restart = true
-			} else {
-				log.V(1).Info(
-					"Runner pod exists but the GitHub runner appears to be still offline. Waiting for runner to get online ...",
-					"runnerName", runnerPod.Name,
-				)
-			}
-		}
-
-		if (notFound || offline) && !registrationDidTimeout {
-			registrationRecheckJitter := 10 * time.Second
-			if r.RegistrationRecheckJitter > 0 {
-				registrationRecheckJitter = r.RegistrationRecheckJitter
-			}
-
-			registrationRecheckDelay = registrationCheckInterval + wait.Jitter(registrationRecheckJitter, 0.1)
-		}
-	}
-
-	// Don't do anything if there's no need to restart the runner
-	if !restart {
-		// This guard enables us to update runner.Status.Phase to `Running` only after
-		// the runner is registered to GitHub.
-		if registrationRecheckDelay > 0 {
-			log.V(1).Info(fmt.Sprintf("Rechecking the runner registration in %s", registrationRecheckDelay))
-
-			updated := runnerPod.DeepCopy()
-			t := time.Now().Format(time.RFC3339)
-			updated.Annotations[AnnotationKeyLastRegistrationCheckTime] = t
-
-			if err := r.Patch(ctx, updated, client.MergeFrom(&runnerPod)); err != nil {
-				log.Error(err, "Failed to update runner pod annotation for LastRegistrationCheckTime")
-				return ctrl.Result{}, err
-			}
-
-			return ctrl.Result{RequeueAfter: registrationRecheckDelay}, nil
-		}
-
-		// Seeing this message, you can expect the runner to become `Running` soon.
-		log.Info(
-			"Runner appears to have registered and running.",
-			"podCreationTimestamp", runnerPod.CreationTimestamp,
-		)
+		// At this point we are sure that the runner has successfully unregistered, hence is safe to be deleted.
+		// But we don't delete the pod here. Instead, let the upstream controller/parent object to delete this pod as
+		// a part of a cascade deletion.
+		// This is to avoid a parent object, like statefulset, to recreate the deleted pod.
+		// If the pod was recreated, it will start a registration process and that may race with the statefulset deleting the pod.
+		log.V(2).Info("Unregistration seems complete")
 
 		return ctrl.Result{}, nil
 	}
 
-	// Delete current pod if recreation is needed
-	if err := r.Delete(ctx, &runnerPod); err != nil {
-		log.Error(err, "Failed to delete pod resource")
-		return ctrl.Result{}, err
-	}
-
-	r.Recorder.Event(&runnerPod, corev1.EventTypeNormal, "PodDeleted", fmt.Sprintf("Deleted pod '%s'", runnerPod.Name))
-	log.Info("Deleted runner pod", "name", runnerPod.Name)
-
 	return ctrl.Result{}, nil
 }
 
-func (r *RunnerPodReconciler) unregisterRunner(ctx context.Context, enterprise, org, repo, name string) (bool, error) {
-	runners, err := r.GitHubClient.ListRunners(ctx, enterprise, org, repo)
-	if err != nil {
-		return false, err
+func (r *RunnerPodReconciler) unregistrationRetryDelay() time.Duration {
+	retryDelay := DefaultUnregistrationRetryDelay
+
+	if r.UnregistrationRetryDelay > 0 {
+		retryDelay = r.UnregistrationRetryDelay
 	}
-
-	var busy bool
-
-	id := int64(0)
-	for _, runner := range runners {
-		if runner.GetName() == name {
-			// Sometimes a runner can stuck "busy" even though it is already "offline".
-			// Thus removing the condition on status can block the runner pod from being terminated forever.
-			busy = runner.GetBusy()
-			if runner.GetStatus() != "offline" && busy {
-				r.Log.Info("This runner will delay the runner pod deletion and the runner deregistration until it becomes either offline or non-busy", "name", runner.GetName(), "status", runner.GetStatus(), "busy", runner.GetBusy())
-				return false, fmt.Errorf("runner is busy")
-			}
-			id = runner.GetID()
-			break
-		}
-	}
-
-	if id == int64(0) {
-		return false, nil
-	}
-
-	// Sometimes a runner can stuck "busy" even though it is already "offline".
-	// Trying to remove the offline but busy runner can result in errors like the following:
-	//    failed to remove runner: DELETE https://api.github.com/repos/actions-runner-controller/mumoshu-actions-test/actions/runners/47: 422 Bad request - Runner \"example-runnerset-0\" is still running a job\" []
-	if !busy {
-		if err := r.GitHubClient.RemoveRunner(ctx, enterprise, org, repo, id); err != nil {
-			return false, err
-		}
-	}
-
-	return true, nil
+	return retryDelay
 }
 
 func (r *RunnerPodReconciler) SetupWithManager(mgr ctrl.Manager) error {

controllers/runner_pod_owner.go

@@ -0,0 +1,600 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type podsForOwner struct {
+	total        int
+	completed    int
+	running      int
+	terminating  int
+	regTimeout   int
+	pending      int
+	templateHash string
+	runner       *v1alpha1.Runner
+	statefulSet  *appsv1.StatefulSet
+	owner        owner
+	object       client.Object
+	synced       bool
+	pods         []corev1.Pod
+}
+
+type owner interface {
+	client.Object
+
+	pods(context.Context, client.Client) ([]corev1.Pod, error)
+	templateHash() (string, bool)
+	withAnnotation(k, v string) client.Object
+	synced() bool
+}
+
+type ownerRunner struct {
+	client.Object
+
+	Log    logr.Logger
+	Runner *v1alpha1.Runner
+}
+
+var _ owner = (*ownerRunner)(nil)
+
+func (r *ownerRunner) pods(ctx context.Context, c client.Client) ([]corev1.Pod, error) {
+	var pod corev1.Pod
+
+	if err := c.Get(ctx, types.NamespacedName{Namespace: r.Runner.Namespace, Name: r.Runner.Name}, &pod); err != nil {
+		if errors.IsNotFound(err) {
+			return nil, nil
+		}
+		r.Log.Error(err, "Failed to get pod managed by runner")
+		return nil, err
+	}
+
+	return []corev1.Pod{pod}, nil
+}
+
+func (r *ownerRunner) templateHash() (string, bool) {
+	return getRunnerTemplateHash(r.Runner)
+}
+
+func (r *ownerRunner) withAnnotation(k, v string) client.Object {
+	copy := r.Runner.DeepCopy()
+	setAnnotation(&copy.ObjectMeta, k, v)
+	return copy
+}
+
+func (r *ownerRunner) synced() bool {
+	return r.Runner.Status.Phase != ""
+}
+
+type ownerStatefulSet struct {
+	client.Object
+
+	Log         logr.Logger
+	StatefulSet *appsv1.StatefulSet
+}
+
+var _ owner = (*ownerStatefulSet)(nil)
+
+func (s *ownerStatefulSet) pods(ctx context.Context, c client.Client) ([]corev1.Pod, error) {
+	var podList corev1.PodList
+
+	if err := c.List(ctx, &podList, client.MatchingLabels(s.StatefulSet.Spec.Template.ObjectMeta.Labels)); err != nil {
+		s.Log.Error(err, "Failed to list pods managed by statefulset")
+		return nil, err
+	}
+
+	var pods []corev1.Pod
+
+	for _, pod := range podList.Items {
+		if owner := metav1.GetControllerOf(&pod); owner == nil || owner.Kind != "StatefulSet" || owner.Name != s.StatefulSet.Name {
+			continue
+		}
+
+		pods = append(pods, pod)
+	}
+
+	return pods, nil
+}
+
+func (s *ownerStatefulSet) templateHash() (string, bool) {
+	return getRunnerTemplateHash(s.StatefulSet)
+}
+
+func (s *ownerStatefulSet) withAnnotation(k, v string) client.Object {
+	copy := s.StatefulSet.DeepCopy()
+	setAnnotation(&copy.ObjectMeta, k, v)
+	return copy
+}
+
+func (s *ownerStatefulSet) synced() bool {
+	var replicas int32 = 1
+	if s.StatefulSet.Spec.Replicas != nil {
+		replicas = *s.StatefulSet.Spec.Replicas
+	}
+
+	if s.StatefulSet.Status.Replicas != replicas {
+		s.Log.V(2).Info("Waiting for statefulset to sync", "desiredReplicas", replicas, "currentReplicas", s.StatefulSet.Status.Replicas)
+		return false
+	}
+
+	return true
+}
+
+func getPodsForOwner(ctx context.Context, c client.Client, log logr.Logger, o client.Object) (*podsForOwner, error) {
+	var (
+		owner       owner
+		runner      *v1alpha1.Runner
+		statefulSet *appsv1.StatefulSet
+		object      client.Object
+	)
+
+	switch v := o.(type) {
+	case *v1alpha1.Runner:
+		owner = &ownerRunner{
+			Log:    log,
+			Runner: v,
+			Object: v,
+		}
+		runner = v
+		object = v
+	case *appsv1.StatefulSet:
+		owner = &ownerStatefulSet{
+			Log:         log,
+			StatefulSet: v,
+			Object:      v,
+		}
+		statefulSet = v
+		object = v
+	default:
+		return nil, fmt.Errorf("BUG: Unsupported runner pods owner %v(%T)", v, v)
+	}
+
+	pods, err := owner.pods(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	var completed, running, terminating, regTimeout, pending, total int
+
+	for _, pod := range pods {
+		total++
+
+		if runnerPodOrContainerIsStopped(&pod) {
+			completed++
+		} else if pod.Status.Phase == corev1.PodRunning {
+			if podRunnerID(&pod) == "" && podConditionTransitionTimeAfter(&pod, corev1.PodReady, registrationTimeout) {
+				log.Info(
+					"Runner failed to register itself to GitHub in timely manner. "+
+						"Recreating the pod to see if it resolves the issue. "+
+						"CAUTION: If you see this a lot, you should investigate the root cause. "+
+						"See https://github.com/actions-runner-controller/actions-runner-controller/issues/288",
+					"creationTimestamp", pod.CreationTimestamp,
+					"readyTransitionTime", podConditionTransitionTime(&pod, corev1.PodReady, corev1.ConditionTrue),
+					"configuredRegistrationTimeout", registrationTimeout,
+				)
+
+				regTimeout++
+			} else {
+				running++
+			}
+		} else if !pod.DeletionTimestamp.IsZero() {
+			terminating++
+		} else {
+			// pending includes running but timedout runner's pod too
+			pending++
+		}
+	}
+
+	templateHash, ok := owner.templateHash()
+	if !ok {
+		log.Info("Failed to get template hash of statefulset. It must be in an invalid state. Please manually delete the statefulset so that it is recreated")
+
+		return nil, nil
+	}
+
+	synced := owner.synced()
+
+	return &podsForOwner{
+		total:        total,
+		completed:    completed,
+		running:      running,
+		terminating:  terminating,
+		regTimeout:   regTimeout,
+		pending:      pending,
+		templateHash: templateHash,
+		runner:       runner,
+		statefulSet:  statefulSet,
+		owner:        owner,
+		object:       object,
+		synced:       synced,
+		pods:         pods,
+	}, nil
+}
+
+func getRunnerTemplateHash(r client.Object) (string, bool) {
+	hash, ok := r.GetLabels()[LabelKeyRunnerTemplateHash]
+
+	return hash, ok
+}
+
+type state struct {
+	podsForOwners map[string][]*podsForOwner
+	lastSyncTime  *time.Time
+}
+
+type result struct {
+	currentObjects []*podsForOwner
+}
+
+// Why `create` must be a function rather than a client.Object? That's becase we use it to create one or more objects on scale up.
+//
+// We use client.Create to create a necessary number of client.Object. client.Create mutates the passed object on a successful creation.
+// It seems to set .Revision at least, and the existence of .Revision let client.Create fail due to K8s restriction that an object being just created
+// can't have .Revision.
+// Now, imagine that you are to add 2 runner replicas on scale up.
+// We create one resource object per a replica that ends up calling 2 client.Create calls.
+// If we were reusing client.Object to be passed to client.Create calls, only the first call suceeeds.
+// The second call fails due to the first call mutated the client.Object to have .Revision.
+// Passing a factory function of client.Object and creating a brand-new client.Object per a client.Create call resolves this issue,
+// allowing us to create two or more replicas in one reconcilation loop without being rejected by K8s.
+func syncRunnerPodsOwners(ctx context.Context, c client.Client, log logr.Logger, effectiveTime *metav1.Time, newDesiredReplicas int, create func() client.Object, ephemeral bool, owners []client.Object) (*result, error) {
+	state, err := collectPodsForOwners(ctx, c, log, owners)
+	if err != nil || state == nil {
+		return nil, err
+	}
+
+	podsForOwnersPerTemplateHash, lastSyncTime := state.podsForOwners, state.lastSyncTime
+
+	// # Why do we recreate statefulsets instead of updating their desired replicas?
+	//
+	// A statefulset cannot add more pods when not all the pods are running.
+	// Our ephemeral runners' pods that have finished running become Completed(Phase=Succeeded).
+	// So creating one statefulset per a batch of ephemeral runners is the only way for us to add more replicas.
+	//
+	// # Why do we recreate statefulsets instead of updating fields other than replicas?
+	//
+	// That's because Kubernetes doesn't allow updating anything other than replicas, template, and updateStrategy.
+	// And the nature of ephemeral runner pods requires you to create a statefulset per a batch of new runner pods so
+	// we have really no other choice.
+	//
+	// If you're curious, the below is the error message you will get when you tried to update forbidden StatefulSet field(s):
+	//
+	// 2021-06-13T07:19:52.760Z        ERROR   actions-runner-controller.runnerset     Failed to patch statefulset
+	// {"runnerset": "default/example-runnerset", "error": "StatefulSet.apps \"example-runnerset\" is invalid: s
+	// pec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'template', and 'updateStrategy'
+	// are forbidden"}
+	//
+	// Even though the error message includes "Forbidden", this error's reason is "Invalid".
+	// So we used to match these errors by using errors.IsInvalid. But that's another story...
+
+	desiredTemplateHash, ok := getRunnerTemplateHash(create())
+	if !ok {
+		log.Info("Failed to get template hash of desired owner resource. It must be in an invalid state. Please manually delete the owner so that it is recreated")
+
+		return nil, nil
+	}
+
+	currentObjects := podsForOwnersPerTemplateHash[desiredTemplateHash]
+
+	sort.SliceStable(currentObjects, func(i, j int) bool {
+		return currentObjects[i].owner.GetCreationTimestamp().Time.Before(currentObjects[j].owner.GetCreationTimestamp().Time)
+	})
+
+	if len(currentObjects) > 0 {
+		timestampFirst := currentObjects[0].owner.GetCreationTimestamp()
+		timestampLast := currentObjects[len(currentObjects)-1].owner.GetCreationTimestamp()
+		var names []string
+		for _, ss := range currentObjects {
+			names = append(names, ss.owner.GetName())
+		}
+		log.V(2).Info("Detected some current object(s)", "creationTimestampFirst", timestampFirst, "creationTimestampLast", timestampLast, "names", names)
+	}
+
+	var total, terminating, pending, running, regTimeout int
+
+	for _, ss := range currentObjects {
+		total += ss.total
+		terminating += ss.terminating
+		pending += ss.pending
+		running += ss.running
+		regTimeout += ss.regTimeout
+	}
+
+	numOwners := len(owners)
+
+	var hashes []string
+	for h, _ := range state.podsForOwners {
+		hashes = append(hashes, h)
+	}
+
+	log.V(2).Info(
+		"Found some pods across owner(s)",
+		"total", total,
+		"terminating", terminating,
+		"pending", pending,
+		"running", running,
+		"regTimeout", regTimeout,
+		"desired", newDesiredReplicas,
+		"owners", numOwners,
+	)
+
+	maybeRunning := pending + running
+
+	wantMoreRunners := newDesiredReplicas > maybeRunning
+	alreadySyncedAfterEffectiveTime := ephemeral && lastSyncTime != nil && effectiveTime != nil && lastSyncTime.After(effectiveTime.Time)
+	runnerPodRecreationDelayAfterWebhookScale := lastSyncTime != nil && time.Now().Before(lastSyncTime.Add(DefaultRunnerPodRecreationDelayAfterWebhookScale))
+
+	log = log.WithValues(
+		"lastSyncTime", lastSyncTime,
+		"effectiveTime", effectiveTime,
+		"templateHashDesired", desiredTemplateHash,
+		"replicasDesired", newDesiredReplicas,
+		"replicasPending", pending,
+		"replicasRunning", running,
+		"replicasMaybeRunning", maybeRunning,
+		"templateHashObserved", hashes,
+	)
+
+	if wantMoreRunners && alreadySyncedAfterEffectiveTime && runnerPodRecreationDelayAfterWebhookScale {
+		// This is our special handling of the situation for ephemeral runners only.
+		//
+		// Handling static runners this way results in scale-up to not work at all,
+		// because then any scale up attempts for static runenrs fall within this condition, for two reasons.
+		// First, static(persistent) runners will never restart on their own.
+		// Second, we don't update EffectiveTime for static runners.
+		//
+		// We do need to skip this condition for static runners, and that's why we take the `ephemeral` flag into account when
+		// computing `alreadySyncedAfterEffectiveTime``.
+
+		log.V(2).Info(
+			"Detected that some ephemeral runners have disappeared. " +
+				"Usually this is due to that ephemeral runner completions " +
+				"so ARC does not create new runners until EffectiveTime is updated, or DefaultRunnerPodRecreationDelayAfterWebhookScale is elapsed.")
+	} else if wantMoreRunners {
+		if alreadySyncedAfterEffectiveTime && !runnerPodRecreationDelayAfterWebhookScale {
+			log.V(2).Info("Adding more replicas because DefaultRunnerPodRecreationDelayAfterWebhookScale has been passed")
+		}
+
+		num := newDesiredReplicas - maybeRunning
+
+		for i := 0; i < num; i++ {
+			// Add more replicas
+			if err := c.Create(ctx, create()); err != nil {
+				return nil, err
+			}
+		}
+
+		log.V(1).Info("Created replica(s)",
+			"created", num,
+		)
+
+		return nil, nil
+	} else if newDesiredReplicas <= running {
+		// If you use ephemeral runners with webhook-based autoscaler and the runner controller is working normally,
+		// you're unlikely to fall into this branch.
+		//
+		// That's because all the stakeholders work like this:
+		//
+		// 1. A runner pod completes with the runner container exiting with code 0
+		// 2. ARC runner controller detects the pod completion, marks the owner(runner or statefulset) resource on k8s for deletion (=Runner.DeletionTimestamp becomes non-zero)
+		// 3. GitHub triggers a corresponding workflow_job "complete" webhook event
+		// 4. ARC github-webhook-server (webhook-based autoscaler) receives the webhook event updates HRA with removing the oldest capacity reservation
+		// 5. ARC horizontalrunnerautoscaler updates RunnerDeployment's desired replicas based on capacity reservations
+		// 6. ARC runnerdeployment controller updates RunnerReplicaSet's desired replicas
+		// 7. (We're here) ARC runnerset or runnerreplicaset controller starts reconciling the owner resource (statefulset or runner)
+		//
+		// In a normally working ARC installation, the runner that was used to run the workflow job should already have been
+		// marked for deletion by the runner controller.
+		// This runnerreplicaset controller doesn't count marked runners into the `running` value, hence you're unlikely to
+		// fall into this branch when you're using ephemeral runners with webhook-based-autoscaler.
+
+		var retained int
+
+		var delete []*podsForOwner
+		for i := len(currentObjects) - 1; i >= 0; i-- {
+			ss := currentObjects[i]
+
+			if ss.running == 0 || retained >= newDesiredReplicas {
+				// In case the desired replicas is satisfied until i-1, or this owner has no running pods,
+				// this owner can be considered safe for deletion.
+				// Note that we already waited on this owner to create pods by waiting for
+				// `.Status.Replicas`(=total number of pods managed by owner, regardless of the runner is Running or Completed) to match the desired replicas in a previous step.
+				// So `.running == 0` means "the owner has created the desired number of pods before, and all of them are completed now".
+				delete = append(delete, ss)
+			} else if retained < newDesiredReplicas {
+				retained += ss.running
+			}
+		}
+
+		if retained == newDesiredReplicas {
+			for _, ss := range delete {
+				log := log.WithValues("owner", types.NamespacedName{Namespace: ss.owner.GetNamespace(), Name: ss.owner.GetName()})
+				// Statefulset termination process 1/4: Set unregistrationRequestTimestamp only after all the pods managed by the statefulset have
+				// started unregistreation process.
+				//
+				// NOTE: We just mark it instead of immediately starting the deletion process.
+				// Otherwise, the runner pod may hit termiationGracePeriod before the unregistration completes(the max terminationGracePeriod is limited to 1h by K8s and a job can be run for more than that),
+				// or actions/runner may potentially misbehave on SIGTERM immediately sent by K8s.
+				// We'd better unregister first and then start a pod deletion process.
+				// The annotation works as a mark to start the pod unregistration and deletion process of ours.
+
+				if _, ok := getAnnotation(ss.owner, AnnotationKeyUnregistrationRequestTimestamp); ok {
+					log.V(2).Info("Still waiting for runner pod(s) unregistration to complete")
+
+					continue
+				}
+
+				for _, po := range ss.pods {
+					if _, err := annotatePodOnce(ctx, c, log, &po, AnnotationKeyUnregistrationRequestTimestamp, time.Now().Format(time.RFC3339)); err != nil {
+						return nil, err
+					}
+				}
+
+				updated := ss.owner.withAnnotation(AnnotationKeyUnregistrationRequestTimestamp, time.Now().Format(time.RFC3339))
+				if err := c.Patch(ctx, updated, client.MergeFrom(ss.owner)); err != nil {
+					log.Error(err, fmt.Sprintf("Failed to patch owner to have %s annotation", AnnotationKeyUnregistrationRequestTimestamp))
+					return nil, err
+				}
+
+				log.V(2).Info("Redundant owner has been annotated to start the unregistration before deletion")
+			}
+		} else if retained > newDesiredReplicas {
+			log.V(2).Info("Waiting sync before scale down", "retained", retained, "newDesiredReplicas", newDesiredReplicas)
+
+			return nil, nil
+		} else {
+			log.Info("Invalid state", "retained", retained, "newDesiredReplicas", newDesiredReplicas)
+			panic("crashed due to invalid state")
+		}
+	}
+
+	for _, sss := range podsForOwnersPerTemplateHash {
+		for _, ss := range sss {
+			if ss.templateHash != desiredTemplateHash {
+				if ss.owner.GetDeletionTimestamp().IsZero() {
+					if err := c.Delete(ctx, ss.object); err != nil {
+						log.Error(err, "Unable to delete object")
+						return nil, err
+					}
+
+					log.V(2).Info("Deleted redundant and outdated object")
+				}
+
+				return nil, nil
+			}
+		}
+	}
+
+	return &result{
+		currentObjects: currentObjects,
+	}, nil
+}
+
+func collectPodsForOwners(ctx context.Context, c client.Client, log logr.Logger, owners []client.Object) (*state, error) {
+	podsForOwnerPerTemplateHash := map[string][]*podsForOwner{}
+
+	// lastSyncTime becomes non-nil only when there are one or more owner(s) hence there are same number of runner pods.
+	// It's used to prevent runnerset-controller from recreating "completed ephemeral runners".
+	// This is needed to prevent runners from being terminated prematurely.
+	// See https://github.com/actions-runner-controller/actions-runner-controller/issues/911 for more context.
+	//
+	// This becomes nil when there are zero statefulset(s). That's fine because then there should be zero stateful(s) to be recreated either hence
+	// we don't need to guard with lastSyncTime.
+	var lastSyncTime *time.Time
+
+	for _, ss := range owners {
+		log := log.WithValues("owner", types.NamespacedName{Namespace: ss.GetNamespace(), Name: ss.GetName()})
+
+		res, err := getPodsForOwner(ctx, c, log, ss)
+		if err != nil {
+			return nil, err
+		}
+
+		if res.templateHash == "" {
+			log.Info("validation error: runner pod owner must have template hash", "object", res.object)
+
+			return nil, nil
+		}
+
+		// Statefulset termination process 4/4: Let Kubernetes cascade-delete the statefulset and the pods.
+		//
+		// If the runner is already marked for deletion(=has a non-zero deletion timestamp) by the runner controller (can be caused by an ephemeral runner completion)
+		// or by this controller (in case it was deleted in the previous reconcilation loop),
+		// we don't need to bother calling GitHub API to re-mark the runner for deletion.
+		// Just hold on, and runners will disappear as long as the runner controller is up and running.
+		if !res.owner.GetDeletionTimestamp().IsZero() {
+			continue
+		}
+
+		// Statefulset termination process 3/4: Set the deletionTimestamp to let Kubernetes start a cascade deletion of the statefulset and the pods.
+		if _, ok := getAnnotation(res.owner, AnnotationKeyUnregistrationCompleteTimestamp); ok {
+			if err := c.Delete(ctx, res.object); err != nil {
+				log.Error(err, "Failed to delete owner")
+				return nil, err
+			}
+
+			log.V(2).Info("Started deletion of owner")
+
+			continue
+		}
+
+		// Statefulset termination process 2/4: Set unregistrationCompleteTimestamp only if all the pods managed by the statefulset
+		// have either unregistered or being deleted.
+		if _, ok := getAnnotation(res.owner, AnnotationKeyUnregistrationRequestTimestamp); ok {
+			var deletionSafe int
+			for _, po := range res.pods {
+				if _, ok := getAnnotation(&po, AnnotationKeyUnregistrationCompleteTimestamp); ok {
+					deletionSafe++
+				} else if !po.DeletionTimestamp.IsZero() {
+					deletionSafe++
+				}
+			}
+
+			if deletionSafe == res.total {
+				log.V(2).Info("Marking owner for unregistration completion", "deletionSafe", deletionSafe, "total", res.total)
+
+				if _, ok := getAnnotation(res.owner, AnnotationKeyUnregistrationCompleteTimestamp); !ok {
+					updated := res.owner.withAnnotation(AnnotationKeyUnregistrationCompleteTimestamp, time.Now().Format(time.RFC3339))
+
+					if err := c.Patch(ctx, updated, client.MergeFrom(res.owner)); err != nil {
+						log.Error(err, fmt.Sprintf("Failed to patch owner to have %s annotation", AnnotationKeyUnregistrationCompleteTimestamp))
+						return nil, err
+					}
+
+					log.V(2).Info("Redundant owner has been annotated to start the deletion")
+				} else {
+					log.V(2).Info("BUG: Redundant owner was already annotated to start the deletion")
+				}
+
+				continue
+			}
+		}
+
+		if annotations := res.owner.GetAnnotations(); annotations != nil {
+			if a, ok := annotations[SyncTimeAnnotationKey]; ok {
+				t, err := time.Parse(time.RFC3339, a)
+				if err == nil {
+					if lastSyncTime == nil || lastSyncTime.Before(t) {
+						lastSyncTime = &t
+					}
+				}
+			}
+		}
+
+		// A completed owner and a completed runner pod can safely be deleted without
+		// a race condition so delete it here,
+		// so that the later process can be a bit simpler.
+		if res.total > 0 && res.total == res.completed {
+			if err := c.Delete(ctx, ss); err != nil {
+				log.Error(err, "Unable to delete owner")
+				return nil, err
+			}
+
+			log.V(2).Info("Deleted completed owner")
+
+			return nil, nil
+		}
+
+		if !res.synced {
+			log.V(1).Info("Skipped reconcilation because owner is not synced yet", "pods", res.pods)
+
+			return nil, nil
+		}
+
+		podsForOwnerPerTemplateHash[res.templateHash] = append(podsForOwnerPerTemplateHash[res.templateHash], res)
+	}
+
+	return &state{podsForOwnerPerTemplateHash, lastSyncTime}, nil
+}

controllers/runnerdeployment_controller.go

@@ -118,6 +118,8 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, err
 		}
 
+		log.Info("Created runnerreplicaset", "runnerreplicaset", desiredRS.Name)
+
 		return ctrl.Result{}, nil
 	}
 
@@ -142,6 +144,8 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, err
 		}
 
+		log.Info("Created runnerreplicaset", "runnerreplicaset", desiredRS.Name)
+
 		// We requeue in order to clean up old runner replica sets later.
 		// Otherwise, they aren't cleaned up until the next re-sync interval.
 		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
@@ -177,6 +181,7 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
 	if currentDesiredReplicas != newDesiredReplicas {
 		newestSet.Spec.Replicas = &newDesiredReplicas
+		newestSet.Spec.EffectiveTime = rd.Spec.EffectiveTime
 
 		if err := r.Client.Update(ctx, newestSet); err != nil {
 			log.Error(err, "Failed to update runnerreplicaset resource")
@@ -221,15 +226,38 @@ func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		for i := range oldSets {
 			rs := oldSets[i]
 
+			rslog := log.WithValues("runnerreplicaset", rs.Name)
+
+			if rs.Status.Replicas != nil && *rs.Status.Replicas > 0 {
+				if rs.Spec.Replicas != nil && *rs.Spec.Replicas == 0 {
+					rslog.V(2).Info("Waiting for runnerreplicaset to scale to zero")
+
+					continue
+				}
+
+				updated := rs.DeepCopy()
+				zero := 0
+				updated.Spec.Replicas = &zero
+				if err := r.Client.Update(ctx, updated); err != nil {
+					rslog.Error(err, "Failed to scale runnerreplicaset to zero")
+
+					return ctrl.Result{}, err
+				}
+
+				rslog.Info("Scaled runnerreplicaset to zero")
+
+				continue
+			}
+
 			if err := r.Client.Delete(ctx, &rs); err != nil {
-				log.Error(err, "Failed to delete runnerreplicaset resource")
+				rslog.Error(err, "Failed to delete runnerreplicaset resource")
 
 				return ctrl.Result{}, err
 			}
 
 			r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerReplicaSetDeleted", fmt.Sprintf("Deleted runnerreplicaset '%s'", rs.Name))
 
-			log.Info("Deleted runnerreplicaset", "runnerdeployment", rd.ObjectMeta.Name, "runnerreplicaset", rs.Name)
+			rslog.Info("Deleted runnerreplicaset")
 		}
 	}
 
@@ -417,9 +445,10 @@ func newRunnerReplicaSet(rd *v1alpha1.RunnerDeployment, commonRunnerLabels []str
 			Labels:       newRSTemplate.ObjectMeta.Labels,
 		},
 		Spec: v1alpha1.RunnerReplicaSetSpec{
-			Replicas: rd.Spec.Replicas,
-			Selector: newRSSelector,
-			Template: newRSTemplate,
+			Replicas:      rd.Spec.Replicas,
+			Selector:      newRSSelector,
+			Template:      newRSTemplate,
+			EffectiveTime: rd.Spec.EffectiveTime,
 		},
 	}
 

controllers/runnerreplicaset_controller.go

@@ -18,21 +18,17 @@ package controllers
 
 import (
 	"context"
-	"errors"
-	"fmt"
 	"reflect"
 	"time"
 
-	gogithub "github.com/google/go-github/v37/github"
-
 	"github.com/go-logr/logr"
+
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
@@ -49,6 +45,10 @@ type RunnerReplicaSetReconciler struct {
 	Name         string
 }
 
+const (
+	SyncTimeAnnotationKey = "sync-time"
+)
+
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
@@ -65,18 +65,42 @@ func (r *RunnerReplicaSetReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	}
 
 	if !rs.ObjectMeta.DeletionTimestamp.IsZero() {
+		// RunnerReplicaSet cannot be gracefuly removed.
+		// That means any runner that is running a job can be prematurely terminated.
+		// To gracefully remove a RunnerReplicaSet, scale it down to zero first, observe RunnerReplicaSet's status replicas,
+		// and remove it only after the status replicas becomes zero.
 		return ctrl.Result{}, nil
 	}
 
+	if rs.ObjectMeta.Labels == nil {
+		rs.ObjectMeta.Labels = map[string]string{}
+	}
+
+	// Template hash is usually set by the upstream controller(RunnerDeplloyment controller) on authoring
+	// RunerReplicaset resource, but it may be missing when the user directly created RunnerReplicaSet.
+	// As a template hash is required by by the runner replica management, we dynamically add it here without ever persisting it.
+	if rs.ObjectMeta.Labels[LabelKeyRunnerTemplateHash] == "" {
+		template := rs.Spec.DeepCopy()
+		template.Replicas = nil
+		template.EffectiveTime = nil
+		templateHash := ComputeHash(template)
+
+		log.Info("Using auto-generated template hash", "value", templateHash)
+
+		rs.ObjectMeta.Labels = CloneAndAddLabel(rs.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash)
+		rs.Spec.Template.ObjectMeta.Labels = CloneAndAddLabel(rs.Spec.Template.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash)
+	}
+
 	selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
+
 	// Get the Runners managed by the target RunnerReplicaSet
-	var allRunners v1alpha1.RunnerList
+	var runnerList v1alpha1.RunnerList
 	if err := r.List(
 		ctx,
-		&allRunners,
+		&runnerList,
 		client.InNamespace(req.Namespace),
 		client.MatchingLabelsSelector{Selector: selector},
 	); err != nil {
@@ -85,209 +109,44 @@ func (r *RunnerReplicaSetReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		}
 	}
 
-	var myRunners []v1alpha1.Runner
+	replicas := 1
+	if rs.Spec.Replicas != nil {
+		replicas = *rs.Spec.Replicas
+	}
+
+	effectiveTime := rs.Spec.EffectiveTime
+	ephemeral := rs.Spec.Template.Spec.Ephemeral == nil || *rs.Spec.Template.Spec.Ephemeral
+
+	desired, err := r.newRunner(rs)
+	if err != nil {
+		log.Error(err, "Could not create runner")
+
+		return ctrl.Result{}, err
+	}
+
+	var live []client.Object
+	for _, r := range runnerList.Items {
+		r := r
+		live = append(live, &r)
+	}
+
+	res, err := syncRunnerPodsOwners(ctx, r.Client, log, effectiveTime, replicas, func() client.Object { return desired.DeepCopy() }, ephemeral, live)
+	if err != nil || res == nil {
+		return ctrl.Result{}, err
+	}
 
 	var (
-		current   int
-		ready     int
-		available int
+		status v1alpha1.RunnerReplicaSetStatus
+
+		current, available, ready int
 	)
 
-	for _, r := range allRunners.Items {
-		// This guard is required to avoid the RunnerReplicaSet created by the controller v0.17.0 or before
-		// to not treat all the runners in the namespace as its children.
-		if metav1.IsControlledBy(&r, &rs) && !metav1.HasAnnotation(r.ObjectMeta, annotationKeyRegistrationOnly) {
-			myRunners = append(myRunners, r)
-
-			current += 1
-
-			if r.Status.Phase == string(corev1.PodRunning) {
-				ready += 1
-				// available is currently the same as ready, as we don't yet have minReadySeconds for runners
-				available += 1
-			}
-		}
+	for _, o := range res.currentObjects {
+		current += o.total
+		available += o.running
+		ready += o.running
 	}
 
-	var desired int
-
-	if rs.Spec.Replicas != nil {
-		desired = *rs.Spec.Replicas
-	} else {
-		desired = 1
-	}
-
-	registrationOnlyRunnerNsName := req.NamespacedName
-	registrationOnlyRunnerNsName.Name = registrationOnlyRunnerNameFor(rs.Name)
-	registrationOnlyRunner := v1alpha1.Runner{}
-	registrationOnlyRunnerExists := false
-	if err := r.Get(
-		ctx,
-		registrationOnlyRunnerNsName,
-		&registrationOnlyRunner,
-	); err != nil {
-		if !kerrors.IsNotFound(err) {
-			return ctrl.Result{}, err
-		}
-	} else {
-		registrationOnlyRunnerExists = true
-	}
-
-	// On scale to zero, we must have fully registered registration-only runner before we start deleting other runners, hence `desired == 0`
-	// On scale from zero, we must retain the registratoin-only runner until one or more other runners get registered, hence `registrationOnlyRunnerExists && available == 0`.
-	// On RunnerReplicaSet creation, it have always 0 replics and no registration-only runner.
-	// In this case We don't need to bother creating a registration-only runner which gets deleted soon after we have 1 or more available repolicas,
-	// hence it's not `available == 0`, but `registrationOnlyRunnerExists && available == 0`.
-	// See https://github.com/actions-runner-controller/actions-runner-controller/issues/516
-	registrationOnlyRunnerNeeded := desired == 0 || (registrationOnlyRunnerExists && current == 0)
-
-	if registrationOnlyRunnerNeeded {
-		if registrationOnlyRunnerExists {
-			if registrationOnlyRunner.Status.Phase == "" {
-				log.Info("Still waiting for the registration-only runner to be registered")
-
-				return ctrl.Result{}, nil
-			}
-		} else {
-			// A registration-only runner does not exist and is needed, hence create it.
-
-			runnerForScaleFromToZero, err := r.newRunner(rs)
-			if err != nil {
-				return ctrl.Result{}, fmt.Errorf("failed to create runner for scale from/to zero: %v", err)
-			}
-
-			runnerForScaleFromToZero.ObjectMeta.Name = registrationOnlyRunnerNsName.Name
-			runnerForScaleFromToZero.ObjectMeta.GenerateName = ""
-			runnerForScaleFromToZero.ObjectMeta.Labels = nil
-			metav1.SetMetaDataAnnotation(&runnerForScaleFromToZero.ObjectMeta, annotationKeyRegistrationOnly, "true")
-
-			if err := r.Client.Create(ctx, &runnerForScaleFromToZero); err != nil {
-				log.Error(err, "Failed to create runner for scale from/to zero")
-
-				return ctrl.Result{}, err
-			}
-
-			// We can continue to deleting runner pods only after the
-			// registration-only runner gets registered.
-			return ctrl.Result{}, nil
-		}
-	} else {
-		// A registration-only runner exists and is not needed, hence delete it.
-		if registrationOnlyRunnerExists {
-			if err := r.Client.Delete(ctx, &registrationOnlyRunner); err != nil {
-				log.Error(err, "Retrying soon because we failed to delete registration-only runner")
-
-				return ctrl.Result{Requeue: true}, nil
-			}
-		}
-	}
-
-	if current > desired {
-		n := current - desired
-
-		log.V(0).Info(fmt.Sprintf("Deleting %d runners", n), "desired", desired, "current", current, "ready", ready)
-
-		// get runners that are currently offline/not busy/timed-out to register
-		var deletionCandidates []v1alpha1.Runner
-
-		for _, runner := range allRunners.Items {
-			busy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
-			if err != nil {
-				notRegistered := false
-				offline := false
-
-				var notFoundException *github.RunnerNotFound
-				var offlineException *github.RunnerOffline
-				if errors.As(err, &notFoundException) {
-					log.V(1).Info("Failed to check if runner is busy. Either this runner has never been successfully registered to GitHub or it still needs more time.", "runnerName", runner.Name)
-					notRegistered = true
-				} else if errors.As(err, &offlineException) {
-					offline = true
-				} else {
-					var e *gogithub.RateLimitError
-					if errors.As(err, &e) {
-						// We log the underlying error when we failed calling GitHub API to list or unregisters,
-						// or the runner is still busy.
-						log.Error(
-							err,
-							fmt.Sprintf(
-								"Failed to check if runner is busy due to GitHub API rate limit. Retrying in %s to avoid excessive GitHub API calls",
-								retryDelayOnGitHubAPIRateLimitError,
-							),
-						)
-
-						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
-					}
-
-					return ctrl.Result{}, err
-				}
-
-				registrationTimeout := 15 * time.Minute
-				currentTime := time.Now()
-				registrationDidTimeout := currentTime.Sub(runner.CreationTimestamp.Add(registrationTimeout)) > 0
-
-				if notRegistered && registrationDidTimeout {
-					log.Info(
-						"Runner failed to register itself to GitHub in timely manner. "+
-							"Marking the runner for scale down. "+
-							"CAUTION: If you see this a lot, you should investigate the root cause. "+
-							"See https://github.com/actions-runner-controller/actions-runner-controller/issues/288",
-						"runnerCreationTimestamp", runner.CreationTimestamp,
-						"currentTime", currentTime,
-						"configuredRegistrationTimeout", registrationTimeout,
-					)
-
-					deletionCandidates = append(deletionCandidates, runner)
-				}
-
-				// offline runners should always be a great target for scale down
-				if offline {
-					deletionCandidates = append(deletionCandidates, runner)
-				}
-			} else if !busy {
-				deletionCandidates = append(deletionCandidates, runner)
-			}
-		}
-
-		if len(deletionCandidates) < n {
-			n = len(deletionCandidates)
-		}
-
-		log.V(0).Info(fmt.Sprintf("Deleting %d runner(s)", n), "desired", desired, "current", current, "ready", ready)
-
-		for i := 0; i < n; i++ {
-			if err := r.Client.Delete(ctx, &deletionCandidates[i]); client.IgnoreNotFound(err) != nil {
-				log.Error(err, "Failed to delete runner resource")
-
-				return ctrl.Result{}, err
-			}
-
-			r.Recorder.Event(&rs, corev1.EventTypeNormal, "RunnerDeleted", fmt.Sprintf("Deleted runner '%s'", deletionCandidates[i].Name))
-			log.Info("Deleted runner")
-		}
-	} else if desired > current {
-		n := desired - current
-
-		log.V(0).Info(fmt.Sprintf("Creating %d runner(s)", n), "desired", desired, "available", current, "ready", ready)
-
-		for i := 0; i < n; i++ {
-			newRunner, err := r.newRunner(rs)
-			if err != nil {
-				log.Error(err, "Could not create runner")
-
-				return ctrl.Result{}, err
-			}
-
-			if err := r.Client.Create(ctx, &newRunner); err != nil {
-				log.Error(err, "Failed to create runner resource")
-
-				return ctrl.Result{}, err
-			}
-		}
-	}
-
-	var status v1alpha1.RunnerReplicaSetStatus
-
 	status.Replicas = &current
 	status.AvailableReplicas = &available
 	status.ReadyReplicas = &ready
@@ -308,10 +167,16 @@ func (r *RunnerReplicaSetReconciler) Reconcile(ctx context.Context, req ctrl.Req
 }
 
 func (r *RunnerReplicaSetReconciler) newRunner(rs v1alpha1.RunnerReplicaSet) (v1alpha1.Runner, error) {
+	// Note that the upstream controller (runnerdeployment) is expected to add
+	// the "runner template hash" label to the template.meta which is necessary to make this controller work correctly
 	objectMeta := rs.Spec.Template.ObjectMeta.DeepCopy()
 
 	objectMeta.GenerateName = rs.ObjectMeta.Name + "-"
 	objectMeta.Namespace = rs.ObjectMeta.Namespace
+	if objectMeta.Annotations == nil {
+		objectMeta.Annotations = map[string]string{}
+	}
+	objectMeta.Annotations[SyncTimeAnnotationKey] = time.Now().Format(time.RFC3339)
 
 	runner := v1alpha1.Runner{
 		TypeMeta:   metav1.TypeMeta{},

controllers/runnerreplicaset_controller_test.go

@@ -2,13 +2,11 @@ package controllers
 
 import (
 	"context"
-	"fmt"
 	"math/rand"
 	"net/http/httptest"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -103,12 +101,40 @@ func intPtr(v int) *int {
 var _ = Context("Inside of a new namespace", func() {
 	ctx := context.TODO()
 	ns := SetupTest(ctx)
+	name := "example-runnerreplicaset"
 
-	Describe("when no existing resources exist", func() {
+	getRunnerCount := func() int {
+		runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}}
 
-		It("should create a new Runner resource from the specified template, add a another Runner on replicas increased, and removes all the replicas when set to 0", func() {
-			name := "example-runnerreplicaset"
+		selector, err := metav1.LabelSelectorAsSelector(
+			&metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"foo": "bar",
+				},
+			},
+		)
+		if err != nil {
+			logf.Log.Error(err, "failed to create labelselector")
+			return -1
+		}
 
+		err = k8sClient.List(
+			ctx,
+			&runners,
+			client.InNamespace(ns.Name),
+			client.MatchingLabelsSelector{Selector: selector},
+		)
+		if err != nil {
+			logf.Log.Error(err, "list runners")
+		}
+
+		runnersList.Sync(runners.Items)
+
+		return len(runners.Items)
+	}
+
+	Describe("RunnerReplicaSet", func() {
+		It("should create a new Runner resource from the specified template", func() {
 			{
 				rs := &actionsv1alpha1.RunnerReplicaSet{
 					ObjectMeta: metav1.ObjectMeta{
@@ -147,142 +173,99 @@ var _ = Context("Inside of a new namespace", func() {
 
 				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
 
-				runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}}
-
 				Eventually(
-					func() int {
-						selector, err := metav1.LabelSelectorAsSelector(
-							&metav1.LabelSelector{
-								MatchLabels: map[string]string{
-									"foo": "bar",
-								},
-							},
-						)
-						if err != nil {
-							logf.Log.Error(err, "failed to create labelselector")
-							return -1
-						}
-						err = k8sClient.List(
-							ctx,
-							&runners,
-							client.InNamespace(ns.Name),
-							client.MatchingLabelsSelector{Selector: selector},
-						)
-						if err != nil {
-							logf.Log.Error(err, "list runners")
-							return -1
-						}
-
-						runnersList.Sync(runners.Items)
-
-						return len(runners.Items)
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+					getRunnerCount,
+					time.Second*5, time.Second).Should(BeEquivalentTo(1))
 			}
+		})
 
+		It("should create 2 runners when specified 2 replicas", func() {
 			{
-				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
-				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
-				//   Operation cannot be fulfilled on runnerreplicasets.actions.summerwind.dev "example-runnerreplicaset": the object has been modified; please apply your changes to the latest version and try again
-				Eventually(func() error {
-					var rs actionsv1alpha1.RunnerReplicaSet
-
-					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rs)
-
-					Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerReplicaSet resource")
-
-					rs.Spec.Replicas = intPtr(2)
-
-					return k8sClient.Update(ctx, &rs)
-				},
-					time.Second*1, time.Millisecond*500).Should(BeNil())
-
-				runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}}
-
-				Eventually(
-					func() int {
-						selector, err := metav1.LabelSelectorAsSelector(
-							&metav1.LabelSelector{
-								MatchLabels: map[string]string{
-									"foo": "bar",
-								},
-							},
-						)
-						if err != nil {
-							logf.Log.Error(err, "failed to create labelselector")
-							return -1
-						}
-						err = k8sClient.List(
-							ctx,
-							&runners,
-							client.InNamespace(ns.Name),
-							client.MatchingLabelsSelector{Selector: selector},
-						)
-						if err != nil {
-							logf.Log.Error(err, "list runners")
-						}
-
-						runnersList.Sync(runners.Items)
-
-						return len(runners.Items)
+				rs := &actionsv1alpha1.RunnerReplicaSet{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
 					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
-			}
-
-			{
-				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
-				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
-				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
-				Eventually(func() error {
-					var rs actionsv1alpha1.RunnerReplicaSet
-
-					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rs)
-
-					Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerReplicaSet resource")
-
-					rs.Spec.Replicas = intPtr(0)
-
-					return k8sClient.Update(ctx, &rs)
-				},
-					time.Second*1, time.Millisecond*500).Should(BeNil())
-
-				runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}}
-
-				Eventually(
-					func() int {
-						selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+					Spec: actionsv1alpha1.RunnerReplicaSetSpec{
+						Replicas: intPtr(2),
+						Selector: &metav1.LabelSelector{
 							MatchLabels: map[string]string{
 								"foo": "bar",
 							},
-						})
-						Expect(err).ToNot(HaveOccurred())
-
-						var regOnly actionsv1alpha1.Runner
-						if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: registrationOnlyRunnerNameFor(name)}, &regOnly); err != nil {
-							logf.Log.Info(fmt.Sprintf("Failed getting registration-only runner in test: %v", err))
-							return -1
-						} else {
-							updated := regOnly.DeepCopy()
-							updated.Status.Phase = "Completed"
-
-							if err := k8sClient.Status().Patch(ctx, updated, client.MergeFrom(&regOnly)); err != nil {
-								logf.Log.Info(fmt.Sprintf("Failed updating registration-only runner in test: %v", err))
-								return -1
-							}
-
-							runnersList.AddOffline([]actionsv1alpha1.Runner{*updated})
-						}
-
-						if err := k8sClient.List(ctx, &runners, client.InNamespace(ns.Name), client.MatchingLabelsSelector{Selector: selector}); err != nil {
-							logf.Log.Error(err, "list runners")
-							return -1
-						}
-
-						runnersList.Sync(runners.Items)
-
-						return len(runners.Items)
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
 					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(0))
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				Eventually(
+					getRunnerCount,
+					time.Second*5, time.Second).Should(BeEquivalentTo(2))
+			}
+		})
+
+		It("should not create any runners when specified 0 replicas", func() {
+			{
+				rs := &actionsv1alpha1.RunnerReplicaSet{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerReplicaSetSpec{
+						Replicas: intPtr(0),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				Consistently(
+					getRunnerCount,
+					time.Second*5, time.Second).Should(BeEquivalentTo(0))
 			}
 		})
 	})

controllers/runnerset_controller.go

@@ -22,10 +22,7 @@ import (
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/types"
 
-	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -36,10 +33,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/controllers/metrics"
-)
-
-const (
-	LabelKeyRunnerSetName = "runnerset-name"
+	"github.com/go-logr/logr"
 )
 
 // RunnerSetReconciler reconciles a Runner object
@@ -51,11 +45,12 @@ type RunnerSetReconciler struct {
 	Recorder record.EventRecorder
 	Scheme   *runtime.Scheme
 
-	CommonRunnerLabels   []string
-	GitHubBaseURL        string
-	RunnerImage          string
-	DockerImage          string
-	DockerRegistryMirror string
+	CommonRunnerLabels     []string
+	GitHubBaseURL          string
+	RunnerImage            string
+	RunnerImagePullSecrets []string
+	DockerImage            string
+	DockerRegistryMirror   string
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets,verbs=get;list;watch;create;update;patch;delete
@@ -89,6 +84,18 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	metrics.SetRunnerSet(*runnerSet)
 
+	var statefulsetList appsv1.StatefulSetList
+	if err := r.List(ctx, &statefulsetList, client.InNamespace(req.Namespace), client.MatchingFields{runnerSetOwnerKey: req.Name}); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	statefulsets := statefulsetList.Items
+
+	if len(statefulsets) > 1000 {
+		log.Info("Postponed reconcilation to prevent potential infinite loop. If you're really scaling more than 1000 statefulsets, do change this hard-coded threshold!")
+		return ctrl.Result{}, nil
+	}
+
 	desiredStatefulSet, err := r.newStatefulSet(runnerSet)
 	if err != nil {
 		r.Recorder.Event(runnerSet, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
@@ -98,108 +105,43 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, err
 	}
 
-	liveStatefulSet := &appsv1.StatefulSet{}
-	if err := r.Get(ctx, types.NamespacedName{Namespace: runnerSet.Namespace, Name: runnerSet.Name}, liveStatefulSet); err != nil {
-		if !errors.IsNotFound(err) {
-			log.Error(err, "Failed to get live statefulset")
-
-			return ctrl.Result{}, err
-		}
-
-		if err := r.Client.Create(ctx, desiredStatefulSet); err != nil {
-			log.Error(err, "Failed to create statefulset resource")
-
-			return ctrl.Result{}, err
-		}
-
-		return ctrl.Result{}, nil
-	}
-
-	liveTemplateHash, ok := getStatefulSetTemplateHash(liveStatefulSet)
-	if !ok {
-		log.Info("Failed to get template hash of newest statefulset resource. It must be in an invalid state. Please manually delete the statefulset so that it is recreated")
-
-		return ctrl.Result{}, nil
-	}
-
-	desiredTemplateHash, ok := getStatefulSetTemplateHash(desiredStatefulSet)
-	if !ok {
-		log.Info("Failed to get template hash of desired statefulset. It must be in an invalid state. Please manually delete the statefulset so that it is recreated")
-
-		return ctrl.Result{}, nil
-	}
-
-	if liveTemplateHash != desiredTemplateHash {
-		copy := liveStatefulSet.DeepCopy()
-		copy.Spec = desiredStatefulSet.Spec
-
-		if err := r.Client.Patch(ctx, copy, client.MergeFrom(liveStatefulSet)); err != nil {
-			log.Error(err, "Failed to patch statefulset", "reason", errors.ReasonForError(err))
-
-			if errors.IsInvalid(err) {
-				// NOTE: This might not be ideal but deal the forbidden error by recreating the statefulset
-				// Probably we'd better create a registration-only runner to prevent queued jobs from immediately failing.
-				//
-				// 2021-06-13T07:19:52.760Z        ERROR   actions-runner-controller.runnerset     Failed to patch statefulset
-				// {"runnerset": "default/example-runnerset", "error": "StatefulSet.apps \"example-runnerset\" is invalid: s
-				// pec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'template', and 'updateStrategy'
-				// are forbidden"}
-				//
-				// Even though the error message includes "Forbidden", this error's reason is "Invalid".
-				// That's why we're using errors.IsInvalid above.
-
-				if err := r.Client.Delete(ctx, liveStatefulSet); err != nil {
-					log.Error(err, "Failed to delete statefulset for force-update")
-					return ctrl.Result{}, err
-				}
-				log.Info("Deleted statefulset for force-update")
-			}
-
-			return ctrl.Result{}, err
-		}
-
-		// We requeue in order to clean up old runner replica sets later.
-		// Otherwise, they aren't cleaned up until the next re-sync interval.
-		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
-	}
+	addedReplicas := int32(1)
+	create := desiredStatefulSet.DeepCopy()
+	create.Spec.Replicas = &addedReplicas
 
 	const defaultReplicas = 1
 
-	var replicasOfLiveStatefulSet *int
-	if liveStatefulSet.Spec.Replicas != nil {
-		v := int(*liveStatefulSet.Spec.Replicas)
-		replicasOfLiveStatefulSet = &v
-	}
-
 	var replicasOfDesiredStatefulSet *int
 	if desiredStatefulSet.Spec.Replicas != nil {
 		v := int(*desiredStatefulSet.Spec.Replicas)
 		replicasOfDesiredStatefulSet = &v
 	}
 
-	currentDesiredReplicas := getIntOrDefault(replicasOfLiveStatefulSet, defaultReplicas)
 	newDesiredReplicas := getIntOrDefault(replicasOfDesiredStatefulSet, defaultReplicas)
 
-	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
-	if currentDesiredReplicas != newDesiredReplicas {
-		v := int32(newDesiredReplicas)
+	effectiveTime := runnerSet.Spec.EffectiveTime
+	ephemeral := runnerSet.Spec.Ephemeral == nil || *runnerSet.Spec.Ephemeral
 
-		updated := liveStatefulSet.DeepCopy()
-		updated.Spec.Replicas = &v
+	var owners []client.Object
 
-		if err := r.Client.Patch(ctx, updated, client.MergeFrom(liveStatefulSet)); err != nil {
-			log.Error(err, "Failed to update statefulset")
-
-			return ctrl.Result{}, err
-		}
-
-		return ctrl.Result{}, nil
+	for _, ss := range statefulsets {
+		ss := ss
+		owners = append(owners, &ss)
 	}
 
-	statusReplicas := int(liveStatefulSet.Status.Replicas)
-	statusReadyReplicas := int(liveStatefulSet.Status.ReadyReplicas)
-	totalCurrentReplicas := int(liveStatefulSet.Status.CurrentReplicas)
-	updatedReplicas := int(liveStatefulSet.Status.UpdatedReplicas)
+	res, err := syncRunnerPodsOwners(ctx, r.Client, log, effectiveTime, newDesiredReplicas, func() client.Object { return create.DeepCopy() }, ephemeral, owners)
+	if err != nil || res == nil {
+		return ctrl.Result{}, err
+	}
+
+	var statusReplicas, statusReadyReplicas, totalCurrentReplicas, updatedReplicas int
+
+	for _, ss := range res.currentObjects {
+		statusReplicas += int(ss.statefulSet.Status.Replicas)
+		statusReadyReplicas += int(ss.statefulSet.Status.ReadyReplicas)
+		totalCurrentReplicas += int(ss.statefulSet.Status.CurrentReplicas)
+		updatedReplicas += int(ss.statefulSet.Status.UpdatedReplicas)
+	}
 
 	status := runnerSet.Status.DeepCopy()
 
@@ -224,12 +166,6 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	return ctrl.Result{}, nil
 }
 
-func getStatefulSetTemplateHash(rs *appsv1.StatefulSet) (string, bool) {
-	hash, ok := rs.Labels[LabelKeyRunnerTemplateHash]
-
-	return hash, ok
-}
-
 func getRunnerSetSelector(runnerSet *v1alpha1.RunnerSet) *metav1.LabelSelector {
 	selector := runnerSet.Spec.Selector
 	if selector == nil {
@@ -249,17 +185,12 @@ func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*ap
 		runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, l)
 	}
 
-	// This label selector is used by default when rd.Spec.Selector is empty.
-	runnerSetWithOverrides.Template.ObjectMeta.Labels = CloneAndAddLabel(runnerSetWithOverrides.Template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerSet.Name)
-
-	runnerSetWithOverrides.Template.ObjectMeta.Labels = CloneAndAddLabel(runnerSetWithOverrides.Template.ObjectMeta.Labels, LabelKeyPodMutation, LabelValuePodMutation)
-
 	template := corev1.Pod{
 		ObjectMeta: runnerSetWithOverrides.StatefulSetSpec.Template.ObjectMeta,
 		Spec:       runnerSetWithOverrides.StatefulSetSpec.Template.Spec,
 	}
 
-	pod, err := newRunnerPod(template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL, false)
+	pod, err := newRunnerPod(runnerSet.Name, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, r.GitHubBaseURL, false)
 	if err != nil {
 		return nil, err
 	}
@@ -288,9 +219,12 @@ func (r *RunnerSetReconciler) newStatefulSet(runnerSet *v1alpha1.RunnerSet) (*ap
 	rs := appsv1.StatefulSet{
 		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      runnerSet.ObjectMeta.Name,
-			Namespace: runnerSet.ObjectMeta.Namespace,
-			Labels:    CloneAndAddLabel(runnerSet.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash),
+			GenerateName: runnerSet.ObjectMeta.Name + "-",
+			Namespace:    runnerSet.ObjectMeta.Namespace,
+			Labels:       CloneAndAddLabel(runnerSet.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash),
+			Annotations: map[string]string{
+				SyncTimeAnnotationKey: time.Now().Format(time.RFC3339),
+			},
 		},
 		Spec: runnerSetWithOverrides.StatefulSetSpec,
 	}
@@ -310,6 +244,22 @@ func (r *RunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	r.Recorder = mgr.GetEventRecorderFor(name)
 
+	if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &appsv1.StatefulSet{}, runnerSetOwnerKey, func(rawObj client.Object) []string {
+		set := rawObj.(*appsv1.StatefulSet)
+		owner := metav1.GetControllerOf(set)
+		if owner == nil {
+			return nil
+		}
+
+		if owner.APIVersion != v1alpha1.GroupVersion.String() || owner.Kind != "RunnerSet" {
+			return nil
+		}
+
+		return []string{owner.Name}
+	}); err != nil {
+		return err
+	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.RunnerSet{}).
 		Owns(&appsv1.StatefulSet{}).

controllers/testdata/org_webhook_workflow_job_with_self_hosted_label_payload.json

@@ -0,0 +1,152 @@
+{
+    "action": "queued",
+    "workflow_job": {
+        "id": 1234567890,
+        "run_id": 1234567890,
+        "run_url": "https://api.github.com/repos/MYORG/MYREPO/actions/runs/1234567890",
+        "node_id": "CR_kwDOGCados7e1x2g",
+        "head_sha": "1234567890123456789012345678901234567890",
+        "url": "https://api.github.com/repos/MYORG/MYREPO/actions/jobs/1234567890",
+        "html_url": "https://github.com/MYORG/MYREPO/runs/1234567890",
+        "status": "queued",
+        "conclusion": null,
+        "started_at": "2021-09-28T23:45:29Z",
+        "completed_at": null,
+        "name": "build",
+        "steps": [],
+        "check_run_url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/1234567890",
+        "labels": [
+            "self-hosted",
+            "label1"
+        ]
+    },
+    "repository": {
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ=",
+        "name": "MYREPO",
+        "full_name": "MYORG/MYREPO",
+        "private": true,
+        "owner": {
+            "login": "MYORG",
+            "id": 1234567890,
+            "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+            "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+            "gravatar_id": "",
+            "url": "https://api.github.com/users/MYORG",
+            "html_url": "https://github.com/MYORG",
+            "followers_url": "https://api.github.com/users/MYORG/followers",
+            "following_url": "https://api.github.com/users/MYORG/following{/other_user}",
+            "gists_url": "https://api.github.com/users/MYORG/gists{/gist_id}",
+            "starred_url": "https://api.github.com/users/MYORG/starred{/owner}{/repo}",
+            "subscriptions_url": "https://api.github.com/users/MYORG/subscriptions",
+            "organizations_url": "https://api.github.com/users/MYORG/orgs",
+            "repos_url": "https://api.github.com/users/MYORG/repos",
+            "events_url": "https://api.github.com/users/MYORG/events{/privacy}",
+            "received_events_url": "https://api.github.com/users/MYORG/received_events",
+            "type": "Organization",
+            "site_admin": false
+        },
+        "html_url": "https://github.com/MYORG/MYREPO",
+        "description": "MYREPO",
+        "fork": false,
+        "url": "https://api.github.com/repos/MYORG/MYREPO",
+        "forks_url": "https://api.github.com/repos/MYORG/MYREPO/forks",
+        "keys_url": "https://api.github.com/repos/MYORG/MYREPO/keys{/key_id}",
+        "collaborators_url": "https://api.github.com/repos/MYORG/MYREPO/collaborators{/collaborator}",
+        "teams_url": "https://api.github.com/repos/MYORG/MYREPO/teams",
+        "hooks_url": "https://api.github.com/repos/MYORG/MYREPO/hooks",
+        "issue_events_url": "https://api.github.com/repos/MYORG/MYREPO/issues/events{/number}",
+        "events_url": "https://api.github.com/repos/MYORG/MYREPO/events",
+        "assignees_url": "https://api.github.com/repos/MYORG/MYREPO/assignees{/user}",
+        "branches_url": "https://api.github.com/repos/MYORG/MYREPO/branches{/branch}",
+        "tags_url": "https://api.github.com/repos/MYORG/MYREPO/tags",
+        "blobs_url": "https://api.github.com/repos/MYORG/MYREPO/git/blobs{/sha}",
+        "git_tags_url": "https://api.github.com/repos/MYORG/MYREPO/git/tags{/sha}",
+        "git_refs_url": "https://api.github.com/repos/MYORG/MYREPO/git/refs{/sha}",
+        "trees_url": "https://api.github.com/repos/MYORG/MYREPO/git/trees{/sha}",
+        "statuses_url": "https://api.github.com/repos/MYORG/MYREPO/statuses/{sha}",
+        "languages_url": "https://api.github.com/repos/MYORG/MYREPO/languages",
+        "stargazers_url": "https://api.github.com/repos/MYORG/MYREPO/stargazers",
+        "contributors_url": "https://api.github.com/repos/MYORG/MYREPO/contributors",
+        "subscribers_url": "https://api.github.com/repos/MYORG/MYREPO/subscribers",
+        "subscription_url": "https://api.github.com/repos/MYORG/MYREPO/subscription",
+        "commits_url": "https://api.github.com/repos/MYORG/MYREPO/commits{/sha}",
+        "git_commits_url": "https://api.github.com/repos/MYORG/MYREPO/git/commits{/sha}",
+        "comments_url": "https://api.github.com/repos/MYORG/MYREPO/comments{/number}",
+        "issue_comment_url": "https://api.github.com/repos/MYORG/MYREPO/issues/comments{/number}",
+        "contents_url": "https://api.github.com/repos/MYORG/MYREPO/contents/{+path}",
+        "compare_url": "https://api.github.com/repos/MYORG/MYREPO/compare/{base}...{head}",
+        "merges_url": "https://api.github.com/repos/MYORG/MYREPO/merges",
+        "archive_url": "https://api.github.com/repos/MYORG/MYREPO/{archive_format}{/ref}",
+        "downloads_url": "https://api.github.com/repos/MYORG/MYREPO/downloads",
+        "issues_url": "https://api.github.com/repos/MYORG/MYREPO/issues{/number}",
+        "pulls_url": "https://api.github.com/repos/MYORG/MYREPO/pulls{/number}",
+        "milestones_url": "https://api.github.com/repos/MYORG/MYREPO/milestones{/number}",
+        "notifications_url": "https://api.github.com/repos/MYORG/MYREPO/notifications{?since,all,participating}",
+        "labels_url": "https://api.github.com/repos/MYORG/MYREPO/labels{/name}",
+        "releases_url": "https://api.github.com/repos/MYORG/MYREPO/releases{/id}",
+        "deployments_url": "https://api.github.com/repos/MYORG/MYREPO/deployments",
+        "created_at": "2021-09-10T18:55:38Z",
+        "updated_at": "2021-09-10T18:55:41Z",
+        "pushed_at": "2021-09-28T23:25:26Z",
+        "git_url": "git://github.com/MYORG/MYREPO.git",
+        "ssh_url": "git@github.com:MYORG/MYREPO.git",
+        "clone_url": "https://github.com/MYORG/MYREPO.git",
+        "svn_url": "https://github.com/MYORG/MYREPO",
+        "homepage": null,
+        "size": 121,
+        "stargazers_count": 0,
+        "watchers_count": 0,
+        "language": null,
+        "has_issues": true,
+        "has_projects": true,
+        "has_downloads": true,
+        "has_wiki": true,
+        "has_pages": false,
+        "forks_count": 0,
+        "mirror_url": null,
+        "archived": false,
+        "disabled": false,
+        "open_issues_count": 1,
+        "license": null,
+        "allow_forking": false,
+        "forks": 0,
+        "open_issues": 1,
+        "watchers": 0,
+        "default_branch": "master"
+    },
+    "organization": {
+        "login": "MYORG",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "url": "https://api.github.com/orgs/MYORG",
+        "repos_url": "https://api.github.com/orgs/MYORG/repos",
+        "events_url": "https://api.github.com/orgs/MYORG/events",
+        "hooks_url": "https://api.github.com/orgs/MYORG/hooks",
+        "issues_url": "https://api.github.com/orgs/MYORG/issues",
+        "members_url": "https://api.github.com/orgs/MYORG/members{/member}",
+        "public_members_url": "https://api.github.com/orgs/MYORG/public_members{/member}",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "description": ""
+    },
+    "sender": {
+        "login": "MYNAME",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/MYNAME",
+        "html_url": "https://github.com/MYNAME",
+        "followers_url": "https://api.github.com/users/MYNAME/followers",
+        "following_url": "https://api.github.com/users/MYNAME/following{/other_user}",
+        "gists_url": "https://api.github.com/users/MYNAME/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/MYNAME/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/MYNAME/subscriptions",
+        "organizations_url": "https://api.github.com/users/MYNAME/orgs",
+        "repos_url": "https://api.github.com/users/MYNAME/repos",
+        "events_url": "https://api.github.com/users/MYNAME/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/MYNAME/received_events",
+        "type": "User",
+        "site_admin": false
+    }
+}

docs/releasenotes/0.22.md

@@ -0,0 +1,74 @@
+# actions-runner-controller v0.22.0
+
+This version of ARC focuses on scalability and reliablity of runners.
+
+## GitHub API Cache
+
+In terms of scalability, ARC now caches GitHub API responses according to their recommendation(=Cache-Control header[^1]).
+As long as GitHub keeps its current behavior, it will result in ARC to cache various List Runners API and List Workflow Jobs calls for 60 seconds.
+
+[^1]: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#conditional-requests
+
+The cache for List Runners API is expecially important, as their responses can be shared between every runner under the same scope (repository, organization, or enterprise).
+
+In previous versions of ARC, the number of List Runners API calls had scaled proportional to the number of runners managed by ARC.
+Thanks to the addition of cache, since v0.22.0, it may scale proportional to the number of runner scopes (=The number of repositories for your repository runners + The number of organizations for your organizational runners + The number of enterprises for your enterprise runners). You might be able to scale to hundreds of runners depending on your environemnt.
+
+Please share your experience if you successfully scaled to a level that wasn't possible with previous versions!
+
+## Improved Runner Scale Down Process
+
+In terms of reliability, the first thing to note is that it has a new scale down process for both RunnerDeployment and RunnerSet.
+
+Previously every runner pod can restart immediately after the completion, while at the same time ARC might mark the same runner pod for deletion due to scale down.
+That resulted in various race conditions that terminated the runner prematurely while running a workflow job[^2].
+
+[^2]: See [this issue](https://github.com/actions-runner-controller/actions-runner-controller/issues/911) for more context.
+
+And it's now fixed. The new scale down process ensures that the runner has been registered successfully and then de-registered from GitHub Actions, before starting the runner pod deletion process.
+Any runner pod can't be terminated while being restarting or running a job now, which makes it impossible to be in the middle of running a workflow job when a runner pod is being terminated. No more race conditions.
+
+## Optimized Ephemeral Runner Termination Makes Less "Remove Runner" API calls
+
+It is also worth mentioning that the new scale down process makes less GitHub Actions `RemoveRunner` API calls, which contributes to more scallability.
+
+Two enhancements had been made on that.
+
+First, every runner managed by ARC now [uses `--ephemeral` by default](https://github.com/actions-runner-controller/actions-runner-controller/pull/1211).
+
+Second, we [removed unnecessary `RemoveRunner` API calls](https://github.com/actions-runner-controller/actions-runner-controller/pull/1204) when it's an ephemeral runner that has already completed running.
+
+[GitHub designed ephemeral runners to be automatically unregistered from GitHub Actions after running their first workflow jobs](https://github.blog/changelog/2021-09-20-github-actions-ephemeral-self-hosted-runners-new-webhooks-for-auto-scaling). It is unnecessary to call `RemoveRunner` API when the ephemeral runner pod has already completed successfully. These two enhancements aligns with that fact and it results in ARC making less API calls.
+
+## Prevention of Unnecessary Runner Pod Recreations
+
+Another reliability enhancement is based on the addition of a new field, `EffectiveTime`, to our RunnerDeployment and RunnerSet specifications.
+
+The field comes in play only for ephemeral runners, and ARC uses it as an indicator of when to add more runner pods, to match the current number of runner pods to the desired number.
+
+How that improves the reliability?
+
+Previously, ARC had been continuously recreating runner pods as they complete, with no delay. That sometimes resulted in a runner pod to get recreated and then immediately terminated without being used at all. Not only this is a waste of cluster resource, it resulted in race conditions we explained in the previous section about "Improved Runner Scale Down Process". We fixed the race conditions as explained in the previous section, but the waste of cluster resource was still problematic.
+
+With `EffectiveTime`, ARC defers the addition(and recreations, as ARC doesn't distinguish addition vs recreation) of
+missing runner pods until the `EffectiveTime` is updated. `EffectiveTime` is updated only when the github-webhook-server of ARC updates the desired replicas number, ARC adds/recreates runner pods only after the webhook server updates it, the issue is resolved.
+
+This can be an unnecessary detail, but anyway- the "defer" mechanism times out after the `DefaultRunnerPodRecreationDelayAfterWebhookScale` duration, which is currently hard-coded to 10 minutes. So in case ARC missed receiving a webhook event for proper scaling, it converges to the desired replicas after 10 minutes anyway, so that the current state eventually syncs up with the desired state.
+
+Note that `EffectiveTime` fields are set by HRA controller for any RunnerDeployment and RunnerSet that manages ephemeral runners. That means, it is enabled regardless of the type of autoscaler you're using, webhook or API polling based ones. It isn't enabled for static(persistent) runners.
+
+There's currently no way to opt-out of `EffectiveTime` because the author of the feature(@mumoshu) thought it's unneeded. Please open a GitHub issue with details on your use-case if you do need to opt-out.
+
+## Generalized Runner Pod Management Logic
+
+This one might not be an user-visible change, but I'm explaining it for anyone who may wonder.
+
+Since this version, ARC uses the same logic for `RunnerDeployment` and `RunnerSet`. `RunnerDeployment` is Pod-based and `RunnerSet` is StatefulSet-based. That remains unchanged. But the most of the logic about how runner pods are managed is shared between the two.
+
+The only difference is that what adapters those variants pass to the generalized logic. `RunnerDeployment` uses `RunnerReplicaSet`(our another Kubernetes custom resource that powers `RunnerDeployment`) as an owner of a runner pod, and `RunnerSet` uses `StatefulSet`(it's vanilla Kubernetes StatefulSet) as an owner of a runner pod.
+
+This refactoring turned out to enable us to make `RunnerSet` as reliable as `RunnerDeployment`. `RunnerSet` has been considered an experimental feature
+even though it is more customizable than `RunnerDeployment` and has a support for Persistent Volume Claim(PVC)s.
+But since it now uses the same logic under the hood, `RunnerSet` can be considered more production-ready than before.
+
+If you staed away from using `RunnerSet` due to that, please try it and report anything you experienced!

github/fake/fake.go

@@ -162,6 +162,10 @@ func NewServer(opts ...Option) *httptest.Server {
 		},
 
 		// For RemoveRunner
+		"/repos/test/valid/actions/runners/0": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
 		"/repos/test/valid/actions/runners/1": &Handler{
 			Status: http.StatusNoContent,
 			Body:   "",

github/fake/runners.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 
-	"github.com/google/go-github/v37/github"
+	"github.com/google/go-github/v39/github"
 	"github.com/gorilla/mux"
 )
 

github/github.go

@@ -11,8 +11,11 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/metrics"
+	"github.com/actions-runner-controller/actions-runner-controller/logging"
 	"github.com/bradleyfalzon/ghinstallation"
-	"github.com/google/go-github/v37/github"
+	"github.com/go-logr/logr"
+	"github.com/google/go-github/v39/github"
+	"github.com/gregjones/httpcache"
 	"golang.org/x/oauth2"
 )
 
@@ -23,6 +26,13 @@ type Config struct {
 	AppInstallationID int64  `split_words:"true"`
 	AppPrivateKey     string `split_words:"true"`
 	Token             string
+	URL               string `split_words:"true"`
+	UploadURL         string `split_words:"true"`
+	BasicauthUsername string `split_words:"true"`
+	BasicauthPassword string `split_words:"true"`
+	RunnerGitHubURL   string `split_words:"true"`
+
+	Log *logr.Logger
 }
 
 // Client wraps GitHub client with some additional
@@ -34,10 +44,22 @@ type Client struct {
 	GithubBaseURL string
 }
 
+type BasicAuthTransport struct {
+	Username string
+	Password string
+}
+
+func (p BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.SetBasicAuth(p.Username, p.Password)
+	return http.DefaultTransport.RoundTrip(req)
+}
+
 // NewClient creates a Github Client
 func (c *Config) NewClient() (*Client, error) {
 	var transport http.RoundTripper
-	if len(c.Token) > 0 {
+	if len(c.BasicauthUsername) > 0 && len(c.BasicauthPassword) > 0 {
+		transport = BasicAuthTransport{Username: c.BasicauthUsername, Password: c.BasicauthPassword}
+	} else if len(c.Token) > 0 {
 		transport = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(&oauth2.Token{AccessToken: c.Token})).Transport
 	} else {
 		var tr *ghinstallation.Transport
@@ -63,8 +85,12 @@ func (c *Config) NewClient() (*Client, error) {
 		}
 		transport = tr
 	}
-	transport = metrics.Transport{Transport: transport}
-	httpClient := &http.Client{Transport: transport}
+
+	cached := httpcache.NewTransport(httpcache.NewMemoryCache())
+	cached.Transport = transport
+	loggingTransport := logging.Transport{Transport: cached, Log: c.Log}
+	metricsTransport := metrics.Transport{Transport: loggingTransport}
+	httpClient := &http.Client{Transport: metricsTransport}
 
 	var client *github.Client
 	var githubBaseURL string
@@ -78,8 +104,39 @@ func (c *Config) NewClient() (*Client, error) {
 	} else {
 		client = github.NewClient(httpClient)
 		githubBaseURL = "https://github.com/"
+
+		if len(c.URL) > 0 {
+			baseUrl, err := url.Parse(c.URL)
+			if err != nil {
+				return nil, fmt.Errorf("github client creation failed: %v", err)
+			}
+			if !strings.HasSuffix(baseUrl.Path, "/") {
+				baseUrl.Path += "/"
+			}
+			client.BaseURL = baseUrl
+		}
+
+		if len(c.UploadURL) > 0 {
+			uploadUrl, err := url.Parse(c.UploadURL)
+			if err != nil {
+				return nil, fmt.Errorf("github client creation failed: %v", err)
+			}
+			if !strings.HasSuffix(uploadUrl.Path, "/") {
+				uploadUrl.Path += "/"
+			}
+			client.UploadURL = uploadUrl
+		}
+
+		if len(c.RunnerGitHubURL) > 0 {
+			githubBaseURL = c.RunnerGitHubURL
+			if !strings.HasSuffix(githubBaseURL, "/") {
+				githubBaseURL += "/"
+			}
+		}
 	}
 
+	client.UserAgent = "actions-runner-controller"
+
 	return &Client{
 		Client:        client,
 		regTokens:     map[string]*github.RegistrationToken{},
@@ -103,7 +160,7 @@ func (c *Client) GetRegistrationToken(ctx context.Context, enterprise, org, repo
 		return rt, nil
 	}
 
-	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
+	enterprise, owner, repo, err := getEnterpriseOrganizationAndRepo(enterprise, org, repo)
 
 	if err != nil {
 		return rt, err
@@ -129,7 +186,7 @@ func (c *Client) GetRegistrationToken(ctx context.Context, enterprise, org, repo
 
 // RemoveRunner removes a runner with specified runner ID from repository.
 func (c *Client) RemoveRunner(ctx context.Context, enterprise, org, repo string, runnerID int64) error {
-	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
+	enterprise, owner, repo, err := getEnterpriseOrganizationAndRepo(enterprise, org, repo)
 
 	if err != nil {
 		return err
@@ -150,7 +207,7 @@ func (c *Client) RemoveRunner(ctx context.Context, enterprise, org, repo string,
 
 // ListRunners returns a list of runners of specified owner/repository name.
 func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string) ([]*github.Runner, error) {
-	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
+	enterprise, owner, repo, err := getEnterpriseOrganizationAndRepo(enterprise, org, repo)
 
 	if err != nil {
 		return nil, err
@@ -176,6 +233,49 @@ func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string)
 	return runners, nil
 }
 
+// ListOrganizationRunnerGroups returns all the runner groups defined in the organization and
+// inherited to the organization from an enterprise.
+func (c *Client) ListOrganizationRunnerGroups(ctx context.Context, org string) ([]*github.RunnerGroup, error) {
+	var runnerGroups []*github.RunnerGroup
+
+	opts := github.ListOptions{PerPage: 100}
+	for {
+		list, res, err := c.Client.Actions.ListOrganizationRunnerGroups(ctx, org, &opts)
+		if err != nil {
+			return runnerGroups, fmt.Errorf("failed to list organization runner groups: %w", err)
+		}
+
+		runnerGroups = append(runnerGroups, list.RunnerGroups...)
+		if res.NextPage == 0 {
+			break
+		}
+		opts.Page = res.NextPage
+	}
+
+	return runnerGroups, nil
+}
+
+func (c *Client) ListRunnerGroupRepositoryAccesses(ctx context.Context, org string, runnerGroupId int64) ([]*github.Repository, error) {
+	var repos []*github.Repository
+
+	opts := github.ListOptions{PerPage: 100}
+	for {
+		list, res, err := c.Client.Actions.ListRepositoryAccessRunnerGroup(ctx, org, runnerGroupId, &opts)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list repository access for runner group: %w", err)
+		}
+
+		repos = append(repos, list.Repositories...)
+		if res.NextPage == 0 {
+			break
+		}
+
+		opts.Page = res.NextPage
+	}
+
+	return repos, nil
+}
+
 // cleanup removes expired registration tokens.
 func (c *Client) cleanup() {
 	c.mu.Lock()
@@ -267,8 +367,8 @@ func (c *Client) listRepositoryWorkflowRuns(ctx context.Context, user string, re
 	return workflowRuns, nil
 }
 
-// Validates enterprise, organisation and repo arguments. Both are optional, but at least one should be specified
-func getEnterpriseOrganisationAndRepo(enterprise, org, repo string) (string, string, string, error) {
+// Validates enterprise, organization and repo arguments. Both are optional, but at least one should be specified
+func getEnterpriseOrganizationAndRepo(enterprise, org, repo string) (string, string, string, error) {
 	if len(repo) > 0 {
 		owner, repository, err := splitOwnerAndRepo(repo)
 		return "", owner, repository, err
@@ -337,7 +437,7 @@ func (r *Client) IsRunnerBusy(ctx context.Context, enterprise, org, repo, name s
 	for _, runner := range runners {
 		if runner.GetName() == name {
 			if runner.GetStatus() == "offline" {
-				return false, &RunnerOffline{runnerName: name}
+				return runner.GetBusy(), &RunnerOffline{runnerName: name}
 			}
 			return runner.GetBusy(), nil
 		}

github/github_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github/fake"
-	"github.com/google/go-github/v37/github"
+	"github.com/google/go-github/v39/github"
 )
 
 var server *httptest.Server
@@ -152,3 +152,10 @@ func TestCleanup(t *testing.T) {
 		t.Errorf("expired token still exists")
 	}
 }
+
+func TestUserAgent(t *testing.T) {
+	client := newTestClient()
+	if client.UserAgent != "actions-runner-controller" {
+		t.Errorf("UserAgent should be set to actions-runner-controller")
+	}
+}

go.mod

@@ -1,27 +1,80 @@
 module github.com/actions-runner-controller/actions-runner-controller
 
-go 1.15
+go 1.17
 
 require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
-	github.com/go-logr/logr v0.4.0
-	github.com/google/go-cmp v0.5.6
-	github.com/google/go-github/v37 v37.0.0
+	github.com/go-logr/logr v1.2.0
+	github.com/google/go-cmp v0.5.7
+	github.com/google/go-github/v39 v39.2.0
 	github.com/gorilla/mux v1.8.0
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/onsi/ginkgo v1.16.4
-	github.com/onsi/gomega v1.13.0
-	github.com/prometheus/client_golang v1.11.0
-	github.com/teambition/rrule-go v1.6.2
-	go.uber.org/zap v1.19.0
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
+	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/gomega v1.17.0
+	github.com/prometheus/client_golang v1.12.1
+	github.com/stretchr/testify v1.7.0
+	github.com/teambition/rrule-go v1.7.2
+	go.uber.org/zap v1.21.0
+	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a
 	gomodules.xyz/jsonpatch/v2 v2.2.0
-	k8s.io/api v0.21.1
-	k8s.io/apimachinery v0.21.1
-	k8s.io/client-go v0.21.1
-	sigs.k8s.io/controller-runtime v0.9.0
-	sigs.k8s.io/yaml v1.2.0
+	k8s.io/api v0.23.4
+	k8s.io/apimachinery v0.23.4
+	k8s.io/client-go v0.23.4
+	sigs.k8s.io/controller-runtime v0.11.1
+	sigs.k8s.io/yaml v1.3.0
 )
 
-replace github.com/google/go-github/v37 => github.com/mumoshu/go-github/v37 v37.0.100
+require (
+	cloud.google.com/go v0.81.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-github/v29 v29.0.2 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/apiextensions-apiserver v0.23.0 // indirect
+	k8s.io/component-base v0.23.0 // indirect
+	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+)
+
+replace github.com/gregjones/httpcache => github.com/actions-runner-controller/httpcache v0.2.0

go.sum

@@ -8,13 +8,17 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0=
 cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
 cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -34,13 +38,14 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -49,15 +54,21 @@ github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMo
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/actions-runner-controller/httpcache v0.2.0 h1:hCNvYuVPJ2xxYBymqBvH0hSiQpqz4PHF/LbU3XghGNI=
+github.com/actions-runner-controller/httpcache v0.2.0/go.mod h1:JLu9/2M/btPz1Zu/vTZ71XzukQHn2YeISPmJoM5exBI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -65,31 +76,37 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bradleyfalzon/ghinstallation v1.1.1 h1:pmBXkxgM1WeF8QYvDLT5kuQiHMcmf+X015GI0KM/E3I=
 github.com/bradleyfalzon/ghinstallation v1.1.1/go.mod h1:vyCmHTciHx/uuyN82Zc3rXN3X2KTK8nUTCrTMwAhcug=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
+github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -99,7 +116,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -107,16 +123,24 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=
-github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -129,31 +153,32 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=
-github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk=
+github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -161,6 +186,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -176,10 +202,14 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
+github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -188,21 +218,25 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
-github.com/google/go-github/v37 v37.0.1-0.20210713230028-465df60a8ec3 h1:YVfdOQRQ95EjQz0qpGdw9LIzJUflL4FV0EEX3fZ7fH8=
-github.com/google/go-github/v37 v37.0.1-0.20210713230028-465df60a8ec3/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ=
+github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -210,27 +244,27 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
 github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -246,8 +280,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
@@ -255,18 +287,21 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -275,26 +310,26 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -307,16 +342,16 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
+github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mumoshu/go-github/v37 v37.0.100 h1:a0S2oEJ8naEW5M4y6S+wu3ufSe9PmKxu77C72VJ6LLw=
-github.com/mumoshu/go-github/v37 v37.0.100/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -328,27 +363,28 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
-github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -357,8 +393,9 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -368,17 +405,20 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -388,26 +428,29 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
+github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -416,50 +459,73 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/teambition/rrule-go v1.6.2 h1:keZiiijltBxYUuhQaySAEGyIFR0UOkAd7i+u6FM5/+I=
-github.com/teambition/rrule-go v1.6.2/go.mod h1:mBJ1Ht5uboJ6jexKdNUJg2NcwP8uUMNvStWXlJD3MvU=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/teambition/rrule-go v1.7.2 h1:goEajFWYydfCgavn2m/3w5U+1b3PGqPUHx/fFSVfTy0=
+github.com/teambition/rrule-go v1.7.2/go.mod h1:mBJ1Ht5uboJ6jexKdNUJg2NcwP8uUMNvStWXlJD3MvU=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
+go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
+go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
+go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
+go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
+go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
+go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
+go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
+go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
+go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
+go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
+go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
+go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
+go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
+go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -481,8 +547,9 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -491,8 +558,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 h1:xUIPaMhvROX9dhPvRCenIJtU78+lbEenGbgqB5hfHCQ=
-golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -511,7 +579,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -529,17 +596,37 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a h1:qfl7ob3DIEs3Ml9oLuPwY2N04gymzAW04WsUQHIClgM=
+golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -550,6 +637,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -564,10 +652,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -587,40 +673,63 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -631,7 +740,6 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -667,10 +775,17 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -694,6 +809,12 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -723,16 +844,30 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -745,6 +880,16 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -756,20 +901,21 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -778,6 +924,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -797,38 +944,45 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c=
-k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s=
-k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo=
-k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA=
-k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs=
-k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
-k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY=
-k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4=
-k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs=
-k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q=
-k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw=
-k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
-k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg=
+k8s.io/api v0.23.4 h1:85gnfXQOWbJa1SiWGpE9EEtHs0UVvDyIsSMpEtl2D4E=
+k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI=
+k8s.io/apiextensions-apiserver v0.23.0 h1:uii8BYmHYiT2ZTAJxmvc3X8UhNYMxl2A0z0Xq3Pm+WY=
+k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4=
+k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc=
+k8s.io/apimachinery v0.23.4 h1:fhnuMd/xUL3Cjfl64j5ULKZ1/J9n8NuQEgNL+WXWfdM=
+k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
+k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4=
+k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
+k8s.io/client-go v0.23.4 h1:YVWvPeerA2gpUudLelvsolzH7c2sFoXXR5wM/sWqNFU=
+k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0=
+k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
+k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8=
+k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI=
+k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s=
-k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw=
+k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
+k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
+k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
+k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.9.0 h1:ZIZ/dtpboPSbZYY7uUz2OzrkaBTOThx2yekLtpGB+zY=
-sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
+sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
+sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
+sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
+sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

logging/logger.go

@@ -0,0 +1,56 @@
+package logging
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/go-logr/logr"
+	zaplib "go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+const (
+	LogLevelDebug = "debug"
+	LogLevelInfo  = "info"
+	LogLevelWarn  = "warn"
+	LogLevelError = "error"
+)
+
+func NewLogger(logLevel string) logr.Logger {
+	log := zap.New(func(o *zap.Options) {
+		switch logLevel {
+		case LogLevelDebug:
+			o.Development = true
+			lvl := zaplib.NewAtomicLevelAt(zaplib.DebugLevel) // maps to logr's V(1)
+			o.Level = &lvl
+		case LogLevelInfo:
+			lvl := zaplib.NewAtomicLevelAt(zaplib.InfoLevel)
+			o.Level = &lvl
+		case LogLevelWarn:
+			lvl := zaplib.NewAtomicLevelAt(zaplib.WarnLevel)
+			o.Level = &lvl
+		case LogLevelError:
+			lvl := zaplib.NewAtomicLevelAt(zaplib.ErrorLevel)
+			o.Level = &lvl
+		default:
+			// We use bitsize of 8 as zapcore.Level is a type alias to int8
+			levelInt, err := strconv.ParseInt(logLevel, 10, 8)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to parse --log-level=%s: %v", logLevel, err)
+				os.Exit(1)
+			}
+
+			// For example, --log-level=debug a.k.a --log-level=-1 maps to zaplib.DebugLevel, which is associated to logr's V(1)
+			// --log-level=-2 maps the specific custom log level that is associated to logr's V(2).
+			level := zapcore.Level(levelInt)
+			atomicLevel := zaplib.NewAtomicLevelAt(level)
+			o.Level = &atomicLevel
+		}
+		o.TimeEncoder = zapcore.TimeEncoderOfLayout(time.RFC3339)
+	})
+
+	return log
+}

logging/transport.go

@@ -0,0 +1,66 @@
+// Package logging provides various logging helpers for ARC
+package logging
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+
+	"github.com/go-logr/logr"
+	"github.com/gregjones/httpcache"
+)
+
+const (
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+	headerRateLimitRemaining = "X-RateLimit-Remaining"
+)
+
+// Transport wraps a transport with metrics monitoring
+type Transport struct {
+	Transport http.RoundTripper
+
+	Log *logr.Logger
+}
+
+func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := t.Transport.RoundTrip(req)
+	if resp != nil {
+		t.log(req, resp)
+	}
+	return resp, err
+}
+
+func (t Transport) log(req *http.Request, resp *http.Response) {
+	if t.Log == nil {
+		return
+	}
+
+	var args []interface{}
+
+	marked := resp.Header.Get(httpcache.XFromCache) == "1"
+
+	args = append(args, "from_cache", marked, "method", req.Method, "url", req.URL.String())
+
+	if !marked {
+		// Do not log outdated rate limit remaining value
+
+		remaining := resp.Header.Get(headerRateLimitRemaining)
+
+		args = append(args, "ratelimit_remaining", remaining)
+	}
+
+	if t.Log.V(4).Enabled() {
+		var buf bytes.Buffer
+
+		if _, err := io.Copy(&buf, resp.Body); err != nil {
+			t.Log.V(3).Info("unable to copy http response", "error", err)
+		}
+		resp.Body.Close()
+
+		t.Log.V(4).Info("Logging HTTP round-trip", "method", req.Method, "requestHeader", req.Header, "statusCode", resp.StatusCode, "responseHeader", resp.Header, "responseBody", buf.String())
+
+		resp.Body = io.NopCloser(&buf)
+	}
+
+	t.Log.V(3).Info("Seen HTTP response", args...)
+}

main.go

@@ -26,24 +26,18 @@ import (
 	actionsv1alpha1 "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1"
 	"github.com/actions-runner-controller/actions-runner-controller/controllers"
 	"github.com/actions-runner-controller/actions-runner-controller/github"
+	"github.com/actions-runner-controller/actions-runner-controller/logging"
 	"github.com/kelseyhightower/envconfig"
-	zaplib "go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	// +kubebuilder:scaffold:imports
 )
 
 const (
 	defaultRunnerImage = "summerwind/actions-runner:latest"
 	defaultDockerImage = "docker:dind"
-
-	logLevelDebug = "debug"
-	logLevelInfo  = "info"
-	logLevelWarn  = "warn"
-	logLevelError = "error"
 )
 
 var (
@@ -58,6 +52,17 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 }
 
+type stringSlice []string
+
+func (i *stringSlice) String() string {
+	return fmt.Sprintf("%v", *i)
+}
+
+func (i *stringSlice) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}
+
 func main() {
 	var (
 		err      error
@@ -70,7 +75,9 @@ func main() {
 
 		gitHubAPICacheDuration time.Duration
 
-		runnerImage          string
+		runnerImage            string
+		runnerImagePullSecrets stringSlice
+
 		dockerImage          string
 		dockerRegistryMirror string
 		namespace            string
@@ -92,33 +99,27 @@ func main() {
 	flag.StringVar(&leaderElectionId, "leader-election-id", "actions-runner-controller", "Controller id for leader election.")
 	flag.StringVar(&runnerImage, "runner-image", defaultRunnerImage, "The image name of self-hosted runner container.")
 	flag.StringVar(&dockerImage, "docker-image", defaultDockerImage, "The image name of docker sidecar container.")
+	flag.Var(&runnerImagePullSecrets, "runner-image-pull-secret", "The default image-pull secret name for self-hosted runner container.")
 	flag.StringVar(&dockerRegistryMirror, "docker-registry-mirror", "", "The default Docker Registry Mirror used by runners.")
 	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
 	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
 	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
 	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
-	flag.DurationVar(&gitHubAPICacheDuration, "github-api-cache-duration", 0, "The duration until the GitHub API cache expires. Setting this to e.g. 10m results in the controller tries its best not to make the same API call within 10m to reduce the chance of being rate-limited. Defaults to mostly the same value as sync-period. If you're tweaking this in order to make autoscaling more responsive, you'll probably want to tweak sync-period, too")
-	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change. . If you're tweaking this in order to make autoscaling more responsive, you'll probably want to tweak github-api-cache-duration, too")
+	flag.StringVar(&c.URL, "github-url", c.URL, "GitHub URL to be used for GitHub API calls")
+	flag.StringVar(&c.UploadURL, "github-upload-url", c.UploadURL, "GitHub Upload URL to be used for GitHub API calls")
+	flag.StringVar(&c.BasicauthUsername, "github-basicauth-username", c.BasicauthUsername, "Username for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
+	flag.StringVar(&c.BasicauthPassword, "github-basicauth-password", c.BasicauthPassword, "Password for GitHub basic auth to use instead of PAT or GitHub APP in case it's running behind a proxy API")
+	flag.StringVar(&c.RunnerGitHubURL, "runner-github-url", c.RunnerGitHubURL, "GitHub URL to be used by runners during registration")
+	flag.DurationVar(&gitHubAPICacheDuration, "github-api-cache-duration", 0, "DEPRECATED: The duration until the GitHub API cache expires. Setting this to e.g. 10m results in the controller tries its best not to make the same API call within 10m to reduce the chance of being rate-limited. Defaults to mostly the same value as sync-period. If you're tweaking this in order to make autoscaling more responsive, you'll probably want to tweak sync-period, too")
+	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change.")
 	flag.Var(&commonRunnerLabels, "common-runner-labels", "Runner labels in the K1=V1,K2=V2,... format that are inherited all the runners created by the controller. See https://github.com/actions-runner-controller/actions-runner-controller/issues/321 for more information")
 	flag.StringVar(&namespace, "watch-namespace", "", "The namespace to watch for custom resources. Set to empty for letting it watch for all namespaces.")
-	flag.StringVar(&logLevel, "log-level", logLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
+	flag.StringVar(&logLevel, "log-level", logging.LogLevelDebug, `The verbosity of the logging. Valid values are "debug", "info", "warn", "error". Defaults to "debug".`)
 	flag.Parse()
 
-	logger := zap.New(func(o *zap.Options) {
-		switch logLevel {
-		case logLevelDebug:
-			o.Development = true
-		case logLevelInfo:
-			lvl := zaplib.NewAtomicLevelAt(zaplib.InfoLevel)
-			o.Level = &lvl
-		case logLevelWarn:
-			lvl := zaplib.NewAtomicLevelAt(zaplib.WarnLevel)
-			o.Level = &lvl
-		case logLevelError:
-			lvl := zaplib.NewAtomicLevelAt(zaplib.ErrorLevel)
-			o.Level = &lvl
-		}
-	})
+	logger := logging.NewLogger(logLevel)
+
+	c.Log = &logger
 
 	ghClient, err = c.NewClient()
 	if err != nil {
@@ -147,9 +148,11 @@ func main() {
 		Log:                  log.WithName("runner"),
 		Scheme:               mgr.GetScheme(),
 		GitHubClient:         ghClient,
-		RunnerImage:          runnerImage,
 		DockerImage:          dockerImage,
 		DockerRegistryMirror: dockerRegistryMirror,
+		// Defaults for self-hosted runner containers
+		RunnerImage:            runnerImage,
+		RunnerImagePullSecrets: runnerImagePullSecrets,
 	}
 
 	if err = runnerReconciler.SetupWithManager(mgr); err != nil {
@@ -186,10 +189,12 @@ func main() {
 		Log:                  log.WithName("runnerset"),
 		Scheme:               mgr.GetScheme(),
 		CommonRunnerLabels:   commonRunnerLabels,
-		RunnerImage:          runnerImage,
 		DockerImage:          dockerImage,
 		DockerRegistryMirror: dockerRegistryMirror,
 		GitHubBaseURL:        ghClient.GithubBaseURL,
+		// Defaults for self-hosted runner containers
+		RunnerImage:            runnerImage,
+		RunnerImagePullSecrets: runnerImagePullSecrets,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {

pkg/githubwebhookdeliveryforwarder/githubwebhookdelivery.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v37/github"
+	gogithub "github.com/google/go-github/v39/github"
 )
 
 type server struct {

pkg/hookdeliveryforwarder/forwarder.go

@@ -12,7 +12,7 @@ import (
 	"time"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v37/github"
+	gogithub "github.com/google/go-github/v39/github"
 )
 
 type Forwarder struct {

pkg/hookdeliveryforwarder/hooks.go

@@ -3,7 +3,7 @@ package hookdeliveryforwarder
 import (
 	"context"
 
-	gogithub "github.com/google/go-github/v37/github"
+	gogithub "github.com/google/go-github/v39/github"
 )
 
 type hooksAPI struct {

pkg/hookdeliveryforwarder/hooks_deliveries.go

@@ -3,7 +3,7 @@ package hookdeliveryforwarder
 import (
 	"context"
 
-	gogithub "github.com/google/go-github/v37/github"
+	gogithub "github.com/google/go-github/v39/github"
 )
 
 type hookDeliveriesAPI struct {

pkg/hookdeliveryforwarder/multiforwarder.go

@@ -8,7 +8,7 @@ import (
 	"sync"
 
 	"github.com/actions-runner-controller/actions-runner-controller/github"
-	gogithub "github.com/google/go-github/v37/github"
+	gogithub "github.com/google/go-github/v39/github"
 )
 
 type MultiForwarder struct {

runner/Dockerfile

@@ -1,9 +1,9 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.280.3
+ARG RUNNER_VERSION=2.287.1
 ARG DOCKER_CHANNEL=stable
-ARG DOCKER_VERSION=20.10.8
+ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
 
 RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)
@@ -111,12 +111,14 @@ RUN mkdir /opt/hostedtoolcache \
     && chmod g+rwx /opt/hostedtoolcache
 
 COPY entrypoint.sh /
-COPY --chown=runner:docker patched $RUNNER_ASSETS_DIR/patched
 
 # Add the Python "User Script Directory" to the PATH
 ENV PATH="${PATH}:${HOME}/.local/bin"
 ENV ImageOS=ubuntu20
 
+RUN echo "PATH=${PATH}" > /etc/environment \
+    && echo "ImageOS=${ImageOS}" >> /etc/environment
+
 USER runner
 
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]

runner/Dockerfile.dindrunner

@@ -1,9 +1,9 @@
 FROM ubuntu:20.04
 
 ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.280.3
+ARG RUNNER_VERSION=2.287.1
 ARG DOCKER_CHANNEL=stable
-ARG DOCKER_VERSION=19.03.13
+ARG DOCKER_VERSION=20.10.12
 ARG DUMB_INIT_VERSION=1.2.5
 
 RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)
@@ -114,12 +114,13 @@ RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
 
 VOLUME /var/lib/docker
 
-COPY --chown=runner:docker patched $RUNNER_ASSETS_DIR/patched
-
 # Add the Python "User Script Directory" to the PATH
 ENV PATH="${PATH}:${HOME}/.local/bin"
 ENV ImageOS=ubuntu20
 
+RUN echo "PATH=${PATH}" > /etc/environment \
+    && echo "ImageOS=${ImageOS}" >> /etc/environment
+
 # No group definition, as that makes it harder to run docker.
 USER runner
 

runner/Dockerfile.ubuntu.1804

@@ -1,104 +0,0 @@
-FROM ubuntu:18.04
-
-ARG TARGETPLATFORM
-ARG RUNNER_VERSION=2.280.3
-ARG DOCKER_CHANNEL=stable
-ARG DOCKER_VERSION=20.10.8
-ARG DUMB_INIT_VERSION=1.2.5
-
-RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update -y \
-    && apt install -y software-properties-common \
-    && add-apt-repository -y ppa:git-core/ppa \
-    && apt update -y \
-    && apt install -y --no-install-recommends \
-    build-essential \
-    curl \
-    ca-certificates \
-    dnsutils \
-    ftp \
-    git \
-    iproute2 \
-    iputils-ping \
-    jq \
-    libunwind8 \
-    locales \
-    netcat \
-    openssh-client \
-    parallel \
-    python3-pip \
-    rsync \
-    shellcheck \
-    sudo \
-    telnet \
-    time \
-    tzdata \
-    unzip \
-    upx \
-    wget \
-    zip \
-    zstd \
-    && ln -sf /usr/bin/python3 /usr/bin/python \
-    && ln -sf /usr/bin/pip3 /usr/bin/pip \
-    && rm -rf /var/lib/apt/lists/*
-
-# arch command on OS X reports "i386" for Intel CPUs regardless of bitness
-RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
-    && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
-    && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x86_64 ; fi \
-    && curl -f -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_${ARCH} \
-    && chmod +x /usr/local/bin/dumb-init
-
-# Docker download supports arm64 as aarch64 & amd64 / i386 as x86_64
-RUN set -vx; \
-    export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
-    && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
-    && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x86_64 ; fi \
-    && curl -f -L -o docker.tgz https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz \
-    && tar zxvf docker.tgz \
-    && install -o root -g root -m 755 docker/docker /usr/local/bin/docker \
-    && rm -rf docker docker.tgz \
-    && adduser --disabled-password --gecos "" --uid 1000 runner \
-    && groupadd docker \
-    && usermod -aG sudo runner \
-    && usermod -aG docker runner \
-    && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
-
-ENV RUNNER_ASSETS_DIR=/runnertmp
-ENV HOME=/home/runner
-
-# Runner download supports amd64 as x64. Externalstmp is needed for making mount points work inside DinD.
-#
-# libyaml-dev is required for ruby/setup-ruby action.
-# It is installed after installdependencies.sh and before removing /var/lib/apt/lists
-# to avoid rerunning apt-update on its own.
-RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
-    && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "x86_64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x64 ; fi \
-    && mkdir -p "$RUNNER_ASSETS_DIR" \
-    && cd "$RUNNER_ASSETS_DIR" \
-    && curl -f -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
-    && tar xzf ./runner.tar.gz \
-    && rm runner.tar.gz \
-    && ./bin/installdependencies.sh \
-    && mv ./externals ./externalstmp \
-    && apt-get install -y libyaml-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-ENV RUNNER_TOOL_CACHE=/opt/hostedtoolcache
-RUN mkdir /opt/hostedtoolcache \
-    && chgrp docker /opt/hostedtoolcache \
-    && chmod g+rwx /opt/hostedtoolcache
-
-COPY entrypoint.sh /
-COPY --chown=runner:docker patched $RUNNER_ASSETS_DIR/patched
-
-# Add the Python "User Script Directory" to the PATH
-ENV PATH="${PATH}:${HOME}/.local/bin"
-ENV ImageOS=ubuntu20
-
-USER runner
-
-ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
-CMD ["/entrypoint.sh"]

runner/Makefile

@@ -1,10 +1,11 @@
-NAME ?= summerwind/actions-runner
-DIND_RUNNER_NAME ?= ${NAME}-dind
+DOCKER_USER ?= summerwind
+NAME ?= ${DOCKER_USER}/actions-runner
+DIND_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind
 TAG ?= latest
-TARGET_PLATFORM ?= $(shell arch)
+TARGETPLATFORM ?= $(shell arch)
 
-RUNNER_VERSION ?= 2.280.3
-DOCKER_VERSION ?= 20.10.8
+RUNNER_VERSION ?= 2.287.1
+DOCKER_VERSION ?= 20.10.12
 
 # default list of platforms for which multiarch image is built
 ifeq (${PLATFORMS}, )
@@ -24,8 +25,8 @@ else
 endif
 
 docker-build-ubuntu:
-	docker build --build-arg TARGETPLATFORM=${TARGET_PLATFORM} --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} .
-	docker build --build-arg TARGETPLATFORM=${TARGET_PLATFORM} --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${DIND_RUNNER_NAME}:${TAG} -f Dockerfile.dindrunner .
+	docker build --build-arg TARGETPLATFORM=${TARGETPLATFORM} --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} .
+	docker build --build-arg TARGETPLATFORM=${TARGETPLATFORM} --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${DIND_RUNNER_NAME}:${TAG} -f Dockerfile.dindrunner .
 
 docker-push-ubuntu:
 	docker push ${NAME}:${TAG}
@@ -39,12 +40,12 @@ docker-buildx-ubuntu:
 	docker buildx build --platform ${PLATFORMS} \
 		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
-		-t "${NAME}:latest" \
+		-t "${NAME}:${TAG}" \
 		-f Dockerfile \
 		. ${PUSH_ARG}
 	docker buildx build --platform ${PLATFORMS} \
 		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
 		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
-		-t "${DIND_RUNNER_NAME}:latest" \
+		-t "${DIND_RUNNER_NAME}:${TAG}" \
 		-f Dockerfile.dindrunner \
 		. ${PUSH_ARG}

runner/entrypoint.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+RUNNER_ASSETS_DIR=${RUNNER_ASSETS_DIR:-/runnertmp}
 RUNNER_HOME=${RUNNER_HOME:-/runner}
 
 LIGHTGREEN="\e[0;32m"
@@ -24,6 +25,13 @@ if [ ! -z "${STARTUP_DELAY_IN_SECONDS}" ]; then
   sleep ${STARTUP_DELAY_IN_SECONDS}
 fi
 
+if [[ "${DISABLE_WAIT_FOR_DOCKER}" != "true" ]] && [[ "${DOCKER_ENABLED}" == "true" ]]; then
+    log "Docker enabled runner detected and Docker daemon wait is enabled"
+    log "Waiting until Docker is avaliable or the timeout is reached"
+    timeout 120s bash -c 'until docker ps ;do sleep 1; done'
+else
+  log "Docker wait check skipped. Either Docker is disabled or the wait is disabled, continuing with entrypoint"
+fi
 
 if [ -z "${GITHUB_URL}" ]; then
   log "Working with public GitHub"
@@ -70,19 +78,28 @@ if [ ! -d "${RUNNER_HOME}" ]; then
 fi
 
 # if this is not a testing environment
-if [ -z "${UNITTEST:-}" ]; then
-  sudo chown -R runner:docker ${RUNNER_HOME}
-  mv /runnertmp/* ${RUNNER_HOME}/
+if [[ "${UNITTEST:-}" == '' ]]; then
+  sudo chown -R runner:docker "$RUNNER_HOME"
+  # enable dotglob so we can copy a ".env" file to load in env vars as part of the service startup if one is provided
+  # loading a .env from the root of the service is part of the actions/runner logic
+  shopt -s dotglob
+  # use cp instead of mv to avoid issues when src and dst are on different devices
+  cp -r "$RUNNER_ASSETS_DIR"/* "$RUNNER_HOME"/
+  shopt -u dotglob
 fi
 
 cd ${RUNNER_HOME}
 # past that point, it's all relative pathes from /runner
 
 config_args=()
-if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" == "true" -a "${RUNNER_EPHEMERAL}" != "false" ]; then
+if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" == "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
   config_args+=(--ephemeral)
   echo "Passing --ephemeral to config.sh to enable the ephemeral runner."
 fi
+if [ "${DISABLE_RUNNER_UPDATE:-}" == "true" ]; then
+  config_args+=(--disableupdate)
+  echo "Passing --disableupdate to config.sh to disable automatic runner updates."
+fi
 
 retries_left=10
 while [[ ${retries_left} -gt 0 ]]; do
@@ -133,28 +150,32 @@ cat .runner
 #     -H "Authorization: bearer ${GITHUB_TOKEN}"
 #     https://api.github.com/repos/USER/REPO/actions/runners/171
 
-if [ -n "${RUNNER_REGISTRATION_ONLY}" ]; then
-  success "This runner is configured to be registration-only. Exiting without starting the runner service..."
-  exit 0
-fi
-
 if [ -z "${UNITTEST:-}" ]; then
   mkdir ./externals
   # Hack due to the DinD volumes
   mv ./externalstmp/* ./externals/
-
-  for f in runsvc.sh RunnerService.js; do
-    diff {bin,patched}/${f} || :
-    sudo mv bin/${f}{,.bak}
-    sudo mv {patched,bin}/${f}
-  done
 fi
 
 args=()
-if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" != "true" -a "${RUNNER_EPHEMERAL}" != "false" ]; then
+if [ "${RUNNER_FEATURE_FLAG_EPHEMERAL:-}" != "true" -a "${RUNNER_EPHEMERAL}" == "true" ]; then
   args+=(--once)
-  echo "Passing --once to runsvc.sh to enable the legacy ephemeral runner."
+  echo "[WARNING] Passing --once is deprecated and will be removed as an option from the image and ARC at the release of 0.24.0."
+  echo "[WARNING] Upgrade to GHES => 3.3 to continue using actions-runner-controller. If you are using github.com ignore this warning."
 fi
 
-unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN
-exec ./bin/runsvc.sh "${args[@]}"
+# Unset entrypoint environment variables so they don't leak into the runner environment
+unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN STARTUP_DELAY_IN_SECONDS DISABLE_WAIT_FOR_DOCKER
+
+# Docker ignores PAM and thus never loads the system environment variables that
+# are meant to be set in every environment of every user. We emulate the PAM
+# behavior by reading the environment variables without interpreting them.
+#
+# https://github.com/actions-runner-controller/actions-runner-controller/issues/1135
+# https://github.com/actions/runner/issues/1703
+
+# /etc/environment may not exist when running unit tests depending on the platform being used
+# (e.g. Mac OS) so we just skip the mapping entirely
+if [ -z "${UNITTEST:-}" ]; then
+  mapfile -t env </etc/environment
+fi
+exec env -- "${env[@]}" ./run.sh "${args[@]}"

runner/patched/RunnerService.js

@@ -1,91 +0,0 @@
-#!/usr/bin/env node
-// Copyright (c) GitHub. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-var childProcess = require("child_process");
-var path = require("path")
-
-var supported = ['linux', 'darwin']
-
-if (supported.indexOf(process.platform) == -1) {
-    console.log('Unsupported platform: ' + process.platform);
-    console.log('Supported platforms are: ' + supported.toString());
-    process.exit(1);
-}
-
-var stopping = false;
-var listener = null;
-
-var runService = function() {
-    var listenerExePath = path.join(__dirname, '../bin/Runner.Listener');
-    var interactive = process.argv[2] === "interactive";
-
-    if(!stopping) {
-        try {
-            if (interactive) {
-                console.log('Starting Runner listener interactively');
-                listener = childProcess.spawn(listenerExePath, ['run'].concat(process.argv.slice(3)), { env: process.env });
-            } else {
-                console.log('Starting Runner listener with startup type: service');
-                listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'].concat(process.argv.slice(2)), { env: process.env });
-            }
-
-            console.log('Started listener process');
-
-            listener.stdout.on('data', (data) => {
-                process.stdout.write(data.toString('utf8'));
-            });
-
-            listener.stderr.on('data', (data) => {
-                process.stdout.write(data.toString('utf8'));
-            });
-
-            listener.on('close', (code) => {
-                console.log(`Runner listener exited with error code ${code}`);
-
-                if (code === 0) {
-                    console.log('Runner listener exit with 0 return code, stop the service, no retry needed.');
-                    stopping = true;
-                } else if (code === 1) {
-                    console.log('Runner listener exit with terminated error, stop the service, no retry needed.');
-                    stopping = true;
-                } else if (code === 2) {
-                    console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.');
-                } else if (code === 3) {
-                    console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.');
-                } else {
-                    console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.');
-                }
-
-                if(!stopping) {
-                    setTimeout(runService, 5000);
-                }
-            });
-
-        } catch(ex) {
-            console.log(ex);
-        }
-    }
-}
-
-runService();
-console.log('Started running service');
-
-var gracefulShutdown = function(code) {
-    console.log('Shutting down runner listener');
-    stopping = true;
-    if (listener) {
-        console.log('Sending SIGINT to runner listener to stop');
-        listener.kill('SIGINT');
-
-        // TODO wait for 30 seconds and send a SIGKILL
-    }
-}
-
-process.on('SIGINT', () => {
-    gracefulShutdown(0);
-});
-
-process.on('SIGTERM', () => {
-    gracefulShutdown(0);
-});

runner/patched/runsvc.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# convert SIGTERM signal to SIGINT
-# for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
-trap 'kill -INT $PID' TERM INT
-
-if [ -f ".path" ]; then
-    # configure
-    export PATH=`cat .path`
-    echo ".path=${PATH}"
-fi
-
-# insert anything to setup env when running as a service
-
-# run the host process which keep the listener alive
-./externals/node12/bin/node ./bin/RunnerService.js $* &
-PID=$!
-wait $PID
-trap - TERM INT
-wait $PID

simulator/runnergroup_visibility.go

@@ -0,0 +1,63 @@
+package simulator
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/actions-runner-controller/actions-runner-controller/github"
+)
+
+type Simulator struct {
+	Client *github.Client
+}
+
+func (c *Simulator) GetRunnerGroupsVisibleToRepository(ctx context.Context, org, repo string, managed *VisibleRunnerGroups) (*VisibleRunnerGroups, error) {
+	visible := NewVisibleRunnerGroups()
+
+	if org == "" {
+		panic(fmt.Sprintf("BUG: owner should not be empty in this context. repo=%v", repo))
+	}
+
+	runnerGroups, err := c.Client.ListOrganizationRunnerGroups(ctx, org)
+	if err != nil {
+		return visible, err
+	}
+
+	for _, runnerGroup := range runnerGroups {
+		ref := NewRunnerGroupFromGitHub(runnerGroup)
+
+		if !managed.Includes(ref) {
+			continue
+		}
+
+		if runnerGroup.GetVisibility() != "all" {
+			hasAccess, err := c.hasRepoAccessToOrganizationRunnerGroup(ctx, org, runnerGroup.GetID(), repo)
+			if err != nil {
+				return visible, err
+			}
+
+			if !hasAccess {
+				continue
+			}
+		}
+
+		visible.Add(ref)
+	}
+
+	return visible, nil
+}
+
+func (c *Simulator) hasRepoAccessToOrganizationRunnerGroup(ctx context.Context, org string, runnerGroupId int64, repo string) (bool, error) {
+	repos, err := c.Client.ListRunnerGroupRepositoryAccesses(ctx, org, runnerGroupId)
+	if err != nil {
+		return false, err
+	}
+
+	for _, githubRepo := range repos {
+		if githubRepo.GetFullName() == repo {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

simulator/runnergroups.go

@@ -0,0 +1,194 @@
+package simulator
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/google/go-github/v39/github"
+)
+
+type RunnerGroupScope int
+
+const (
+	Organization RunnerGroupScope = iota
+	Enterprise
+)
+
+func (s RunnerGroupScope) String() string {
+	switch s {
+	case Organization:
+		return "Organization"
+	case Enterprise:
+		return "Enterprise"
+	default:
+		panic(fmt.Sprintf("unimplemented RunnerGroupScope: %v", int(s)))
+	}
+}
+
+type RunnerGroupKind int
+
+const (
+	Default RunnerGroupKind = iota
+	Custom
+)
+
+func (s RunnerGroupKind) String() string {
+	switch s {
+	case Default:
+		return "Default"
+	case Custom:
+		return "Custom"
+	default:
+		panic(fmt.Sprintf("unimplemented RunnerGroupKind: %v", int(s)))
+	}
+}
+
+func NewRunnerGroupFromGitHub(g *github.RunnerGroup) RunnerGroup {
+	var name string
+	if !g.GetDefault() {
+		name = g.GetName()
+	}
+
+	var scope RunnerGroupScope
+
+	if g.GetInherited() {
+		scope = Enterprise
+	} else {
+		scope = Organization
+	}
+
+	return newRunnerGroup(scope, name)
+}
+
+func NewRunnerGroupFromProperties(enterprise, organization, group string) RunnerGroup {
+	var scope RunnerGroupScope
+
+	if enterprise != "" {
+		scope = Enterprise
+	} else {
+		scope = Organization
+	}
+
+	return newRunnerGroup(scope, group)
+}
+
+// newRunnerGroup creates a new RunnerGroup instance from the provided arguments.
+// There's a convention that an empty name implies a default runner group.
+func newRunnerGroup(scope RunnerGroupScope, name string) RunnerGroup {
+	if name == "" {
+		return RunnerGroup{
+			Scope: scope,
+			Kind:  Default,
+			Name:  "",
+		}
+	}
+
+	return RunnerGroup{
+		Scope: scope,
+		Kind:  Custom,
+		Name:  name,
+	}
+}
+
+type RunnerGroup struct {
+	Scope RunnerGroupScope
+	Kind  RunnerGroupKind
+	Name  string
+}
+
+func (r RunnerGroup) String() string {
+	return fmt.Sprintf("RunnerGroup{Scope:%s, Kind:%s, Name:%s}", r.Scope, r.Kind, r.Name)
+}
+
+// VisibleRunnerGroups is a set of enterprise and organization runner groups
+// that are visible to a GitHub repository.
+// GitHub Actions chooses one of such visible group on which the workflow job is scheduled.
+// ARC chooses the same group as Actions as the scale target.
+type VisibleRunnerGroups struct {
+	// sortedGroups is a pointer to a mutable list of RunnerGroups that contains all the runner sortedGroups
+	// that are visible to the repository, including organization sortedGroups defined at the organization level,
+	// and enterprise sortedGroups that are inherited down to the organization.
+	sortedGroups []RunnerGroup
+}
+
+func NewVisibleRunnerGroups() *VisibleRunnerGroups {
+	return &VisibleRunnerGroups{}
+}
+
+func (g *VisibleRunnerGroups) String() string {
+	var gs []string
+	for _, g := range g.sortedGroups {
+		gs = append(gs, g.String())
+	}
+
+	return strings.Join(gs, ", ")
+}
+
+func (g *VisibleRunnerGroups) IsEmpty() bool {
+	return len(g.sortedGroups) == 0
+}
+
+func (r *VisibleRunnerGroups) Includes(ref RunnerGroup) bool {
+	for _, r := range r.sortedGroups {
+		if r.Scope == ref.Scope && r.Kind == ref.Kind && r.Name == ref.Name {
+			return true
+		}
+	}
+	return false
+}
+
+// Add adds a runner group into VisibleRunnerGroups
+// at a certain position in the list so that
+// Traverse can return runner groups in order of higher precedence to lower precedence.
+func (g *VisibleRunnerGroups) Add(rg RunnerGroup) error {
+	n := len(g.sortedGroups)
+	i := sort.Search(n, func(i int) bool {
+		data := g.sortedGroups[i]
+
+		if rg.Kind > data.Kind {
+			return false
+		} else if rg.Kind < data.Kind {
+			return true
+		}
+
+		if rg.Scope > data.Scope {
+			return false
+		} else if rg.Scope < data.Scope {
+			return true
+		}
+
+		return false
+	})
+
+	g.insert(rg, i)
+
+	return nil
+}
+
+func (g *VisibleRunnerGroups) insert(rg RunnerGroup, i int) {
+	var result []RunnerGroup
+
+	result = append(result, g.sortedGroups[:i]...)
+	result = append(result, rg)
+	result = append(result, g.sortedGroups[i:]...)
+
+	g.sortedGroups = result
+}
+
+// Traverse traverses all the runner groups visible to a repository
+// in order of higher precedence to lower precedence.
+func (g *VisibleRunnerGroups) Traverse(f func(RunnerGroup) (bool, error)) error {
+	for _, rg := range g.sortedGroups {
+		ok, err := f(rg)
+		if err != nil {
+			return err
+		}
+
+		if ok {
+			return nil
+		}
+	}
+
+	return nil
+}

simulator/runnergroups_test.go

@@ -0,0 +1,94 @@
+package simulator
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestVisibleRunnerGroupsInsert(t *testing.T) {
+	g := NewVisibleRunnerGroups()
+
+	orgDefault := NewRunnerGroupFromProperties("", "myorg1", "")
+	orgCustom := NewRunnerGroupFromProperties("", "myorg1", "myorg1group1")
+	enterpriseDefault := NewRunnerGroupFromProperties("myenterprise1", "", "")
+
+	g.insert(orgCustom, 0)
+	g.insert(orgDefault, 0)
+	g.insert(enterpriseDefault, 1)
+
+	var got []RunnerGroup
+
+	err := g.Traverse(func(rg RunnerGroup) (bool, error) {
+		got = append(got, rg)
+		return false, nil
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, []RunnerGroup{orgDefault, enterpriseDefault, orgCustom}, got, "Unexpected result")
+}
+
+func TestVisibleRunnerGroups(t *testing.T) {
+	v := NewVisibleRunnerGroups()
+
+	requireGroups := func(t *testing.T, included, notIncluded []RunnerGroup) {
+		t.Helper()
+
+		for _, rg := range included {
+			if !v.Includes(rg) {
+				t.Errorf("%v must be included", rg)
+			}
+		}
+
+		for _, rg := range notIncluded {
+			if v.Includes(rg) {
+				t.Errorf("%v must not be included", rg)
+			}
+		}
+
+		var got []RunnerGroup
+
+		err := v.Traverse(func(rg RunnerGroup) (bool, error) {
+			got = append(got, rg)
+
+			return false, nil
+		})
+
+		require.NoError(t, err)
+		require.Equal(t, included, got)
+	}
+
+	orgDefault := NewRunnerGroupFromProperties("", "myorg1", "")
+	orgCustom := NewRunnerGroupFromProperties("", "myorg1", "myorg1group1")
+	enterpriseDefault := NewRunnerGroupFromProperties("myenterprise1", "", "")
+	enterpriseCustom := NewRunnerGroupFromProperties("myenterprise1", "", "myenterprise1group1")
+
+	requireGroups(t, nil, []RunnerGroup{orgDefault, enterpriseDefault, orgCustom, enterpriseCustom})
+
+	v.Add(orgCustom)
+
+	requireGroups(t, []RunnerGroup{orgCustom}, []RunnerGroup{orgDefault, enterpriseDefault, enterpriseCustom})
+
+	v.Add(orgDefault)
+
+	requireGroups(t, []RunnerGroup{orgDefault, orgCustom}, []RunnerGroup{enterpriseDefault, enterpriseCustom})
+
+	v.Add(enterpriseCustom)
+
+	requireGroups(t, []RunnerGroup{orgDefault, orgCustom, enterpriseCustom}, []RunnerGroup{enterpriseDefault})
+
+	v.Add(enterpriseDefault)
+
+	requireGroups(t, []RunnerGroup{orgDefault, enterpriseDefault, orgCustom, enterpriseCustom}, nil)
+
+	var first []RunnerGroup
+
+	err := v.Traverse(func(rg RunnerGroup) (bool, error) {
+		first = append(first, rg)
+
+		return true, nil
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, []RunnerGroup{orgDefault}, first)
+}