scale{Up,Down}Adjustment to add/remove constant number of replicas on scaling (#315)

* `scale{Up,Down}Adjustment` to add/remove constant number of replicas on scaling

Ref #305

* Bump chart version

.github/workflows/build-and-release-runners.yml

@@ -0,0 +1,91 @@
+name: Build and Release Runners
+
+on:
+  pull_request:
+    branches:
+      - '**'
+    paths:
+      - 'runner/**'
+      - .github/workflows/build-runner.yml
+  push:
+    branches:
+      - master
+    paths:
+      - runner/patched/*
+      - runner/Dockerfile
+      - runner/dindrunner.Dockerfile
+      - runner/entrypoint.sh
+      - .github/workflows/build-runner.yml
+  release:
+    types: [published]
+name: Runner
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Build ${{ matrix.name }}
+    strategy:
+      matrix:
+        include:
+          - name: actions-runner
+            dockerfile: Dockerfile
+          - name: actions-runner-dind
+            dockerfile: dindrunner.Dockerfile
+    env:
+      RUNNER_VERSION: 2.276.1
+      DOCKER_VERSION: 19.03.12
+      DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+    steps:
+      - name: Set outputs
+        id: vars
+        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: latest
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        if: ${{ github.event_name == 'push' }}
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      # Considered unstable builds
+      # Mutable (no sha) and immutable (include sha) tags are created, see Issue 285 and PR 286 for why
+      - name: Build and push canary builds
+        uses: docker/build-push-action@v2
+        with:
+          context: ./runner
+          file: ./runner/${{ matrix.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' && github.event_name != 'release' }}
+          build-args: |
+            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
+            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-canary
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-canary-${{ steps.vars.outputs.sha_short }}
+
+      # Considered stable builds
+      # Mutable (no sha) and immutable (include sha) tags are created, see Issue 285 and PR 286 for why
+      - name: Build and push release builds
+        uses: docker/build-push-action@v2
+        with:
+          context: ./runner
+          file: ./runner/${{ matrix.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name == 'release' }}
+          build-args: |
+            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
+            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-${{ steps.vars.outputs.sha_short }}
+            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:latest

.github/workflows/build-runner.yml

@@ -1,22 +0,0 @@
-on:
-  push:
-    branches:
-    - master
-    paths:
-    - 'runner/**'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    name: Build runner
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Build container image
-      run: make docker-build
-      working-directory: runner
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
-      working-directory: runner

.github/workflows/on-push-lint-charts.yml

@@ -0,0 +1,75 @@
+name: Lint and Test Charts
+
+on:
+  push:
+    paths:
+      - 'charts/**'
+      - '.github/**'
+  workflow_dispatch:
+
+env:
+  KUBE_SCORE_VERSION: 1.10.0
+  HELM_VERSION: v3.4.1
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: ${{ env.HELM_VERSION }}
+
+      - name: Set up kube-score
+        run: |
+          wget https://github.com/zegl/kube-score/releases/download/v${{ env.KUBE_SCORE_VERSION }}/kube-score_${{ env.KUBE_SCORE_VERSION }}_linux_amd64 -O kube-score
+          chmod 755 kube-score
+
+      - name: Kube-score generated manifests
+        run: helm template  --values charts/.ci/values-kube-score.yaml charts/* | ./kube-score score -
+              --ignore-test pod-networkpolicy
+              --ignore-test deployment-has-poddisruptionbudget
+              --ignore-test deployment-has-host-podantiaffinity
+              --ignore-test container-security-context
+              --ignore-test pod-probes
+              --ignore-test container-image-tag
+              --enable-optional-test container-security-context-privileged
+              --enable-optional-test container-security-context-readonlyrootfilesystem
+
+      # python is a requirement for the chart-testing action below (supports yamllint among other tests)
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.0.1
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config charts/.ci/ct-config.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Run chart-testing (lint)
+        run: ct lint --config charts/.ci/ct-config.yaml
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1.0.0
+        if: steps.list-changed.outputs.changed == 'true'
+
+      # We need cert-manager already installed in the cluster because we assume the CRDs exist
+      - name: Install cert-manager
+        run: |
+          helm repo add jetstack https://charts.jetstack.io --force-update
+          helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
+        if: steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart-testing (install)
+        run: ct install --config charts/.ci/ct-config.yaml

.github/workflows/on-push-master-publish-chart.yml

@@ -0,0 +1,101 @@
+name: Publish helm chart
+
+on:
+  push:
+    branches:
+      - master
+      - main # assume that the branch name may change in future
+    paths:
+      - 'charts/**'
+      - '.github/**'
+  workflow_dispatch:
+
+env:
+  KUBE_SCORE_VERSION: 1.10.0
+  HELM_VERSION: v3.4.1
+
+jobs:
+  lint-chart:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: ${{ env.HELM_VERSION }}
+
+      - name: Set up kube-score
+        run: |
+          wget https://github.com/zegl/kube-score/releases/download/v${{ env.KUBE_SCORE_VERSION }}/kube-score_${{ env.KUBE_SCORE_VERSION }}_linux_amd64 -O kube-score
+          chmod 755 kube-score
+
+      - name: Kube-score generated manifests
+        run: helm template  --values charts/.ci/values-kube-score.yaml charts/* | ./kube-score score -
+              --ignore-test pod-networkpolicy
+              --ignore-test deployment-has-poddisruptionbudget
+              --ignore-test deployment-has-host-podantiaffinity
+              --ignore-test container-security-context
+              --ignore-test pod-probes
+              --ignore-test container-image-tag
+              --enable-optional-test container-security-context-privileged
+              --enable-optional-test container-security-context-readonlyrootfilesystem
+
+      # python is a requirement for the chart-testing action below (supports yamllint among other tests)
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.0.1
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config charts/.ci/ct-config.yaml)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Run chart-testing (lint)
+        run: ct lint --config charts/.ci/ct-config.yaml
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1.0.0
+        if: steps.list-changed.outputs.changed == 'true'
+
+      # We need cert-manager already installed in the cluster because we assume the CRDs exist
+      - name: Install cert-manager
+        run: |
+          helm repo add jetstack https://charts.jetstack.io --force-update
+          helm install cert-manager jetstack/cert-manager --set installCRDs=true --wait
+        if: steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart-testing (install)
+        run: ct install --config charts/.ci/ct-config.yaml
+        if: steps.list-changed.outputs.changed == 'true'
+
+  publish-chart:
+
+    runs-on: ubuntu-latest
+    needs: lint-chart
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.1.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+

.github/workflows/release.yml

@@ -6,28 +6,57 @@ jobs:
   build:
     runs-on: ubuntu-latest
     name: Release
+    env:
+      DOCKERHUB_USERNAME: ${{ github.repository_owner }}
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Install tools
-      run: |
-        curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
-        tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz
-        sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
-        curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
-        sudo mv kustomize /usr/local/bin
-        curl -L -O https://github.com/tcnksm/ghr/releases/download/v0.13.0/ghr_v0.13.0_linux_amd64.tar.gz
-        tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
-        sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
-    - name: Set version
-      run: echo "::set-env name=VERSION::$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')"
-    - name: Upload artifacts
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: make github-release
-    - name: Build container image
-      run: make docker-build
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
+      - name: Set outputs
+        id: vars
+        run: echo ::set-output name=sha_short::${GITHUB_SHA::7}
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install tools
+        run: |
+          curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
+          tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz
+          sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
+          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
+          sudo mv kustomize /usr/local/bin
+          curl -L -O https://github.com/tcnksm/ghr/releases/download/v0.13.0/ghr_v0.13.0_linux_amd64.tar.gz
+          tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
+          sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
+
+      - name: Set version
+        run: echo "VERSION=$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')" >> $GITHUB_ENV
+
+      - name: Upload artifacts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make github-release
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: latest
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        with:
+          file: Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
+

.github/workflows/test.yaml

@@ -1,15 +1,16 @@
+name: CI
+
 on:
-  push:
+  pull_request:
     branches:
-    - master
+      - master
     paths-ignore:
     - 'runner/**'
-    - '.github/**'
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
-    name: Build
+    name: Test
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -20,9 +21,7 @@ jobs:
         sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
     - name: Run tests
       run: make test
-    - name: Build container image
-      run: make docker-build
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
+    - name: Verify manifests are up-to-date
+      run: |
+        make manifests
+        git diff --exit-code

.github/workflows/wip.yml

@@ -0,0 +1,40 @@
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - "runner/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: release-latest
+    env:
+      DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: latest
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        with:
+          file: Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest 
+

.gitignore

@@ -23,3 +23,9 @@ bin
 *.swp
 *.swo
 *~
+
+.envrc
+*.pem
+
+# OS
+.DS_STORE

Dockerfile

@@ -1,27 +1,39 @@
 # Build the manager binary
-FROM golang:1.13 as builder
+FROM golang:1.15 as builder
+
+ARG TARGETPLATFORM
 
 WORKDIR /workspace
+
+ENV GO111MODULE=on \
+  CGO_ENABLED=0
+
 # Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
+COPY go.mod go.sum ./
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
-COPY api/ api/
-COPY controllers/ controllers/
+COPY . .
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
+RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
+  export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
+  GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \
+  go build -a -o manager main.go && \
+  go build -a -o github-webhook-server ./cmd/githubwebhookserver
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
+
 WORKDIR /
+
 COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/github-webhook-server .
+
 USER nonroot:nonroot
 
 ENTRYPOINT ["/manager"]

Makefile

@@ -1,5 +1,8 @@
 NAME ?= summerwind/actions-runner-controller
 VERSION ?= latest
+# From https://github.com/VictoriaMetrics/operator/pull/44
+YAML_DROP=$(YQ) delete --inplace
+YAML_DROP_PREFIX=spec.validation.openAPIV3Schema.properties.spec.properties
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
@@ -11,6 +14,23 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
+
 all: manager
 
 # Run tests
@@ -39,9 +59,14 @@ deploy: manifests
 	kustomize build config/default | kubectl apply -f -
 
 # Generate manifests e.g. CRD, RBAC etc.
-manifests: controller-gen
+manifests: manifests-118 fix118 chart-crds
+
+manifests-118: controller-gen
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
+chart-crds:
+	cp config/crd/bases/*.yaml charts/actions-runner-controller/crds/
+
 # Run go fmt against code
 fmt:
 	go fmt ./...
@@ -50,6 +75,22 @@ fmt:
 vet:
 	go vet ./...
 
+# workaround for CRD issue with k8s 1.18 & controller-gen
+# ref: https://github.com/kubernetes/kubernetes/issues/91395
+fix118: yq
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).ephemeralContainers.items.properties
+
 # Generate code
 generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
@@ -62,12 +103,55 @@ docker-build: test
 docker-push:
 	docker push ${NAME}:${VERSION}
 
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:${VERSION}" \
+		-f Dockerfile \
+		. ${PUSH_ARG}
+
 # Generate the release manifest file
 release: manifests
 	cd config/manager && kustomize edit set image controller=${NAME}:${VERSION}
 	mkdir -p release
 	kustomize build config/default > release/actions-runner-controller.yaml
 
+.PHONY: release/clean
+release/clean:
+	rm -rf release
+
+.PHONY: acceptance
+acceptance: release/clean docker-build docker-push release
+	ACCEPTANCE_TEST_SECRET_TYPE=token make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+	ACCEPTANCE_TEST_SECRET_TYPE=app make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+	ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=token make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+	ACCEPTANCE_TEST_DEPLOYMENT_TOOL=helm ACCEPTANCE_TEST_SECRET_TYPE=app make acceptance/kind acceptance/setup acceptance/tests acceptance/teardown
+
+acceptance/kind:
+	kind create cluster --name acceptance
+	kubectl cluster-info --context kind-acceptance
+
+acceptance/setup:
+	kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.yaml	#kubectl create namespace actions-runner-system
+	kubectl -n cert-manager wait deploy/cert-manager-cainjector --for condition=available --timeout 60s
+	kubectl -n cert-manager wait deploy/cert-manager-webhook --for condition=available --timeout 60s
+	kubectl -n cert-manager wait deploy/cert-manager --for condition=available --timeout 60s
+	kubectl create namespace actions-runner-system || true
+	# Adhocly wait for some time until cert-manager's admission webhook gets ready
+	sleep 5
+
+acceptance/teardown:
+	kind delete cluster --name acceptance
+
+acceptance/tests:
+	acceptance/deploy.sh
+	acceptance/checks.sh
+
 # Upload release file to GitHub.
 github-release: release
 	ghr ${VERSION} release/
@@ -76,15 +160,34 @@ github-release: release
 # download controller-gen if necessary
 controller-gen:
 ifeq (, $(shell which controller-gen))
+ifeq (, $(wildcard $(GOBIN)/controller-gen))
 	@{ \
 	set -e ;\
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
+endif
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 else
 CONTROLLER_GEN=$(shell which controller-gen)
 endif
+
+# find or download yq
+# download yq if necessary
+# Use always go-version to get consistent line wraps etc.
+yq:
+ifeq (, $(wildcard $(GOBIN)/yq))
+	echo "Downloading yq"
+	@{ \
+	set -e ;\
+	YQ_TMP_DIR=$$(mktemp -d) ;\
+	cd $$YQ_TMP_DIR ;\
+	go mod init tmp ;\
+	go get github.com/mikefarah/yq/v3@3.4.0 ;\
+	rm -rf $$YQ_TMP_DIR ;\
+	}
+endif
+YQ=$(GOBIN)/yq

README.md

@@ -2,38 +2,192 @@
 
 This controller operates self-hosted runners for GitHub Actions on your Kubernetes cluster.
 
+ToC:
+
+- [Motivation](#motivation)
+- [Installation](#installation)
+  - [GitHub Enterprise support](#github-enterprise-support)
+- [Setting up authentication with GitHub API](#setting-up-authentication-with-github-api)
+  - [Using GitHub App](#using-github-app)
+  - [Using Personal AccessToken ](#using-personal-access-token)
+- [Usage](#usage)
+  - [Repository Runners](#repository-runners)
+  - [Organization Runners](#organization-runners)
+  - [Runner Deployments](#runnerdeployments)
+    - [Autoscaling](#autoscaling)
+      - [Faster Autoscaling with GitHub Webhook](#faster-autoscaling-with-github-webhook)
+  - [Runner with DinD](#runner-with-dind)
+  - [Additional tweaks](#additional-tweaks)
+  - [Runner labels](#runner-labels)
+  - [Runer groups](#runner-groups)
+  - [Using EKS IAM role for service accounts](#using-eks-iam-role-for-service-accounts)
+  - [Software installed in the runner image](#software-installed-in-the-runner-image)
+  - [Common errors](#common-errors)
+- [Developing](#developing)
+- [Alternatives](#alternatives)
+
 ## Motivation
 
-[GitHub Actions](https://github.com/features/actions) is very useful as a tool for automating development. GitHub Actions job is run in the cloud by default, but you may want to run your jobs in your environment. [Self-hosted runner](https://github.com/actions/runner) can be used for such use cases, but requires the provision of a virtual machine instance and configuration. If you already have a Kubernetes cluster, you'll want to run the self-hosted runner on top of it.
+[GitHub Actions](https://github.com/features/actions) is a very useful tool for automating development. GitHub Actions jobs are run in the cloud by default, but you may want to run your jobs in your environment. [Self-hosted runner](https://github.com/actions/runner) can be used for such use cases, but requires the provisioning and configuration of a virtual machine instance. Instead if you already have a Kubernetes cluster, it makes more sense to run the self-hosted runner on top of it.
 
-*actions-runner-controller* makes that possible. Just create a *Runner* resource on your Kubernetes, and it will run and operate the self-hosted runner of the specified repository. Combined with Kubernetes RBAC, you can also build simple Self-hosted runners as a Service.
+**actions-runner-controller** makes that possible. Just create a *Runner* resource on your Kubernetes, and it will run and operate the self-hosted runner for the specified repository. Combined with Kubernetes RBAC, you can also build simple Self-hosted runners as a Service.
 
 ## Installation
 
-First, install *actions-runner-controller* with a manifest file. This will create a *actions-runner-system* namespace in your Kubernetes and deploy the required resources.
+actions-runner-controller uses [cert-manager](https://cert-manager.io/docs/installation/kubernetes/) for certificate management of Admission Webhook. Make sure you have already installed cert-manager before you install. The installation instructions for cert-manager can be found below.
+
+- [Installing cert-manager on Kubernetes](https://cert-manager.io/docs/installation/kubernetes/)
+
+Install the custom resource and actions-runner-controller with `kubectl` or `helm`. This will create actions-runner-system namespace in your Kubernetes and deploy the required resources.
+
+`kubectl`:
 
 ```
-$ kubectl apply -f https://github.com/summerwind/actions-runner-controller/releases/latest/download/actions-runner-controller.yaml
+# REPLACE "v0.16.1" with the latest release
+kubectl apply -f https://github.com/summerwind/actions-runner-controller/releases/download/v0.16.1/actions-runner-controller.yaml
 ```
 
-Set your access token of GitHub to the secret. `${GITHUB_TOKEN}` is the value you must replace with your access token. This token is used to register Self-hosted runner by *actions-runner-controller*.
+`helm`:
 
 ```
-$ kubectl create secret generic controller-manager --from-literal=github_token=${GITHUB_TOKEN} -n actions-runner-system
+helm repo add actions-runner-controller https://summerwind.github.io/actions-runner-controller
+helm upgrade --install -n actions-runner-system actions-runner-controller/actions-runner-controller
+```
+
+### Github Enterprise support
+
+If you use either Github Enterprise Cloud or Server, you can use **actions-runner-controller**  with those, too.
+Authentication works same way as with public Github (repo and organization level).
+The minimum version of Github Enterprise Server is 3.0.0 (or rc1/rc2).
+In most cases maintainers do not have environment where to test changes and are reliant on the community for testing.
+
+
+```shell
+kubectl set env deploy controller-manager -c manager GITHUB_ENTERPRISE_URL=<GHEC/S URL> --namespace actions-runner-system
+```
+
+#### Enterprise runners usage
+
+In order to use enterprise runners you must have Admin access to Github Enterprise and you should do Personal Access Token (PAT)
+with `enterprise:admin` access. Enterprise runners are not possible to run with Github APP or any other permission.
+
+When you use enterprise runners those will get access to Github Organisations. However, access to the repositories is **NOT**
+allowed by default. Each Github Organisation must allow Enterprise runner groups to be used in repositories.
+This is needed only one time and is permanent after that.
+
+Example:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: ghe-runner-deployment
+spec:
+  replicas: 2
+  template:
+    spec:
+      enterprise: your-enterprise-name
+      dockerdWithinRunnerContainer: true
+      resources:
+        limits:
+          cpu: "4000m"
+          memory: "2Gi"
+        requests:
+          cpu: "200m"
+          memory: "200Mi"
+      volumeMounts:
+      - mountPath: /runner
+        name: runner
+      volumes:
+      - name: runner
+        emptyDir: {}
+
+```
+
+## Setting up authentication with GitHub API
+
+There are two ways for actions-runner-controller to authenticate with the GitHub API:
+
+1. Using GitHub App.
+2. Using Personal Access Token.
+
+Regardless of which authentication method you use, the same permissions are required, those permissions are:
+- Repository: Administration (read/write)
+- Repository: Actions (read)
+- Organization: Self-hosted runners (read/write)
+
+
+**NOTE: It is extremely important to only follow one of the sections below and not both.**
+
+### Using GitHub App
+
+You can create a GitHub App for either your account or any organization. If you want to create a GitHub App for your account, open the following link to the creation page, enter any unique name in the "GitHub App name" field, and hit the "Create GitHub App" button at the bottom of the page.
+
+- [Create GitHub Apps on your account](https://github.com/settings/apps/new?url=http://github.com/summerwind/actions-runner-controller&webhook_active=false&public=false&administration=write&actions=read)
+
+If you want to create a GitHub App for your organization, replace the `:org` part of the following URL with your organization name before opening it. Then enter any unique name in the "GitHub App name" field, and hit the "Create GitHub App" button at the bottom of the page to create a GitHub App.
+
+- [Create GitHub Apps on your organization](https://github.com/organizations/:org/settings/apps/new?url=http://github.com/summerwind/actions-runner-controller&webhook_active=false&public=false&administration=write&organization_self_hosted_runners=write&actions=read)
+
+You will see an *App ID* on the page of the GitHub App you created as follows, the value of this App ID will be used later.
+
+<img width="750" alt="App ID" src="https://user-images.githubusercontent.com/230145/78968802-6e7c8880-7b40-11ea-8b08-0c1b8e6a15f0.png">
+
+Download the private key file by pushing the "Generate a private key" button at the bottom of the GitHub App page. This file will also be used later.
+
+<img width="750" alt="Generate a private key" src="https://user-images.githubusercontent.com/230145/78968805-71777900-7b40-11ea-97e6-55c48dfc44ac.png">
+
+Go to the "Install App" tab on the left side of the page and install the GitHub App that you created for your account or organization.
+
+<img width="750" alt="Install App" src="https://user-images.githubusercontent.com/230145/78968806-72100f80-7b40-11ea-810d-2bd3261e9d40.png">
+
+When the installation is complete, you will be taken to a URL in one of the following formats, the last number of the URL will be used as the Installation ID later (For example, if the URL ends in `settings/installations/12345`, then the Installation ID is `12345`).
+
+- `https://github.com/settings/installations/${INSTALLATION_ID}`
+- `https://github.com/organizations/eventreactor/settings/installations/${INSTALLATION_ID}`
+
+Finally, register the App ID (`APP_ID`), Installation ID (`INSTALLATION_ID`), and downloaded private key file (`PRIVATE_KEY_FILE_PATH`) to Kubernetes as Secret.
+
+```shell
+$ kubectl create secret generic controller-manager \
+    -n actions-runner-system \
+    --from-literal=github_app_id=${APP_ID} \
+    --from-literal=github_app_installation_id=${INSTALLATION_ID} \
+    --from-file=github_app_private_key=${PRIVATE_KEY_FILE_PATH}
+```
+
+### Using Personal Access Token
+
+From an account that has `admin` privileges for the repository, create a [personal access token](https://github.com/settings/tokens) with `repo` scope. This token is used to register a self-hosted runner by *actions-runner-controller*.
+
+Self-hosted runners in GitHub can either be connected to a single repository, or to a GitHub organization (so they are available to all repositories in the organization). This token is used to register a self-hosted runner by *actions-runner-controller*.
+
+For adding a runner to a repository, the token should have `repo` scope. If the runner should be added to an organization, the token should have `admin:org` scope. Note that to use a Personal Access Token, you must issue the token with an account that has `admin` privileges (on the repository and/or the organization).
+
+Open the Create Token page from the following link, grant the `repo` and/or `admin:org` scope, and press the "Generate Token" button at the bottom of the page to create the token.
+
+- [Create personal access token](https://github.com/settings/tokens/new)
+
+Register the created token (`GITHUB_TOKEN`) as a Kubernetes secret.
+
+```shell
+kubectl create secret generic controller-manager \
+    -n actions-runner-system \
+    --from-literal=github_token=${GITHUB_TOKEN}
 ```
 
 ## Usage
 
-There's generally two ways to use this controller:
+There are two ways to use this controller:
 
-- Manage runners one by one with `Runner`
-- Manage a set of runners with `RunnerDeployment`
+- Manage runners one by one with `Runner`.
+- Manage a set of runners with `RunnerDeployment`.
 
-### Runners
+### Repository runners
 
-To launch a single Self-hosted runner, you need to create a manifest file includes *Runner* resource as follows. This example launches a self-hosted runner with name *example-runner* for the *summerwind/actions-runner-controller* repository.
+To launch a single self-hosted runner, you need to create a manifest file includes *Runner* resource as follows. This example launches a self-hosted runner with name *example-runner* for the *summerwind/actions-runner-controller* repository.
 
-```
+```yaml
 # runner.yaml
 apiVersion: actions.summerwind.dev/v1alpha1
 kind: Runner
@@ -46,14 +200,14 @@ spec:
 
 Apply the created manifest file to your Kubernetes.
 
-```
+```shell
 $ kubectl apply -f runner.yaml
 runner.actions.summerwind.dev/example-runner created
 ```
 
 You can see that the Runner resource has been created.
 
-```
+```shell
 $ kubectl get runners
 NAME             REPOSITORY                             STATUS
 example-runner   summerwind/actions-runner-controller   Running
@@ -61,21 +215,37 @@ example-runner   summerwind/actions-runner-controller   Running
 
 You can also see that the runner pod has been running.
 
-```
+```shell
 $ kubectl get pods
 NAME           READY   STATUS    RESTARTS   AGE
 example-runner 2/2     Running   0          1m
 ```
 
-The runner you created has been registerd to your repository.
+The runner you created has been registered to your repository.
 
 <img width="756" alt="Actions tab in your repository settings" src="https://user-images.githubusercontent.com/230145/73618667-8cbf9700-466c-11ea-80b6-c67e6d3f70e7.png">
 
-Now your can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
+Now you can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
+
+### Organization Runners
+
+To add the runner to an organization, you only need to replace the `repository` field with `organization`, so the runner will register itself to the organization.
+
+```yaml
+# runner.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: Runner
+metadata:
+  name: example-org-runner
+spec:
+  organization: your-organization-name
+```
+
+Now you can see the runner on the organization level (if you have organization owner permissions).
 
 ### RunnerDeployments
 
-There's also `RunnerReplicaSet` and `RunnerDeployment` that corresponds to `ReplicaSet` and `Deployment` but for `Runner`.
+There are `RunnerReplicaSet` and `RunnerDeployment` that corresponds to `ReplicaSet` and `Deployment` but for `Runner`.
 
 You usually need only `RunnerDeployment` rather than `RunnerReplicaSet` as the former is for managing the latter.
 
@@ -95,17 +265,481 @@ spec:
 
 Apply the manifest file to your cluster:
 
-```
+```shell
 $ kubectl apply -f runner.yaml
 runnerdeployment.actions.summerwind.dev/example-runnerdeploy created
 ```
 
-You can see that 2 runners has been created as specified by `replicas: 2`:
+You can see that 2 runners have been created as specified by `replicas: 2`:
 
-```
+```shell
 $ kubectl get runners
-NAME             REPOSITORY                             STATUS
 NAME                             REPOSITORY                             STATUS
 example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
 example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 ```
+
+#### Autoscaling
+
+`RunnerDeployment` can scale the number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
+
+In the below example, `actions-runner` checks for pending workflow runs for each sync period, and scale to e.g. 3 if there're 3 pending jobs at sync time.
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runner-deployment
+spec:
+  template:
+    spec:
+      repository: summerwind/actions-runner-controller
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    name: example-runner-deployment
+  minReplicas: 1
+  maxReplicas: 3
+  metrics:
+  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
+    repositoryNames:
+    - summerwind/actions-runner-controller
+```
+
+The scale out performance is controlled via the manager containers startup `--sync-period` argument. The default value is 10 minutes to prevent unconfigured deployments rate limiting themselves from the GitHub API. The period can be customised in the `config/default/manager_auth_proxy_patch.yaml` patch for those that are building the solution via the kustomize setup.
+
+Additionally, the autoscaling feature has an anti-flapping option that prevents periodic loop of scaling up and down.
+By default, it doesn't scale down until the grace period of 10 minutes passes after a scale up. The grace period can be configured by setting `scaleDownDelaySecondsAfterScaleUp`:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runner-deployment
+spec:
+  template:
+    spec:
+      repository: summerwind/actions-runner-controller
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    name: example-runner-deployment
+  minReplicas: 1
+  maxReplicas: 3
+  scaleDownDelaySecondsAfterScaleOut: 60
+  metrics:
+  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
+    repositoryNames:
+    - summerwind/actions-runner-controller
+```
+
+If you do not want to manage an explicit list of repositories to scale, an alternate autoscaling scheme that can be applied is the PercentageRunnersBusy scheme. The number of desired pods are evaulated by checking how many runners are currently busy and applying a scaleup or scale down factor if certain thresholds are met. By setting the metric type to PercentageRunnersBusy, the HorizontalRunnerAutoscaler will query github for the number of busy runners which live in the RunnerDeployment namespace. Scaleup and scaledown thresholds are the percentage of busy runners at which the number of desired runners are re-evaluated. Scaleup and scaledown factors are the multiplicative factor applied to the current number of runners used to calculate the number of desired runners. This scheme is also especially useful if you want multiple controllers in various clusters, each responsible for scaling their own runner pods per namespace.
+
+```yaml
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    name: example-runner-deployment
+  minReplicas: 1
+  maxReplicas: 3
+  scaleDownDelaySecondsAfterScaleOut: 60
+  metrics:
+  - type: PercentageRunnersBusy
+    scaleUpThreshold: '0.75'
+    scaleDownThreshold: '0.3'
+    scaleUpFactor: '1.4'
+    scaleDownFactor: '0.7'
+```
+
+#### Faster Autoscaling with GitHub Webhook
+
+> This feature is an ADVANCED feature which may require more work to set up.
+> Please get prepared to put some time and effort to learn and leverage this feature!
+
+`actions-runner-controller` has an optional Webhook server that receives GitHub Webhook events and scale
+[`RunnerDeployment`s](#runnerdeployments) by updating corresponding [`HorizontalRunnerAutoscaler`s](#autoscaling).
+
+Today, the Webhook server can be configured to respond GitHub `check_run`, `pull_request`, and `push` events
+by scaling up the matching `HorizontalRunnerAutoscaler` by N replica(s), where `N` is configurable within
+`HorizontalRunerAutoscaler`'s `Spec`.
+
+More concretely, you can configure the targeted GitHub event types and the `N` in
+`scaleUpTriggers`:
+
+```yaml
+kind: HorizontalRunnerAutoscaler
+spec:
+  scaleTargetRef:
+    name: myrunners
+  scaleUpTrigggers:
+  - githubEvent:
+      checkRun:
+        types: ["created"]
+        status: "queued"
+    amount: 1
+    duration: "5m"
+```
+
+With the above example, the webhook server scales `myrunners` by `1` replica for 5 minutes on each `check_run` event
+with the type of `created` and the status of `queued` received. 
+
+The primary benefit of autoscaling on Webhook compared to the standard autoscaling is that this one allows you to
+immediately add "resource slack" for future GitHub Actions job runs.
+
+In contrast, the standard autoscaling requires you to wait next sync period to add
+insufficient runners. You can definitely shorten the sync period to make the standard autoscaling more responsive.
+But doing so eventually result in the controller not functional due to GitHub API rate limit.
+
+> You can learn the implementation details in #282
+
+To enable this feature, you firstly need to install the webhook server.
+
+Currently, only our Helm chart has the ability install it.
+
+```console
+$ helm --upgrade install actions-runner-controller/actions-runner-controller \
+  githubWebhookServer.enabled=true \
+  githubWebhookServer.ports[0].nodePort=33080
+```
+
+The above command will result in exposing the node port 33080 for Webhook events. Usually, you need to create an
+external loadbalancer targeted to the node port, and register the hostname or the IP address of the external loadbalancer
+to the GitHub Webhook.
+
+Once you were able to confirm that the Webhook server is ready and running from GitHub - this is usually verified by the
+GitHub sending PING events to the Webhook server - create or update your `HorizontalRunnerAutoscaler` resources
+by learning the following configuration examples.
+
+- [Example 1: Scale up on each `check_run` event](#example-1-scale-up-on-each-check_run-event)
+- [Example 2: Scale on each `pull_request` event against `develop` or `main` branches](#example-2-scale-on-each-pull_request-event-against-develop-or-main-branches)
+
+##### Example 1: Scale up on each `check_run` event
+
+> Note: This should work almost like https://github.com/philips-labs/terraform-aws-github-runner
+
+To scale up replicas of the runners for `example/myrepo` by 1 for 5 minutes on each `check_run`, you write manifests like the below:
+
+```yaml
+kind: RunnerDeployment
+metadata:
+   name: myrunners
+spec:
+  repository: example/myrepo
+---
+kind: HorizontalRunnerAutoscaler
+spec:
+  scaleTargetRef:
+    name: myrunners
+  scaleUpTrigggers:
+  - githubEvent:
+      checkRun:
+        types: ["created"]
+        status: "queued"
+    amount: 1
+    duration: "5m"
+```
+
+###### Example 2: Scale on each `pull_request` event against `develop` or `main` branches
+
+```yaml
+kind: RunnerDeployment:
+metadata:
+   name: myrunners
+spec:
+  repository: example/myrepo
+---
+kind: HorizontalRunnerAutoscaler
+spec:
+  scaleTargetRef:
+    name: myrunners
+  scaleUpTrigggers:
+  - githubEvent:
+      pullRequest:
+        types: ["synchronize"]
+        branches: ["main", "develop"]
+    amount: 1
+    duration: "5m"
+```
+
+See ["activity types"](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request) for the list of valid values for `scaleUpTriggers[].githubEvent.pullRequest.types`.
+
+### Runner with DinD
+
+When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
+
+```yaml
+# dindrunnerdeployment.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-dindrunnerdeploy
+spec:
+  replicas: 2
+  template:
+    spec:
+      image: summerwind/actions-runner-dind
+      dockerdWithinRunnerContainer: true
+      repository: mumoshu/actions-runner-controller-ci
+      env: []
+```
+
+This also helps with resources, as you don't need to give resources separately to docker and runner.
+
+### Additional tweaks
+
+You can pass details through the spec selector. Here's an eg. of what you may like to do:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: actions-runner
+  namespace: default
+spec:
+  replicas: 2
+  template:
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/test: ""
+
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/test
+        operator: Exists
+
+      repository: mumoshu/actions-runner-controller-ci
+      image: custom-image/actions-runner:latest
+      imagePullPolicy: Always
+      resources:
+        limits:
+          cpu: "4.0"
+          memory: "8Gi"
+        requests:
+          cpu: "2.0"
+          memory: "4Gi"
+      
+      # Timeout after a node crashed or became unreachable to evict your pods somewhere else (default 5mins)
+      tolerations:
+        - key: "node.kubernetes.io/unreachable"
+          operator: "Exists"
+          effect: "NoExecute"
+          tolerationSeconds: 10
+          
+      # If set to false, there are no privileged container and you cannot use docker.
+      dockerEnabled: false
+      # If set to true, runner pod container only 1 container that's expected to be able to run docker, too.
+      # image summerwind/actions-runner-dind or custom one should be used with true -value
+      dockerdWithinRunnerContainer: false
+      # Valid if dockerdWithinRunnerContainer is not true
+      dockerdContainerResources:
+        limits:
+          cpu: "4.0"
+          memory: "8Gi"
+        requests:
+          cpu: "2.0"
+          memory: "4Gi"
+      sidecarContainers:
+        - name: mysql
+          image: mysql:5.7
+          env:
+            - name: MYSQL_ROOT_PASSWORD
+              value: abcd1234
+          securityContext:
+            runAsUser: 0
+      # if workDir is not specified, the default working directory is /runner/_work
+      # this setting allows you to customize the working directory location
+      # for example, the below setting is the same as on the ubuntu-18.04 image
+      workDir: /home/runner/work
+```
+
+### Runner labels
+
+To run a workflow job on a self-hosted runner, you can use the following syntax in your workflow:
+
+```yaml
+jobs:
+  release:
+    runs-on: self-hosted
+```
+
+When you have multiple kinds of self-hosted runners, you can distinguish between them using labels. In order to do so, you can specify one or more labels in your `Runner` or `RunnerDeployment` spec.
+
+```yaml
+# runnerdeployment.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: custom-runner
+spec:
+  replicas: 1
+  template:
+    spec:
+      repository: summerwind/actions-runner-controller
+      labels:
+        - custom-runner
+```
+
+Once this spec is applied, you can observe the labels for your runner from the repository or organization in the GitHub settings page for the repository or organization. You can now select a specific runner from your workflow by using the label in `runs-on`:
+
+```yaml
+jobs:
+  release:
+    runs-on: custom-runner
+```
+
+Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
+
+### Runner Groups
+
+Runner groups can be used to limit which repositories are able to use the GitHub Runner at an Organisation level. Runner groups have to be [created in GitHub first](https://docs.github.com/en/actions/hosting-your-own-runners/managing-access-to-self-hosted-runners-using-groups) before they can be referenced.
+
+To add the runner to the group `NewGroup`, specify the group in your `Runner` or `RunnerDeployment` spec.
+
+```yaml
+# runnerdeployment.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: custom-runner
+spec:
+  replicas: 1
+  template:
+    spec:
+      group: NewGroup
+```
+
+### Using EKS IAM role for service accounts
+
+`actions-runner-controller` v0.15.0 or later has support for EKS IAM role for service accounts.
+
+As similar as for regular pods and deployments, you firstly need an existing service account with the IAM role associated.
+Create one using e.g. `eksctl`. You can refer to [the EKS documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) for more details.
+
+Once you set up the service account, all you need is to add `serviceAccountName` and `fsGroup` to any pods that uses
+the IAM-role enabled service account.
+
+For `RunnerDeployment`, you can set those two fields under the runner spec at `RunnerDeployment.Spec.Template`:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeploy
+spec:
+  template:
+    spec:
+      repository: USER/REO
+      serviceAccountName: my-service-account
+      securityContext:
+        fsGroup: 1447
+```
+
+### Software installed in the runner image
+
+The GitHub hosted runners include a large amount of pre-installed software packages. For Ubuntu 18.04, this list can be found at <https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md>
+
+The container image is based on Ubuntu 18.04, but it does not contain all of the software installed on the GitHub runners. It contains the following subset of packages from the GitHub runners:
+
+- Basic CLI packages
+- git (2.26)
+- docker
+- build-essentials
+
+The virtual environments from GitHub contain a lot more software packages (different versions of Java, Node.js, Golang, .NET, etc) which are not provided in the runner image. Most of these have dedicated setup actions which allow the tools to be installed on-demand in a workflow, for example: `actions/setup-java` or `actions/setup-node`
+
+If there is a need to include packages in the runner image for which there is no setup action, then this can be achieved by building a custom container image for the runner. The easiest way is to start with the `summerwind/actions-runner` image and installing the extra dependencies directly in the docker image:
+
+```shell
+FROM summerwind/actions-runner:latest
+
+RUN sudo apt update -y \
+  && sudo apt install YOUR_PACKAGE
+  && sudo rm -rf /var/lib/apt/lists/*
+```
+
+You can then configure the runner to use a custom docker image by configuring the `image` field of a `Runner` or `RunnerDeployment`:
+
+```yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: Runner
+metadata:
+  name: custom-runner
+spec:
+  repository: summerwind/actions-runner-controller
+  image: YOUR_CUSTOM_DOCKER_IMAGE
+```
+
+### Common Errors
+
+#### invalid header field value
+
+```json
+2020-11-12T22:17:30.693Z	ERROR	controller-runtime.controller	Reconciler error	{"controller": "runner", "request": "actions-runner-system/runner-deployment-dk7q8-dk5c9", "error": "failed to create registration token: Post \"https://api.github.com/orgs/$YOUR_ORG_HERE/actions/runners/registration-token\": net/http: invalid header field value \"Bearer $YOUR_TOKEN_HERE\\n\" for key Authorization"}
+```
+
+**Solutions**<br />
+Your base64'ed PAT token has a new line at the end, it needs to be created without a `\n` added
+* `echo -n $TOKEN | base64`
+* Create the secret as described in the docs using the shell and documeneted flags
+
+# Developing
+
+If you'd like to modify the controller to fork or contribute, I'd suggest using the following snippet for running
+the acceptance test:
+
+```shell
+# This sets `VERSION` envvar to some appropriate value
+. hack/make-env.sh
+
+NAME=$DOCKER_USER/actions-runner-controller \
+  GITHUB_TOKEN=*** \
+  APP_ID=*** \
+  PRIVATE_KEY_FILE_PATH=path/to/pem/file \
+  INSTALLATION_ID=*** \
+  make docker-build docker-push acceptance
+```
+
+Please follow the instructions explained in [Using Personal Access Token](#using-personal-access-token) to obtain
+`GITHUB_TOKEN`, and those in [Using GitHub App](#using-github-app) to obtain `APP_ID`, `INSTALLATION_ID`, and
+`PRIAVTE_KEY_FILE_PATH`.
+
+The test creates a one-off `kind` cluster, deploys `cert-manager` and `actions-runner-controller`,
+creates a `RunnerDeployment` custom resource for a public Git repository to confirm that the
+controller is able to bring up a runner pod with the actions runner registration token installed.
+
+If you prefer to test in a non-kind cluster, you can instead run:
+
+```shell script
+KUBECONFIG=path/to/kubeconfig \
+NAME=$DOCKER_USER/actions-runner-controller \
+  GITHUB_TOKEN=*** \
+  APP_ID=*** \
+  PRIVATE_KEY_FILE_PATH=path/to/pem/file \
+  INSTALLATION_ID=*** \
+  ACCEPTANCE_TEST_SECRET_TYPE=token \
+  make docker-build docker-push \
+       acceptance/setup acceptance/tests
+```
+# Alternatives
+
+The following is a list of alternative solutions that may better fit you depending on your use-case:
+
+- <https://github.com/evryfs/github-actions-runner-operator/>
+
+Although the situation can change over time, as of writing this sentence, the benefits of using `actions-runner-controller` over the alternatives are:
+
+- `actions-runner-controller` has the ability to autoscale runners based on number of pending/progressing jobs (#99)
+- `actions-runner-controller` is able to gracefully stop runners (#103)
+- `actions-runner-controller` has ARM support

acceptance/checks.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -e
+
+runner_name=
+
+while [ -z "${runner_name}" ]; do
+  echo Finding the runner... 1>&2
+  sleep 1
+  runner_name=$(kubectl get runner --output=jsonpath="{.items[*].metadata.name}")
+done
+
+echo Found runner ${runner_name}.
+
+pod_name=
+
+while [ -z "${pod_name}" ]; do
+  echo Finding the runner pod... 1>&2
+  sleep 1
+  pod_name=$(kubectl get pod --output=jsonpath="{.items[*].metadata.name}" | grep ${runner_name})
+done
+
+echo Found pod ${pod_name}.
+
+echo Waiting for pod ${runner_name} to become ready... 1>&2
+
+kubectl wait pod/${runner_name} --for condition=ready --timeout 180s
+
+echo All tests passed. 1>&2

acceptance/deploy.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -e
+
+tpe=${ACCEPTANCE_TEST_SECRET_TYPE}
+
+if [ "${tpe}" == "token" ]; then
+  kubectl create secret generic controller-manager \
+	  -n actions-runner-system \
+	  --from-literal=github_token=${GITHUB_TOKEN:?GITHUB_TOKEN must not be empty}
+elif [ "${tpe}" == "app" ]; then
+  kubectl create secret generic controller-manager \
+    -n actions-runner-system \
+    --from-literal=github_app_id=${APP_ID:?must not be empty} \
+    --from-literal=github_app_installation_id=${INSTALLATION_ID:?must not be empty} \
+    --from-file=github_app_private_key=${PRIVATE_KEY_FILE_PATH:?must not be empty}
+else
+  echo "ACCEPTANCE_TEST_SECRET_TYPE must be set to either \"token\" or \"app\"" 1>&2
+  exit 1
+fi
+
+tool=${ACCEPTANCE_TEST_DEPLOYMENT_TOOL}
+
+if [ "${tool}" == "helm" ]; then
+  helm upgrade --install actions-runner-controller \
+    charts/actions-runner-controller \
+    -n actions-runner-system \
+    --create-namespace \
+    --set syncPeriod=5m
+  kubectl -n actions-runner-system wait deploy/actions-runner-controller --for condition=available
+else
+  kubectl apply \
+    -n actions-runner-system \
+    -f release/actions-runner-controller.yaml
+  kubectl -n actions-runner-system wait deploy/controller-manager --for condition=available --timeout 60s
+fi
+
+# Adhocly wait for some time until actions-runner-controller's admission webhook gets ready
+sleep 20
+
+kubectl apply \
+  -f acceptance/testdata/runnerdeploy.yaml

acceptance/testdata/runnerdeploy.yaml

@@ -0,0 +1,9 @@
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runnerdeploy
+spec:
+#  replicas: 1
+  template:
+    spec:
+      repository: mumoshu/actions-runner-controller-ci

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -0,0 +1,193 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// HorizontalRunnerAutoscalerSpec defines the desired state of HorizontalRunnerAutoscaler
+type HorizontalRunnerAutoscalerSpec struct {
+	// ScaleTargetRef sis the reference to scaled resource like RunnerDeployment
+	ScaleTargetRef ScaleTargetRef `json:"scaleTargetRef,omitempty"`
+
+	// MinReplicas is the minimum number of replicas the deployment is allowed to scale
+	// +optional
+	MinReplicas *int `json:"minReplicas,omitempty"`
+
+	// MinReplicas is the maximum number of replicas the deployment is allowed to scale
+	// +optional
+	MaxReplicas *int `json:"maxReplicas,omitempty"`
+
+	// ScaleDownDelaySecondsAfterScaleUp is the approximate delay for a scale down followed by a scale up
+	// Used to prevent flapping (down->up->down->... loop)
+	// +optional
+	ScaleDownDelaySecondsAfterScaleUp *int `json:"scaleDownDelaySecondsAfterScaleOut,omitempty"`
+
+	// Metrics is the collection of various metric targets to calculate desired number of runners
+	// +optional
+	Metrics []MetricSpec `json:"metrics,omitempty"`
+
+	// ScaleUpTriggers is an experimental feature to increase the desired replicas by 1
+	// on each webhook requested received by the webhookBasedAutoscaler.
+	//
+	// This feature requires you to also enable and deploy the webhookBasedAutoscaler onto your cluster.
+	//
+	// Note that the added runners remain until the next sync period at least,
+	// and they may or may not be used by GitHub Actions depending on the timing.
+	// They are intended to be used to gain "resource slack" immediately after you
+	// receive a webhook from GitHub, so that you can loosely expect MinReplicas runners to be always available.
+	ScaleUpTriggers []ScaleUpTrigger `json:"scaleUpTriggers,omitempty"`
+
+	CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
+}
+
+type ScaleUpTrigger struct {
+	GitHubEvent *GitHubEventScaleUpTriggerSpec `json:"githubEvent,omitempty"`
+	Amount      int                            `json:"amount,omitempty"`
+	Duration    metav1.Duration                `json:"duration,omitempty"`
+}
+
+type GitHubEventScaleUpTriggerSpec struct {
+	CheckRun    *CheckRunSpec    `json:"checkRun,omitempty"`
+	PullRequest *PullRequestSpec `json:"pullRequest,omitempty"`
+	Push        *PushSpec        `json:"push,omitempty"`
+}
+
+// https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run
+type CheckRunSpec struct {
+	Types  []string `json:"types,omitempty"`
+	Status string   `json:"status,omitempty"`
+}
+
+// https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+type PullRequestSpec struct {
+	Types    []string `json:"types,omitempty"`
+	Branches []string `json:"branches,omitempty"`
+}
+
+// PushSpec is the condition for triggering scale-up on push event
+// Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
+type PushSpec struct {
+}
+
+// CapacityReservation specifies the number of replicas temporarily added
+// to the scale target until ExpirationTime.
+type CapacityReservation struct {
+	Name           string      `json:"name,omitempty"`
+	ExpirationTime metav1.Time `json:"expirationTime,omitempty"`
+	Replicas       int         `json:"replicas,omitempty"`
+}
+
+type ScaleTargetRef struct {
+	Name string `json:"name,omitempty"`
+}
+
+type MetricSpec struct {
+	// Type is the type of metric to be used for autoscaling.
+	// The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+	Type string `json:"type,omitempty"`
+
+	// RepositoryNames is the list of repository names to be used for calculating the metric.
+	// For example, a repository name is the REPO part of `github.com/USER/REPO`.
+	// +optional
+	RepositoryNames []string `json:"repositoryNames,omitempty"`
+
+	// ScaleUpThreshold is the percentage of busy runners greater than which will
+	// trigger the hpa to scale runners up.
+	// +optional
+	ScaleUpThreshold string `json:"scaleUpThreshold,omitempty"`
+
+	// ScaleDownThreshold is the percentage of busy runners less than which will
+	// trigger the hpa to scale the runners down.
+	// +optional
+	ScaleDownThreshold string `json:"scaleDownThreshold,omitempty"`
+
+	// ScaleUpFactor is the multiplicative factor applied to the current number of runners used
+	// to determine how many pods should be added.
+	// +optional
+	ScaleUpFactor string `json:"scaleUpFactor,omitempty"`
+
+	// ScaleDownFactor is the multiplicative factor applied to the current number of runners used
+	// to determine how many pods should be removed.
+	// +optional
+	ScaleDownFactor string `json:"scaleDownFactor,omitempty"`
+
+	// ScaleUpAdjustment is the number of runners added on scale-up.
+	// You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+	// +optional
+	ScaleUpAdjustment int `json:"scaleUpAdjustment,omitempty"`
+
+	// ScaleDownAdjustment is the number of runners removed on scale-down.
+	// You can only specify either ScaleDownFactor or ScaleDownAdjustment.
+	// +optional
+	ScaleDownAdjustment int `json:"scaleDownAdjustment,omitempty"`
+}
+
+type HorizontalRunnerAutoscalerStatus struct {
+	// ObservedGeneration is the most recent generation observed for the target. It corresponds to e.g.
+	// RunnerDeployment's generation, which is updated on mutation by the API Server.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// DesiredReplicas is the total number of desired, non-terminated and latest pods to be set for the primary RunnerSet
+	// This doesn't include outdated pods while upgrading the deployment and replacing the runnerset.
+	// +optional
+	DesiredReplicas *int `json:"desiredReplicas,omitempty"`
+
+	// +optional
+	LastSuccessfulScaleOutTime *metav1.Time `json:"lastSuccessfulScaleOutTime,omitempty"`
+
+	// +optional
+	CacheEntries []CacheEntry `json:"cacheEntries,omitempty"`
+}
+
+const CacheEntryKeyDesiredReplicas = "desiredReplicas"
+
+type CacheEntry struct {
+	Key            string      `json:"key,omitempty"`
+	Value          int         `json:"value,omitempty"`
+	ExpirationTime metav1.Time `json:"expirationTime,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:JSONPath=".spec.minReplicas",name=Min,type=number
+// +kubebuilder:printcolumn:JSONPath=".spec.maxReplicas",name=Max,type=number
+// +kubebuilder:printcolumn:JSONPath=".status.desiredReplicas",name=Desired,type=number
+
+// HorizontalRunnerAutoscaler is the Schema for the horizontalrunnerautoscaler API
+type HorizontalRunnerAutoscaler struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   HorizontalRunnerAutoscalerSpec   `json:"spec,omitempty"`
+	Status HorizontalRunnerAutoscalerStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// HorizontalRunnerAutoscalerList contains a list of HorizontalRunnerAutoscaler
+type HorizontalRunnerAutoscalerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []HorizontalRunnerAutoscaler `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&HorizontalRunnerAutoscaler{}, &HorizontalRunnerAutoscalerList{})
+}

api/v1alpha1/runner_types.go

@@ -17,19 +17,37 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"errors"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // RunnerSpec defines the desired state of Runner
 type RunnerSpec struct {
-	// +kubebuilder:validation:MinLength=3
+	// +optional
+	// +kubebuilder:validation:Pattern=`^[^/]+$`
+	Enterprise string `json:"enterprise,omitempty"`
+
+	// +optional
+	// +kubebuilder:validation:Pattern=`^[^/]+$`
+	Organization string `json:"organization,omitempty"`
+
+	// +optional
 	// +kubebuilder:validation:Pattern=`^[^/]+/[^/]+$`
-	Repository string `json:"repository"`
+	Repository string `json:"repository,omitempty"`
+
+	// +optional
+	Labels []string `json:"labels,omitempty"`
+
+	// +optional
+	Group string `json:"group,omitempty"`
 
 	// +optional
 	Containers []corev1.Container `json:"containers,omitempty"`
 	// +optional
+	DockerdContainerResources corev1.ResourceRequirements `json:"dockerdContainerResources,omitempty"`
+	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 	// +optional
 	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
@@ -39,10 +57,14 @@ type RunnerSpec struct {
 	// +optional
 	Image string `json:"image"`
 	// +optional
+	ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
+	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
 
 	// +optional
 	Volumes []corev1.Volume `json:"volumes,omitempty"`
+	// +optional
+	WorkDir string `json:"workDir,omitempty"`
 
 	// +optional
 	InitContainers []corev1.Container `json:"initContainers,omitempty"`
@@ -66,6 +88,33 @@ type RunnerSpec struct {
 	EphemeralContainers []corev1.EphemeralContainer `json:"ephemeralContainers,omitempty"`
 	// +optional
 	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
+	// +optional
+	DockerdWithinRunnerContainer *bool `json:"dockerdWithinRunnerContainer,omitempty"`
+	// +optional
+	DockerEnabled *bool `json:"dockerEnabled,omitempty"`
+}
+
+// ValidateRepository validates repository field.
+func (rs *RunnerSpec) ValidateRepository() error {
+	// Enterprise, Organization and repository are both exclusive.
+	foundCount := 0
+	if len(rs.Organization) > 0 {
+		foundCount += 1
+	}
+	if len(rs.Repository) > 0 {
+		foundCount += 1
+	}
+	if len(rs.Enterprise) > 0 {
+		foundCount += 1
+	}
+	if foundCount == 0 {
+		return errors.New("Spec needs enterprise, organization or repository")
+	}
+	if foundCount > 1 {
+		return errors.New("Spec cannot have many fields defined enterprise, organization and repository")
+	}
+
+	return nil
 }
 
 // RunnerStatus defines the observed state of Runner
@@ -76,15 +125,22 @@ type RunnerStatus struct {
 	Message      string                   `json:"message"`
 }
 
+// RunnerStatusRegistration contains runner registration status
 type RunnerStatusRegistration struct {
-	Repository string      `json:"repository"`
-	Token      string      `json:"token"`
-	ExpiresAt  metav1.Time `json:"expiresAt"`
+	Enterprise   string      `json:"enterprise,omitempty"`
+	Organization string      `json:"organization,omitempty"`
+	Repository   string      `json:"repository,omitempty"`
+	Labels       []string    `json:"labels,omitempty"`
+	Token        string      `json:"token"`
+	ExpiresAt    metav1.Time `json:"expiresAt"`
 }
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:JSONPath=".spec.enterprise",name=Enterprise,type=string
+// +kubebuilder:printcolumn:JSONPath=".spec.organization",name=Organization,type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.repository",name=Repository,type=string
+// +kubebuilder:printcolumn:JSONPath=".spec.labels",name=Labels,type=string
 // +kubebuilder:printcolumn:JSONPath=".status.phase",name=Status,type=string
 
 // Runner is the Schema for the runners API

api/v1alpha1/runner_webhook.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+// log is for logging in this package.
+var runnerLog = logf.Log.WithName("runner-resource")
+
+func (r *Runner) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(r).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runner,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runners,versions=v1alpha1,name=mutate.runner.actions.summerwind.dev
+
+var _ webhook.Defaulter = &Runner{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (r *Runner) Default() {
+	// Nothing to do.
+}
+
+// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runner,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runners,versions=v1alpha1,name=validate.runner.actions.summerwind.dev
+
+var _ webhook.Validator = &Runner{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
+func (r *Runner) ValidateCreate() error {
+	runnerLog.Info("validate resource to be created", "name", r.Name)
+	return r.Validate()
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
+func (r *Runner) ValidateUpdate(old runtime.Object) error {
+	runnerLog.Info("validate resource to be updated", "name", r.Name)
+	return r.Validate()
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
+func (r *Runner) ValidateDelete() error {
+	return nil
+}
+
+// Validate validates resource spec.
+func (r *Runner) Validate() error {
+	var (
+		errList field.ErrorList
+		err     error
+	)
+
+	err = r.Spec.ValidateRepository()
+	if err != nil {
+		errList = append(errList, field.Invalid(field.NewPath("spec", "repository"), r.Spec.Repository, err.Error()))
+	}
+
+	if len(errList) > 0 {
+		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)
+	}
+
+	return nil
+}

api/v1alpha1/runnerdeployment_types.go

@@ -20,9 +20,16 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns = "TotalNumberOfQueuedAndInProgressWorkflowRuns"
+	AutoscalingMetricTypePercentageRunnersBusy                        = "PercentageRunnersBusy"
+)
+
 // RunnerReplicaSetSpec defines the desired state of RunnerDeployment
 type RunnerDeploymentSpec struct {
-	Replicas *int `json:"replicas"`
+	// +optional
+	// +nullable
+	Replicas *int `json:"replicas,omitempty"`
 
 	Template RunnerTemplate `json:"template"`
 }
@@ -30,6 +37,11 @@ type RunnerDeploymentSpec struct {
 type RunnerDeploymentStatus struct {
 	AvailableReplicas int `json:"availableReplicas"`
 	ReadyReplicas     int `json:"readyReplicas"`
+
+	// Replicas is the total number of desired, non-terminated and latest pods to be set for the primary RunnerSet
+	// This doesn't include outdated pods while upgrading the deployment and replacing the runnerset.
+	// +optional
+	Replicas *int `json:"desiredReplicas,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1alpha1/runnerdeployment_webhook.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+// log is for logging in this package.
+var runenrDeploymentLog = logf.Log.WithName("runnerdeployment-resource")
+
+func (r *RunnerDeployment) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(r).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runnerdeployment,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerdeployments,versions=v1alpha1,name=mutate.runnerdeployment.actions.summerwind.dev
+
+var _ webhook.Defaulter = &RunnerDeployment{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (r *RunnerDeployment) Default() {
+	// Nothing to do.
+}
+
+// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runnerdeployment,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerdeployments,versions=v1alpha1,name=validate.runnerdeployment.actions.summerwind.dev
+
+var _ webhook.Validator = &RunnerDeployment{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
+func (r *RunnerDeployment) ValidateCreate() error {
+	runenrDeploymentLog.Info("validate resource to be created", "name", r.Name)
+	return r.Validate()
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
+func (r *RunnerDeployment) ValidateUpdate(old runtime.Object) error {
+	runenrDeploymentLog.Info("validate resource to be updated", "name", r.Name)
+	return r.Validate()
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
+func (r *RunnerDeployment) ValidateDelete() error {
+	return nil
+}
+
+// Validate validates resource spec.
+func (r *RunnerDeployment) Validate() error {
+	var (
+		errList field.ErrorList
+		err     error
+	)
+
+	err = r.Spec.Template.Spec.ValidateRepository()
+	if err != nil {
+		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
+	}
+
+	if len(errList) > 0 {
+		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)
+	}
+
+	return nil
+}

api/v1alpha1/runnerreplicaset_types.go

@@ -22,7 +22,9 @@ import (
 
 // RunnerReplicaSetSpec defines the desired state of RunnerReplicaSet
 type RunnerReplicaSetSpec struct {
-	Replicas *int `json:"replicas"`
+	// +optional
+	// +nullable
+	Replicas *int `json:"replicas,omitempty"`
 
 	Template RunnerTemplate `json:"template"`
 }

api/v1alpha1/runnerreplicaset_webhook.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+// log is for logging in this package.
+var runnerReplicaSetLog = logf.Log.WithName("runnerreplicaset-resource")
+
+func (r *RunnerReplicaSet) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(r).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runnerreplicaset,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerreplicasets,versions=v1alpha1,name=mutate.runnerreplicaset.actions.summerwind.dev
+
+var _ webhook.Defaulter = &RunnerReplicaSet{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (r *RunnerReplicaSet) Default() {
+	// Nothing to do.
+}
+
+// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runnerreplicaset,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerreplicasets,versions=v1alpha1,name=validate.runnerreplicaset.actions.summerwind.dev
+
+var _ webhook.Validator = &RunnerReplicaSet{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
+func (r *RunnerReplicaSet) ValidateCreate() error {
+	runnerReplicaSetLog.Info("validate resource to be created", "name", r.Name)
+	return r.Validate()
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
+func (r *RunnerReplicaSet) ValidateUpdate(old runtime.Object) error {
+	runnerReplicaSetLog.Info("validate resource to be updated", "name", r.Name)
+	return r.Validate()
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
+func (r *RunnerReplicaSet) ValidateDelete() error {
+	return nil
+}
+
+// Validate validates resource spec.
+func (r *RunnerReplicaSet) Validate() error {
+	var (
+		errList field.ErrorList
+		err     error
+	)
+
+	err = r.Spec.Template.Spec.ValidateRepository()
+	if err != nil {
+		errList = append(errList, field.Invalid(field.NewPath("spec", "template", "spec", "repository"), r.Spec.Template.Spec.Repository, err.Error()))
+	}
+
+	if len(errList) > 0 {
+		return apierrors.NewInvalid(r.GroupVersionKind().GroupKind(), r.Name, errList)
+	}
+
+	return nil
+}

api/v1alpha1/zz_generated.deepcopy.go

@@ -22,9 +22,293 @@ package v1alpha1
 
 import (
 	"k8s.io/api/core/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CacheEntry) DeepCopyInto(out *CacheEntry) {
+	*out = *in
+	in.ExpirationTime.DeepCopyInto(&out.ExpirationTime)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheEntry.
+func (in *CacheEntry) DeepCopy() *CacheEntry {
+	if in == nil {
+		return nil
+	}
+	out := new(CacheEntry)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CapacityReservation) DeepCopyInto(out *CapacityReservation) {
+	*out = *in
+	in.ExpirationTime.DeepCopyInto(&out.ExpirationTime)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservation.
+func (in *CapacityReservation) DeepCopy() *CapacityReservation {
+	if in == nil {
+		return nil
+	}
+	out := new(CapacityReservation)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CheckRunSpec) DeepCopyInto(out *CheckRunSpec) {
+	*out = *in
+	if in.Types != nil {
+		in, out := &in.Types, &out.Types
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CheckRunSpec.
+func (in *CheckRunSpec) DeepCopy() *CheckRunSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(CheckRunSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GitHubEventScaleUpTriggerSpec) DeepCopyInto(out *GitHubEventScaleUpTriggerSpec) {
+	*out = *in
+	if in.CheckRun != nil {
+		in, out := &in.CheckRun, &out.CheckRun
+		*out = new(CheckRunSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.PullRequest != nil {
+		in, out := &in.PullRequest, &out.PullRequest
+		*out = new(PullRequestSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Push != nil {
+		in, out := &in.Push, &out.Push
+		*out = new(PushSpec)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitHubEventScaleUpTriggerSpec.
+func (in *GitHubEventScaleUpTriggerSpec) DeepCopy() *GitHubEventScaleUpTriggerSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(GitHubEventScaleUpTriggerSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscaler) DeepCopyInto(out *HorizontalRunnerAutoscaler) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscaler.
+func (in *HorizontalRunnerAutoscaler) DeepCopy() *HorizontalRunnerAutoscaler {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscaler)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *HorizontalRunnerAutoscaler) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscalerList) DeepCopyInto(out *HorizontalRunnerAutoscalerList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]HorizontalRunnerAutoscaler, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerList.
+func (in *HorizontalRunnerAutoscalerList) DeepCopy() *HorizontalRunnerAutoscalerList {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscalerList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *HorizontalRunnerAutoscalerList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAutoscalerSpec) {
+	*out = *in
+	out.ScaleTargetRef = in.ScaleTargetRef
+	if in.MinReplicas != nil {
+		in, out := &in.MinReplicas, &out.MinReplicas
+		*out = new(int)
+		**out = **in
+	}
+	if in.MaxReplicas != nil {
+		in, out := &in.MaxReplicas, &out.MaxReplicas
+		*out = new(int)
+		**out = **in
+	}
+	if in.ScaleDownDelaySecondsAfterScaleUp != nil {
+		in, out := &in.ScaleDownDelaySecondsAfterScaleUp, &out.ScaleDownDelaySecondsAfterScaleUp
+		*out = new(int)
+		**out = **in
+	}
+	if in.Metrics != nil {
+		in, out := &in.Metrics, &out.Metrics
+		*out = make([]MetricSpec, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.ScaleUpTriggers != nil {
+		in, out := &in.ScaleUpTriggers, &out.ScaleUpTriggers
+		*out = make([]ScaleUpTrigger, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.CapacityReservations != nil {
+		in, out := &in.CapacityReservations, &out.CapacityReservations
+		*out = make([]CapacityReservation, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
+func (in *HorizontalRunnerAutoscalerSpec) DeepCopy() *HorizontalRunnerAutoscalerSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscalerSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscalerStatus) DeepCopyInto(out *HorizontalRunnerAutoscalerStatus) {
+	*out = *in
+	if in.DesiredReplicas != nil {
+		in, out := &in.DesiredReplicas, &out.DesiredReplicas
+		*out = new(int)
+		**out = **in
+	}
+	if in.LastSuccessfulScaleOutTime != nil {
+		in, out := &in.LastSuccessfulScaleOutTime, &out.LastSuccessfulScaleOutTime
+		*out = (*in).DeepCopy()
+	}
+	if in.CacheEntries != nil {
+		in, out := &in.CacheEntries, &out.CacheEntries
+		*out = make([]CacheEntry, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerStatus.
+func (in *HorizontalRunnerAutoscalerStatus) DeepCopy() *HorizontalRunnerAutoscalerStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscalerStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricSpec) DeepCopyInto(out *MetricSpec) {
+	*out = *in
+	if in.RepositoryNames != nil {
+		in, out := &in.RepositoryNames, &out.RepositoryNames
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricSpec.
+func (in *MetricSpec) DeepCopy() *MetricSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PullRequestSpec) DeepCopyInto(out *PullRequestSpec) {
+	*out = *in
+	if in.Types != nil {
+		in, out := &in.Types, &out.Types
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Branches != nil {
+		in, out := &in.Branches, &out.Branches
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PullRequestSpec.
+func (in *PullRequestSpec) DeepCopy() *PullRequestSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PullRequestSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PushSpec) DeepCopyInto(out *PushSpec) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushSpec.
+func (in *PushSpec) DeepCopy() *PushSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PushSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Runner) DeepCopyInto(out *Runner) {
 	*out = *in
@@ -58,7 +342,7 @@ func (in *RunnerDeployment) DeepCopyInto(out *RunnerDeployment) {
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	in.Spec.DeepCopyInto(&out.Spec)
-	out.Status = in.Status
+	in.Status.DeepCopyInto(&out.Status)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerDeployment.
@@ -135,6 +419,11 @@ func (in *RunnerDeploymentSpec) DeepCopy() *RunnerDeploymentSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RunnerDeploymentStatus) DeepCopyInto(out *RunnerDeploymentStatus) {
 	*out = *in
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerDeploymentStatus.
@@ -277,6 +566,11 @@ func (in *RunnerReplicaSetStatus) DeepCopy() *RunnerReplicaSetStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.Containers != nil {
 		in, out := &in.Containers, &out.Containers
 		*out = make([]v1.Container, len(*in))
@@ -284,6 +578,7 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.DockerdContainerResources.DeepCopyInto(&out.DockerdContainerResources)
 	in.Resources.DeepCopyInto(&out.Resources)
 	if in.VolumeMounts != nil {
 		in, out := &in.VolumeMounts, &out.VolumeMounts
@@ -373,6 +668,16 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 		*out = new(int64)
 		**out = **in
 	}
+	if in.DockerdWithinRunnerContainer != nil {
+		in, out := &in.DockerdWithinRunnerContainer, &out.DockerdWithinRunnerContainer
+		*out = new(bool)
+		**out = **in
+	}
+	if in.DockerEnabled != nil {
+		in, out := &in.DockerEnabled, &out.DockerEnabled
+		*out = new(bool)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSpec.
@@ -404,6 +709,11 @@ func (in *RunnerStatus) DeepCopy() *RunnerStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RunnerStatusRegistration) DeepCopyInto(out *RunnerStatusRegistration) {
 	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	in.ExpiresAt.DeepCopyInto(&out.ExpiresAt)
 }
 
@@ -433,3 +743,39 @@ func (in *RunnerTemplate) DeepCopy() *RunnerTemplate {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ScaleTargetRef) DeepCopyInto(out *ScaleTargetRef) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleTargetRef.
+func (in *ScaleTargetRef) DeepCopy() *ScaleTargetRef {
+	if in == nil {
+		return nil
+	}
+	out := new(ScaleTargetRef)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ScaleUpTrigger) DeepCopyInto(out *ScaleUpTrigger) {
+	*out = *in
+	if in.GitHubEvent != nil {
+		in, out := &in.GitHubEvent, &out.GitHubEvent
+		*out = new(GitHubEventScaleUpTriggerSpec)
+		(*in).DeepCopyInto(*out)
+	}
+	out.Duration = in.Duration
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleUpTrigger.
+func (in *ScaleUpTrigger) DeepCopy() *ScaleUpTrigger {
+	if in == nil {
+		return nil
+	}
+	out := new(ScaleUpTrigger)
+	in.DeepCopyInto(out)
+	return out
+}

charts/.ci/ct-config.yaml

@@ -0,0 +1,4 @@
+# This file defines the config for "ct" (chart tester) used by the helm linting GitHub workflow
+lint-conf: charts/.ci/lint-config.yaml
+chart-repos:
+  - jetstack=https://charts.jetstack.io

charts/.ci/lint-config.yaml

@@ -0,0 +1,6 @@
+rules:
+  # One blank line is OK
+  empty-lines: 
+    max-start: 1
+    max-end: 1
+    max: 1

charts/.ci/scripts/local-ct-lint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run --rm -it -w /repo -v $(pwd):/repo quay.io/helmpack/chart-testing ct lint --all --config charts/.ci/ct-config.yaml

charts/.ci/scripts/local-kube-score.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+
+for chart in `ls charts`;
+do
+helm template --values charts/$chart/ci/ci-values.yaml charts/$chart | kube-score score - \
+    --ignore-test pod-networkpolicy \
+    --ignore-test deployment-has-poddisruptionbudget \
+    --ignore-test deployment-has-host-podantiaffinity \
+    --ignore-test pod-probes \
+    --ignore-test container-image-tag \
+    --enable-optional-test container-security-context-privileged \
+    --enable-optional-test container-security-context-readonlyrootfilesystem \
+    --ignore-test container-security-context
+done

charts/actions-runner-controller/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/actions-runner-controller/Chart.yaml

@@ -0,0 +1,31 @@
+apiVersion: v2
+name: actions-runner-controller
+description: A Kubernetes controller that operates self-hosted runners for GitHub Actions on your Kubernetes cluster.
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.5.0
+
+home: https://github.com/summerwind/actions-runner-controller
+
+sources:
+    - https://github.com/summerwind/actions-runner-controller
+
+maintainers:
+    - name: summerwind
+      email: contact@summerwind.jp
+      url: https://github.com/summerwind
+    - name: funkypenguin
+      email: davidy@funkypenguin.co.nz
+      url: https://www.funkypenguin.co.nz

charts/actions-runner-controller/ci/ci-values.yaml

@@ -0,0 +1,27 @@
+# This file sets some opinionated values for kube-score to use
+# when parsing the chart
+image:
+  pullPolicy: Always
+
+podSecurityContext:
+  fsGroup: 2000
+
+securityContext: 
+  capabilities:
+    drop:
+    - ALL
+  readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 2000
+
+resources: 
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 100m
+    memory: 128Mi  
+
+# Set the following to true to create a dummy secret, allowing the manager pod to start
+# This is only useful in CI
+createDummySecret: true

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -0,0 +1,218 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.3.0
+  creationTimestamp: null
+  name: horizontalrunnerautoscalers.actions.summerwind.dev
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .spec.minReplicas
+    name: Min
+    type: number
+  - JSONPath: .spec.maxReplicas
+    name: Max
+    type: number
+  - JSONPath: .status.desiredReplicas
+    name: Desired
+    type: number
+  group: actions.summerwind.dev
+  names:
+    kind: HorizontalRunnerAutoscaler
+    listKind: HorizontalRunnerAutoscalerList
+    plural: horizontalrunnerautoscalers
+    singular: horizontalrunnerautoscaler
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: HorizontalRunnerAutoscaler is the Schema for the horizontalrunnerautoscaler
+        API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: HorizontalRunnerAutoscalerSpec defines the desired state of
+            HorizontalRunnerAutoscaler
+          properties:
+            capacityReservations:
+              items:
+                description: CapacityReservation specifies the number of replicas
+                  temporarily added to the scale target until ExpirationTime.
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  name:
+                    type: string
+                  replicas:
+                    type: integer
+                type: object
+              type: array
+            maxReplicas:
+              description: MinReplicas is the maximum number of replicas the deployment
+                is allowed to scale
+              type: integer
+            metrics:
+              description: Metrics is the collection of various metric targets to
+                calculate desired number of runners
+              items:
+                properties:
+                  repositoryNames:
+                    description: RepositoryNames is the list of repository names to
+                      be used for calculating the metric. For example, a repository
+                      name is the REPO part of `github.com/USER/REPO`.
+                    items:
+                      type: string
+                    type: array
+                  scaleDownAdjustment:
+                    description: ScaleDownAdjustment is the number of runners removed
+                      on scale-down. You can only specify either ScaleDownFactor or
+                      ScaleDownAdjustment.
+                    type: integer
+                  scaleDownFactor:
+                    description: ScaleDownFactor is the multiplicative factor applied
+                      to the current number of runners used to determine how many
+                      pods should be removed.
+                    type: string
+                  scaleDownThreshold:
+                    description: ScaleDownThreshold is the percentage of busy runners
+                      less than which will trigger the hpa to scale the runners down.
+                    type: string
+                  scaleUpAdjustment:
+                    description: ScaleUpAdjustment is the number of runners added
+                      on scale-up. You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+                    type: integer
+                  scaleUpFactor:
+                    description: ScaleUpFactor is the multiplicative factor applied
+                      to the current number of runners used to determine how many
+                      pods should be added.
+                    type: string
+                  scaleUpThreshold:
+                    description: ScaleUpThreshold is the percentage of busy runners
+                      greater than which will trigger the hpa to scale runners up.
+                    type: string
+                  type:
+                    description: Type is the type of metric to be used for autoscaling.
+                      The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                    type: string
+                type: object
+              type: array
+            minReplicas:
+              description: MinReplicas is the minimum number of replicas the deployment
+                is allowed to scale
+              type: integer
+            scaleDownDelaySecondsAfterScaleOut:
+              description: ScaleDownDelaySecondsAfterScaleUp is the approximate delay
+                for a scale down followed by a scale up Used to prevent flapping (down->up->down->...
+                loop)
+              type: integer
+            scaleTargetRef:
+              description: ScaleTargetRef sis the reference to scaled resource like
+                RunnerDeployment
+              properties:
+                name:
+                  type: string
+              type: object
+            scaleUpTriggers:
+              description: "ScaleUpTriggers is an experimental feature to increase
+                the desired replicas by 1 on each webhook requested received by the
+                webhookBasedAutoscaler. \n This feature requires you to also enable
+                and deploy the webhookBasedAutoscaler onto your cluster. \n Note that
+                the added runners remain until the next sync period at least, and
+                they may or may not be used by GitHub Actions depending on the timing.
+                They are intended to be used to gain \"resource slack\" immediately
+                after you receive a webhook from GitHub, so that you can loosely expect
+                MinReplicas runners to be always available."
+              items:
+                properties:
+                  amount:
+                    type: integer
+                  duration:
+                    type: string
+                  githubEvent:
+                    properties:
+                      checkRun:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run
+                        properties:
+                          status:
+                            type: string
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      pullRequest:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+                        properties:
+                          branches:
+                            items:
+                              type: string
+                            type: array
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      push:
+                        description: PushSpec is the condition for triggering scale-up
+                          on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
+                        type: object
+                    type: object
+                type: object
+              type: array
+          type: object
+        status:
+          properties:
+            cacheEntries:
+              items:
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  key:
+                    type: string
+                  value:
+                    type: integer
+                type: object
+              type: array
+            desiredReplicas:
+              description: DesiredReplicas is the total number of desired, non-terminated
+                and latest pods to be set for the primary RunnerSet This doesn't include
+                outdated pods while upgrading the deployment and replacing the runnerset.
+              type: integer
+            lastSuccessfulScaleOutTime:
+              format: date-time
+              type: string
+            observedGeneration:
+              description: ObservedGeneration is the most recent generation observed
+                for the target. It corresponds to e.g. RunnerDeployment's generation,
+                which is updated on mutation by the API Server.
+              format: int64
+              type: integer
+          type: object
+      type: object
+  version: v1alpha1
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

charts/actions-runner-controller/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "actions-runner-controller.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "actions-runner-controller.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "actions-runner-controller.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "actions-runner-controller.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

charts/actions-runner-controller/templates/_github_webhook_server_helpers.tpl

@@ -0,0 +1,52 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.name" -}}
+{{- default .Chart.Name .Values.githubWebhookServer.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "actions-runner-controller-github-webhook-server.instance" -}}
+{{- printf "%s-%s" .Release.Name "github-webhook-server" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.fullname" -}}
+{{- if .Values.githubWebhookServer.fullnameOverride }}
+{{- .Values.githubWebhookServer.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.githubWebhookServer.nameOverride }}
+{{- $instance := include "actions-runner-controller-github-webhook-server.instance" . }}
+{{- if contains $name $instance }}
+{{- $instance | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s-%s" .Release.Name $name "github-webhook-server" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "actions-runner-controller-github-webhook-server.name" . }}
+app.kubernetes.io/instance: {{ include "actions-runner-controller-github-webhook-server.instance" . }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "actions-runner-controller-github-webhook-server.serviceAccountName" -}}
+{{- if .Values.githubWebhookServer.serviceAccount.create }}
+{{- default (include "actions-runner-controller-github-webhook-server.fullname" .) .Values.githubWebhookServer.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.githubWebhookServer.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{- define "actions-runner-controller-github-webhook-server.roleName" -}}
+{{- include "actions-runner-controller-github-webhook-server.fullname" . }}
+{{- end }}

charts/actions-runner-controller/templates/_helpers.tpl

@@ -0,0 +1,101 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "actions-runner-controller.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "actions-runner-controller.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "actions-runner-controller.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "actions-runner-controller.labels" -}}
+helm.sh/chart: {{ include "actions-runner-controller.chart" . }}
+{{ include "actions-runner-controller.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- range $k, $v := .Values.labels }}
+{{ $k }}: {{ $v }}
+{{- end }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "actions-runner-controller.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "actions-runner-controller.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "actions-runner-controller.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "actions-runner-controller.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{- define "actions-runner-controller.leaderElectionRoleName" -}}
+{{- include "actions-runner-controller.fullname" . }}-leader-election
+{{- end }}
+
+{{- define "actions-runner-controller.authProxyRoleName" -}}
+{{- include "actions-runner-controller.fullname" . }}-proxy
+{{- end }}
+
+{{- define "actions-runner-controller.managerRoleName" -}}
+{{- include "actions-runner-controller.fullname" . }}-manager
+{{- end }}
+
+{{- define "actions-runner-controller.runnerEditorRoleName" -}}
+{{- include "actions-runner-controller.fullname" . }}-runner-editor
+{{- end }}
+
+{{- define "actions-runner-controller.runnerViewerRoleName" -}}
+{{- include "actions-runner-controller.fullname" . }}-runner-viewer
+{{- end }}
+
+{{- define "actions-runner-controller.webhookServiceName" -}}
+{{- include "actions-runner-controller.fullname" . }}-webhook
+{{- end }}
+
+{{- define "actions-runner-controller.authProxyServiceName" -}}
+{{- include "actions-runner-controller.fullname" . }}-metrics-service
+{{- end }}
+
+{{- define "actions-runner-controller.selfsignedIssuerName" -}}
+{{- include "actions-runner-controller.fullname" . }}-selfsigned-issuer
+{{- end }}
+
+{{- define "actions-runner-controller.servingCertName" -}}
+{{- include "actions-runner-controller.fullname" . }}-serving-cert
+{{- end }}

charts/actions-runner-controller/templates/auth_proxy_role.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "actions-runner-controller.authProxyRoleName" . }}
+rules:
+- apiGroups: ["authentication.k8s.io"]
+  resources:
+  - tokenreviews
+  verbs: ["create"]
+- apiGroups: ["authorization.k8s.io"]
+  resources:
+  - subjectaccessreviews
+  verbs: ["create"]

charts/actions-runner-controller/templates/auth_proxy_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "actions-runner-controller.authProxyRoleName" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "actions-runner-controller.authProxyRoleName" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "actions-runner-controller.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}

charts/actions-runner-controller/templates/auth_proxy_service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  name: {{ include "actions-runner-controller.authProxyServiceName" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+  - name: https
+    port: 8443
+    targetPort: https
+  selector:
+    {{- include "actions-runner-controller.selectorLabels" . | nindent 4 }}

charts/actions-runner-controller/templates/certificate.yaml

@@ -0,0 +1,24 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "actions-runner-controller.selfsignedIssuerName" . }}
+  namespace: {{ .Namespace }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "actions-runner-controller.servingCertName" . }}
+  namespace: {{ .Namespace }}
+spec:
+  dnsNames:
+  - {{ include "actions-runner-controller.webhookServiceName" . }}.{{ .Release.Namespace }}.svc
+  - {{ include "actions-runner-controller.webhookServiceName" . }}.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: {{ include "actions-runner-controller.selfsignedIssuerName" . }}
+  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

charts/actions-runner-controller/templates/ci-secret.yaml

@@ -0,0 +1,10 @@
+# This template only exists to facilitate CI testing of the chart, since
+# a secret is expected to be found in the namespace by the controller manager
+{{ if .Values.createDummySecret -}}
+apiVersion: v1
+data:
+  github_token: dGVzdA==
+kind: Secret
+metadata:
+  name: controller-manager
+{{- end }}

charts/actions-runner-controller/templates/deployment.yaml

@@ -0,0 +1,121 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "actions-runner-controller.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "actions-runner-controller.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "actions-runner-controller.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "actions-runner-controller.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.priorityClassName }}
+      priorityClassName: "{{ . }}"
+      {{- end }}
+      containers:
+      - args:
+        - "--metrics-addr=127.0.0.1:8080"
+        - "--enable-leader-election"
+        - "--sync-period={{ .Values.syncPeriod }}"
+        - "--docker-image={{ .Values.image.dindSidecarRepositoryAndTag }}"
+        command:
+        - "/manager"
+        env:
+        - name: GITHUB_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: github_token
+              name: controller-manager
+              optional: true
+        - name: GITHUB_APP_ID
+          valueFrom:
+            secretKeyRef:
+              key: github_app_id
+              name: controller-manager
+              optional: true
+        - name: GITHUB_APP_INSTALLATION_ID
+          valueFrom:
+            secretKeyRef:
+              key: github_app_installation_id
+              name: controller-manager
+              optional: true
+        - name: GITHUB_APP_PRIVATE_KEY
+          value: /etc/actions-runner-controller/github_app_private_key
+        {{- range $key, $val := .Values.env }}
+        - name: {{ $key }}
+          value: {{ $val | quote }}
+        {{- end }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
+        name: manager
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}          
+        volumeMounts:
+        - mountPath: "/etc/actions-runner-controller"
+          name: controller-manager
+          readOnly: true
+        - mountPath: /tmp
+          name: tmp         
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      - args:
+        - "--secure-listen-address=0.0.0.0:8443"
+        - "--upstream=http://127.0.0.1:8080/"
+        - "--logtostderr=true"
+        - "--v=10"
+        image: "{{ .Values.kube_rbac_proxy.image.repository }}:{{ .Values.kube_rbac_proxy.image.tag }}"
+        name: kube-rbac-proxy
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: 8443
+          name: https
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}   
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}     
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: controller-manager
+        secret:
+          secretName: controller-manager
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: webhook-server-cert
+      - name: tmp
+        emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -0,0 +1,93 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.githubWebhookServer.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.githubWebhookServer.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.githubWebhookServer.podSecurityContext | nindent 8 }}
+      {{- with .Values.githubWebhookServer.priorityClassName }}
+      priorityClassName: "{{ . }}"
+      {{- end }}
+      containers:
+      - args:
+        - "--metrics-addr=127.0.0.1:8080"
+        - "--enable-leader-election"
+        - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}"
+        command:
+        - "/github-webhook-server"
+        env:
+        - name: GITHUB_WEBHOOK_SECRET_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: github_webhook_secret_token
+              name: github-webhook-server
+              optional: true
+        {{- range $key, $val := .Values.githubWebhookServer.env }}
+        - name: {{ $key }}
+          value: {{ $val | quote }}
+        {{- end }}
+        image: "{{ .Values.githubWebhookServer.image.repository }}:{{ .Values.githubWebhookServer.image.tag | default (cat "v" .Chart.AppVersion | replace " " "") }}"
+        name: github-webhook-server
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: 8000
+          name: github-webhook-server
+          protocol: TCP
+        resources:
+          {{- toYaml .Values.githubWebhookServer.resources | nindent 12 }}
+        securityContext:
+          {{- toYaml .Values.githubWebhookServer.securityContext | nindent 12 }}
+      - args:
+        - "--secure-listen-address=0.0.0.0:8443"
+        - "--upstream=http://127.0.0.1:8080/"
+        - "--logtostderr=true"
+        - "--v=10"
+        image: "{{ .Values.kube_rbac_proxy.image.repository }}:{{ .Values.kube_rbac_proxy.image.tag }}"
+        name: kube-rbac-proxy
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - containerPort: 8443
+          name: https
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}   
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}     
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: github-webhook-server
+        secret:
+          secretName: github-webhook-server
+      {{- with .Values.githubWebhookServer.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.githubWebhookServer.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.githubWebhookServer.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.role.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: {{ include "actions-runner-controller-github-webhook-server.roleName" . }}
+rules:
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/status
+  verbs:
+  - get
+  - patch
+  - update
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.secrets.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.githubWebhookServer.enabled }}
+{{- if .Values.githubWebhookServer.secret.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: github-webhook-server
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+type: Opaque
+data:
+{{- range $k, $v := .Values.githubWebhookServer.secret }}
+  {{ $k }}: {{ $v | toString | b64enc }}
+{{- end }} 
+{{- end }}
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.githubWebhookServer.service.type }}
+  ports:
+    {{ range $_, $port := .Values.githubWebhookServer.service.ports -}}
+    - {{ $port | toYaml | nindent 6 }}
+    {{- end }}
+  selector:
+    {{- include "actions-runner-controller-github-webhook-server.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/actions-runner-controller/templates/githubwebhook.serviceaccount.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.githubWebhookServer.enabled -}}
+{{- if .Values.githubWebhookServer.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  {{- with .Values.githubWebhookServer.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
+{{- end }}

charts/actions-runner-controller/templates/leader_election_role.yaml

@@ -0,0 +1,33 @@
+# permissions to do leader election.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "actions-runner-controller.leaderElectionRoleName" . }}
+  namespace: {{ .Release.Namespace }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create

charts/actions-runner-controller/templates/leader_election_role_binding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "actions-runner-controller.leaderElectionRoleName" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "actions-runner-controller.leaderElectionRoleName" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "actions-runner-controller.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}

charts/actions-runner-controller/templates/manager_role.yaml

@@ -0,0 +1,165 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: {{ include "actions-runner-controller.managerRoleName" . }}
+rules:
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerreplicasets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerreplicasets/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerreplicasets/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

charts/actions-runner-controller/templates/manager_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "actions-runner-controller.managerRoleName" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "actions-runner-controller.managerRoleName" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "actions-runner-controller.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}

charts/actions-runner-controller/templates/manager_secrets.yaml

@@ -0,0 +1,14 @@
+{{- if or .Values.authSecret.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: controller-manager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+type: Opaque
+data:
+{{- range $k, $v := .Values.authSecret }}
+  {{ $k }}: {{ $v | toString | b64enc }}
+{{- end }} 
+{{- end }}

charts/actions-runner-controller/templates/runner_editor_role.yaml

@@ -0,0 +1,26 @@
+# permissions to do edit runners.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "actions-runner-controller.runnerEditorRoleName" . }}
+rules:
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/status
+  verbs:
+  - get
+  - patch
+  - update

charts/actions-runner-controller/templates/runner_viewer_role.yaml

@@ -0,0 +1,20 @@
+# permissions to do viewer runners.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "actions-runner-controller.runnerViewerRoleName" . }}
+rules:
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/status
+  verbs:
+  - get

charts/actions-runner-controller/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "actions-runner-controller.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/actions-runner-controller/templates/webhook_configs.yaml

@@ -0,0 +1,128 @@
+
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: {{ include "actions-runner-controller.fullname" . }}-mutating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "actions-runner-controller.servingCertName" . }}
+webhooks:
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "actions-runner-controller.webhookServiceName" . }}
+      namespace: {{ .Release.Namespace }}
+      path: /mutate-actions-summerwind-dev-v1alpha1-runner
+  failurePolicy: Fail
+  name: mutate.runner.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runners
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "actions-runner-controller.webhookServiceName" . }}
+      namespace: {{ .Release.Namespace }}
+      path: /mutate-actions-summerwind-dev-v1alpha1-runnerdeployment
+  failurePolicy: Fail
+  name: mutate.runnerdeployment.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerdeployments
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "actions-runner-controller.webhookServiceName" . }}
+      namespace: {{ .Release.Namespace }}
+      path: /mutate-actions-summerwind-dev-v1alpha1-runnerreplicaset
+  failurePolicy: Fail
+  name: mutate.runnerreplicaset.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerreplicasets
+
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: {{ include "actions-runner-controller.fullname" . }}-validating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "actions-runner-controller.servingCertName" . }}
+webhooks:
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "actions-runner-controller.webhookServiceName" . }}
+      namespace: {{ .Release.Namespace }}
+      path: /validate-actions-summerwind-dev-v1alpha1-runner
+  failurePolicy: Fail
+  name: validate.runner.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runners
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "actions-runner-controller.webhookServiceName" . }}
+      namespace: {{ .Release.Namespace }}
+      path: /validate-actions-summerwind-dev-v1alpha1-runnerdeployment
+  failurePolicy: Fail
+  name: validate.runnerdeployment.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerdeployments
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: {{ include "actions-runner-controller.webhookServiceName" . }}
+      namespace: {{ .Release.Namespace }}
+      path: /validate-actions-summerwind-dev-v1alpha1-runnerreplicaset
+  failurePolicy: Fail
+  name: validate.runnerreplicaset.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerreplicasets

charts/actions-runner-controller/templates/webhook_service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "actions-runner-controller.webhookServiceName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "actions-runner-controller.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: 443
+      targetPort: 9443
+      protocol: TCP
+      name: https
+  selector:
+    {{- include "actions-runner-controller.selectorLabels" . | nindent 4 }}

charts/actions-runner-controller/values.yaml

@@ -0,0 +1,153 @@
+# Default values for actions-runner-controller.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+labels: {}
+
+replicaCount: 1
+
+syncPeriod: 10m
+
+# Only 1 authentication method can be deployed at a time
+# Uncomment the configuration you are applying and fill in the details
+authSecret:
+  enabled: false
+  ### GitHub Apps Configuration
+  #github_app_id: ""
+  #github_app_installation_id: ""
+  #github_app_private_key: |
+  ### GitHub PAT Configuration
+  #github_token: ""
+
+image:
+  repository: summerwind/actions-runner-controller
+  # Overrides the manager image tag whose default is the chart appVersion if the tag key is commented out
+  tag: "latest"
+  dindSidecarRepositoryAndTag: "docker:dind"
+  pullPolicy: IfNotPresent
+
+kube_rbac_proxy:
+  image:
+    repository: quay.io/brancz/kube-rbac-proxy
+    tag: v0.8.0
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 443
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths: []
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+# Leverage a PriorityClass to ensure your pods survive resource shortages
+# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+# PriorityClass: system-cluster-critical
+priorityClassName: ""
+
+env: {}
+  # http_proxy: "proxy.com:8080"
+  # https_proxy: "proxy.com:8080"
+  # no_proxy: ""
+
+githubWebhookServer:
+  enabled: false
+  labels: {}
+  replicaCount: 1
+  syncPeriod: 10m
+  secret:
+    enabled: false
+    ### GitHub Webhook Configuration
+    #github_webhook_secret_token: ""
+  image:
+    repository: summerwind/actions-runner-controller
+    # Overrides the manager image tag whose default is the chart appVersion if the tag key is commented out
+    tag: "latest"
+    pullPolicy: IfNotPresent
+  imagePullSecrets: []
+  nameOverride: ""
+  fullnameOverride: ""
+  serviceAccount:
+    # Specifies whether a service account should be created
+    create: true
+    # Annotations to add to the service account
+    annotations: {}
+    # The name of the service account to use.
+    # If not set and create is true, a name is generated using the fullname template
+    name: ""
+  podAnnotations: {}
+  podSecurityContext: {}
+  # fsGroup: 2000
+  securityContext: {}
+  resources: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  priorityClassName: ""
+  service:
+    type: NodePort
+    ports:
+    - port: 80
+      targetPort: 8000
+      protocol: TCP
+      name: http
+      #nodePort: someFixedPortForUseWithTerraformCdkCfnEtc

cmd/githubwebhookserver/main.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2021 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/controllers"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/exec"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	// +kubebuilder:scaffold:imports
+)
+
+var (
+	scheme   = runtime.NewScheme()
+	setupLog = ctrl.Log.WithName("setup")
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(scheme)
+
+	_ = actionsv1alpha1.AddToScheme(scheme)
+	// +kubebuilder:scaffold:scheme
+}
+
+func main() {
+	var (
+		err error
+
+		webhookAddr string
+		metricsAddr string
+
+		// The secret token of the GitHub Webhook. See https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks
+		webhookSecretToken string
+
+		watchNamespace string
+
+		enableLeaderElection bool
+		syncPeriod           time.Duration
+	)
+
+	webhookSecretToken = os.Getenv("GITHUB_WEBHOOK_SECRET_TOKEN")
+
+	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&watchNamespace, "watch-namespace", "", "The namespace to watch for HorizontalRunnerAutoscaler's to scale on Webhook. Set to empty for letting it watch for all namespaces.")
+	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
+		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
+	flag.Parse()
+
+	if webhookSecretToken == "" {
+		setupLog.Info("-webhook-secret-token is missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks")
+	}
+
+	if watchNamespace == "" {
+		setupLog.Info("-watch-namespace is empty. HorizontalRunnerAutoscalers in all the namespaces are watched, cached, and considered as scale targets.")
+	} else {
+		setupLog.Info("-watch-namespace is %q. Only HorizontalRunnerAutoscalers in %q are watched, cached, and considered as scale targets.")
+	}
+
+	logger := zap.New(func(o *zap.Options) {
+		o.Development = true
+	})
+
+	ctrl.SetLogger(logger)
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme:             scheme,
+		SyncPeriod:         &syncPeriod,
+		LeaderElection:     enableLeaderElection,
+		Namespace:          watchNamespace,
+		MetricsBindAddress: metricsAddr,
+		Port:               9443,
+	})
+	if err != nil {
+		setupLog.Error(err, "unable to start manager")
+		os.Exit(1)
+	}
+
+	hraGitHubWebhook := &controllers.HorizontalRunnerAutoscalerGitHubWebhook{
+		Client:         mgr.GetClient(),
+		Log:            ctrl.Log.WithName("controllers").WithName("Runner"),
+		Recorder:       nil,
+		Scheme:         mgr.GetScheme(),
+		SecretKeyBytes: []byte(webhookSecretToken),
+		WatchNamespace: watchNamespace,
+	}
+
+	if err = hraGitHubWebhook.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Runner")
+		os.Exit(1)
+	}
+
+	var wg sync.WaitGroup
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+
+		setupLog.Info("starting webhook server")
+		if err := mgr.Start(ctx.Done()); err != nil {
+			setupLog.Error(err, "problem running manager")
+			os.Exit(1)
+		}
+	}()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", hraGitHubWebhook.Handle)
+
+	srv := http.Server{
+		Addr:    webhookAddr,
+		Handler: mux,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+
+		go func() {
+			<-ctx.Done()
+
+			srv.Shutdown(context.Background())
+		}()
+
+		if err := srv.ListenAndServe(); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				setupLog.Error(err, "problem running http server")
+			}
+		}
+	}()
+
+	go func() {
+		<-ctrl.SetupSignalHandler()
+		cancel()
+	}()
+
+	wg.Wait()
+}

config/certmanager/certificate.yaml

@@ -1,7 +1,7 @@
 # The following manifests contain a self-signed issuer CR and a certificate CR.
 # More document can be found at https://docs.cert-manager.io
 # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: selfsigned-issuer
@@ -9,7 +9,7 @@ metadata:
 spec:
   selfSigned: {}
 ---
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -0,0 +1,218 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.3.0
+  creationTimestamp: null
+  name: horizontalrunnerautoscalers.actions.summerwind.dev
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .spec.minReplicas
+    name: Min
+    type: number
+  - JSONPath: .spec.maxReplicas
+    name: Max
+    type: number
+  - JSONPath: .status.desiredReplicas
+    name: Desired
+    type: number
+  group: actions.summerwind.dev
+  names:
+    kind: HorizontalRunnerAutoscaler
+    listKind: HorizontalRunnerAutoscalerList
+    plural: horizontalrunnerautoscalers
+    singular: horizontalrunnerautoscaler
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: HorizontalRunnerAutoscaler is the Schema for the horizontalrunnerautoscaler
+        API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: HorizontalRunnerAutoscalerSpec defines the desired state of
+            HorizontalRunnerAutoscaler
+          properties:
+            capacityReservations:
+              items:
+                description: CapacityReservation specifies the number of replicas
+                  temporarily added to the scale target until ExpirationTime.
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  name:
+                    type: string
+                  replicas:
+                    type: integer
+                type: object
+              type: array
+            maxReplicas:
+              description: MinReplicas is the maximum number of replicas the deployment
+                is allowed to scale
+              type: integer
+            metrics:
+              description: Metrics is the collection of various metric targets to
+                calculate desired number of runners
+              items:
+                properties:
+                  repositoryNames:
+                    description: RepositoryNames is the list of repository names to
+                      be used for calculating the metric. For example, a repository
+                      name is the REPO part of `github.com/USER/REPO`.
+                    items:
+                      type: string
+                    type: array
+                  scaleDownAdjustment:
+                    description: ScaleDownAdjustment is the number of runners removed
+                      on scale-down. You can only specify either ScaleDownFactor or
+                      ScaleDownAdjustment.
+                    type: integer
+                  scaleDownFactor:
+                    description: ScaleDownFactor is the multiplicative factor applied
+                      to the current number of runners used to determine how many
+                      pods should be removed.
+                    type: string
+                  scaleDownThreshold:
+                    description: ScaleDownThreshold is the percentage of busy runners
+                      less than which will trigger the hpa to scale the runners down.
+                    type: string
+                  scaleUpAdjustment:
+                    description: ScaleUpAdjustment is the number of runners added
+                      on scale-up. You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+                    type: integer
+                  scaleUpFactor:
+                    description: ScaleUpFactor is the multiplicative factor applied
+                      to the current number of runners used to determine how many
+                      pods should be added.
+                    type: string
+                  scaleUpThreshold:
+                    description: ScaleUpThreshold is the percentage of busy runners
+                      greater than which will trigger the hpa to scale runners up.
+                    type: string
+                  type:
+                    description: Type is the type of metric to be used for autoscaling.
+                      The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                    type: string
+                type: object
+              type: array
+            minReplicas:
+              description: MinReplicas is the minimum number of replicas the deployment
+                is allowed to scale
+              type: integer
+            scaleDownDelaySecondsAfterScaleOut:
+              description: ScaleDownDelaySecondsAfterScaleUp is the approximate delay
+                for a scale down followed by a scale up Used to prevent flapping (down->up->down->...
+                loop)
+              type: integer
+            scaleTargetRef:
+              description: ScaleTargetRef sis the reference to scaled resource like
+                RunnerDeployment
+              properties:
+                name:
+                  type: string
+              type: object
+            scaleUpTriggers:
+              description: "ScaleUpTriggers is an experimental feature to increase
+                the desired replicas by 1 on each webhook requested received by the
+                webhookBasedAutoscaler. \n This feature requires you to also enable
+                and deploy the webhookBasedAutoscaler onto your cluster. \n Note that
+                the added runners remain until the next sync period at least, and
+                they may or may not be used by GitHub Actions depending on the timing.
+                They are intended to be used to gain \"resource slack\" immediately
+                after you receive a webhook from GitHub, so that you can loosely expect
+                MinReplicas runners to be always available."
+              items:
+                properties:
+                  amount:
+                    type: integer
+                  duration:
+                    type: string
+                  githubEvent:
+                    properties:
+                      checkRun:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#check_run
+                        properties:
+                          status:
+                            type: string
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      pullRequest:
+                        description: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request
+                        properties:
+                          branches:
+                            items:
+                              type: string
+                            type: array
+                          types:
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      push:
+                        description: PushSpec is the condition for triggering scale-up
+                          on push event Also see https://docs.github.com/en/actions/reference/events-that-trigger-workflows#push
+                        type: object
+                    type: object
+                type: object
+              type: array
+          type: object
+        status:
+          properties:
+            cacheEntries:
+              items:
+                properties:
+                  expirationTime:
+                    format: date-time
+                    type: string
+                  key:
+                    type: string
+                  value:
+                    type: integer
+                type: object
+              type: array
+            desiredReplicas:
+              description: DesiredReplicas is the total number of desired, non-terminated
+                and latest pods to be set for the primary RunnerSet This doesn't include
+                outdated pods while upgrading the deployment and replacing the runnerset.
+              type: integer
+            lastSuccessfulScaleOutTime:
+              format: date-time
+              type: string
+            observedGeneration:
+              description: ObservedGeneration is the most recent generation observed
+                for the target. It corresponds to e.g. RunnerDeployment's generation,
+                which is updated on mutation by the API Server.
+              format: int64
+              type: integer
+          type: object
+      type: object
+  version: v1alpha1
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/actions.summerwind.dev_runners.yaml
 - bases/actions.summerwind.dev_runnerreplicasets.yaml
 - bases/actions.summerwind.dev_runnerdeployments.yaml
+- bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/default/kustomization.yaml

@@ -17,9 +17,9 @@ bases:
 - ../rbac
 - ../manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
-#- ../webhook
+- ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
-#- ../certmanager
+- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 
 #- ../prometheus
 
@@ -36,39 +36,39 @@ patchesStrategicMerge:
 #- manager_prometheus_metrics_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml
-#- manager_webhook_patch.yaml
+- manager_webhook_patch.yaml
 
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
-#- webhookcainjection_patch.yaml
+- webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
 vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
-#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1alpha2
-#    name: serving-cert # this name should match the one in certificate.yaml
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: CERTIFICATE_NAME
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1alpha2
-#    name: serving-cert # this name should match the one in certificate.yaml
-#- name: SERVICE_NAMESPACE # namespace of the service
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: SERVICE_NAME
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
+- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert # this name should match the one in certificate.yaml
+  fieldref:
+    fieldpath: metadata.namespace
+- name: CERTIFICATE_NAME
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert # this name should match the one in certificate.yaml
+- name: SERVICE_NAMESPACE # namespace of the service
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service
+  fieldref:
+    fieldpath: metadata.namespace
+- name: SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service

config/default/manager_auth_proxy_patch.yaml

@@ -10,7 +10,7 @@ spec:
     spec:
       containers:
       - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
+        image: quay.io/brancz/kube-rbac-proxy:v0.8.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8080/"
@@ -23,3 +23,4 @@ spec:
         args:
         - "--metrics-addr=127.0.0.1:8080"
         - "--enable-leader-election"
+        - "--sync-period=10m"

config/manager/manager.yaml

@@ -35,11 +35,34 @@ spec:
             secretKeyRef:
               name: controller-manager
               key: github_token
+              optional: true
+        - name: GITHUB_APP_ID
+          valueFrom:
+            secretKeyRef:
+              name: controller-manager
+              key: github_app_id
+              optional: true
+        - name: GITHUB_APP_INSTALLATION_ID
+          valueFrom:
+            secretKeyRef:
+              name: controller-manager
+              key: github_app_installation_id
+              optional: true
+        - name: GITHUB_APP_PRIVATE_KEY
+          value: /etc/actions-runner-controller/github_app_private_key
+        volumeMounts:
+        - name: controller-manager
+          mountPath: "/etc/actions-runner-controller"
+          readOnly: true
         resources:
           limits:
             cpu: 100m
-            memory: 30Mi
+            memory: 100Mi
           requests:
             cpu: 100m
             memory: 20Mi
+      volumes:
+      - name: controller-manager
+        secret:
+          secretName: controller-manager
       terminationGracePeriodSeconds: 10

config/rbac/role.yaml

@@ -6,6 +6,38 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -18,6 +50,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -38,6 +82,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerreplicasets/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -58,6 +114,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -66,6 +134,13 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -78,3 +153,15 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

config/webhook/manifests.yaml

@@ -0,0 +1,124 @@
+
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: mutating-webhook-configuration
+webhooks:
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-actions-summerwind-dev-v1alpha1-runner
+  failurePolicy: Fail
+  name: mutate.runner.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runners
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-actions-summerwind-dev-v1alpha1-runnerdeployment
+  failurePolicy: Fail
+  name: mutate.runnerdeployment.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerdeployments
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-actions-summerwind-dev-v1alpha1-runnerreplicaset
+  failurePolicy: Fail
+  name: mutate.runnerreplicaset.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerreplicasets
+
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: validating-webhook-configuration
+webhooks:
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-actions-summerwind-dev-v1alpha1-runner
+  failurePolicy: Fail
+  name: validate.runner.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runners
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-actions-summerwind-dev-v1alpha1-runnerdeployment
+  failurePolicy: Fail
+  name: validate.runnerdeployment.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerdeployments
+- clientConfig:
+    caBundle: Cg==
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-actions-summerwind-dev-v1alpha1-runnerreplicaset
+  failurePolicy: Fail
+  name: validate.runnerreplicaset.actions.summerwind.dev
+  rules:
+  - apiGroups:
+    - actions.summerwind.dev
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - runnerreplicasets

controllers/autoscaling.go

@@ -0,0 +1,337 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	defaultScaleUpThreshold   = 0.8
+	defaultScaleDownThreshold = 0.3
+	defaultScaleUpFactor      = 1.3
+	defaultScaleDownFactor    = 0.7
+)
+
+func getValueAvailableAt(now time.Time, from, to *time.Time, reservedValue int) *int {
+	if to != nil && now.After(*to) {
+		return nil
+	}
+
+	if from != nil && now.Before(*from) {
+		return nil
+	}
+
+	return &reservedValue
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) getDesiredReplicasFromCache(hra v1alpha1.HorizontalRunnerAutoscaler) *int {
+	var entry *v1alpha1.CacheEntry
+
+	for i := range hra.Status.CacheEntries {
+		ent := hra.Status.CacheEntries[i]
+
+		if ent.Key != v1alpha1.CacheEntryKeyDesiredReplicas {
+			continue
+		}
+
+		if !time.Now().Before(ent.ExpirationTime.Time) {
+			continue
+		}
+
+		entry = &ent
+
+		break
+	}
+
+	if entry != nil {
+		v := getValueAvailableAt(time.Now(), nil, &entry.ExpirationTime.Time, entry.Value)
+		if v != nil {
+			return v
+		}
+	}
+
+	return nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+	if hra.Spec.MinReplicas == nil {
+		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
+	} else if hra.Spec.MaxReplicas == nil {
+		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing maxReplicas", hra.Namespace, hra.Name)
+	}
+
+	metrics := hra.Spec.Metrics
+	if len(metrics) == 0 || metrics[0].Type == v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns {
+		return r.calculateReplicasByQueuedAndInProgressWorkflowRuns(rd, hra)
+	} else if metrics[0].Type == v1alpha1.AutoscalingMetricTypePercentageRunnersBusy {
+		return r.calculateReplicasByPercentageRunnersBusy(rd, hra)
+	} else {
+		return nil, fmt.Errorf("validting autoscaling metrics: unsupported metric type %q", metrics[0].Type)
+	}
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByQueuedAndInProgressWorkflowRuns(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+
+	var repos [][]string
+	metrics := hra.Spec.Metrics
+	repoID := rd.Spec.Template.Spec.Repository
+	if repoID == "" {
+		orgName := rd.Spec.Template.Spec.Organization
+		if orgName == "" {
+			return nil, fmt.Errorf("asserting runner deployment spec to detect bug: spec.template.organization should not be empty on this code path")
+		}
+
+		if len(metrics[0].RepositoryNames) == 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment")
+		}
+
+		for _, repoName := range metrics[0].RepositoryNames {
+			repos = append(repos, []string{orgName, repoName})
+		}
+	} else {
+		repo := strings.Split(repoID, "/")
+
+		repos = append(repos, repo)
+	}
+
+	var total, inProgress, queued, completed, unknown int
+	type callback func()
+	listWorkflowJobs := func(user string, repoName string, runID int64, fallback_cb callback) {
+		if runID == 0 {
+			fallback_cb()
+			return
+		}
+		jobs, _, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, nil)
+		if err != nil {
+			r.Log.Error(err, "Error listing workflow jobs")
+			fallback_cb()
+		} else if len(jobs.Jobs) == 0 {
+			fallback_cb()
+		} else {
+			for _, job := range jobs.Jobs {
+				switch job.GetStatus() {
+				case "completed":
+					// We add a case for `completed` so it is not counted in `unknown`.
+					// And we do not increment the counter for completed because
+					// that counter only refers to workflows. The reason for
+					// this is because we do not get a list of jobs for
+					// completed workflows in order to keep the number of API
+					// calls to a minimum.
+				case "in_progress":
+					inProgress++
+				case "queued":
+					queued++
+				default:
+					unknown++
+				}
+			}
+		}
+	}
+
+	for _, repo := range repos {
+		user, repoName := repo[0], repo[1]
+		workflowRuns, err := r.GitHubClient.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, run := range workflowRuns {
+			total++
+
+			// In May 2020, there are only 3 statuses.
+			// Follow the below links for more details:
+			// - https://developer.github.com/v3/actions/workflow-runs/#list-repository-workflow-runs
+			// - https://developer.github.com/v3/checks/runs/#create-a-check-run
+			switch run.GetStatus() {
+			case "completed":
+				completed++
+			case "in_progress":
+				listWorkflowJobs(user, repoName, run.GetID(), func() { inProgress++ })
+			case "queued":
+				listWorkflowJobs(user, repoName, run.GetID(), func() { queued++ })
+			default:
+				unknown++
+			}
+		}
+	}
+
+	minReplicas := *hra.Spec.MinReplicas
+	maxReplicas := *hra.Spec.MaxReplicas
+	necessaryReplicas := queued + inProgress
+
+	var desiredReplicas int
+
+	if necessaryReplicas < minReplicas {
+		desiredReplicas = minReplicas
+	} else if necessaryReplicas > maxReplicas {
+		desiredReplicas = maxReplicas
+	} else {
+		desiredReplicas = necessaryReplicas
+	}
+
+	rd.Status.Replicas = &desiredReplicas
+	replicas := desiredReplicas
+
+	r.Log.V(1).Info(
+		"Calculated desired replicas",
+		"computed_replicas_desired", desiredReplicas,
+		"spec_replicas_min", minReplicas,
+		"spec_replicas_max", maxReplicas,
+		"workflow_runs_completed", completed,
+		"workflow_runs_in_progress", inProgress,
+		"workflow_runs_queued", queued,
+		"workflow_runs_unknown", unknown,
+		"namespace", hra.Namespace,
+		"runner_deployment", rd.Name,
+		"horizontal_runner_autoscaler", hra.Name,
+	)
+
+	return &replicas, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunnersBusy(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+	ctx := context.Background()
+	minReplicas := *hra.Spec.MinReplicas
+	maxReplicas := *hra.Spec.MaxReplicas
+	metrics := hra.Spec.Metrics[0]
+	scaleUpThreshold := defaultScaleUpThreshold
+	scaleDownThreshold := defaultScaleDownThreshold
+	scaleUpFactor := defaultScaleUpFactor
+	scaleDownFactor := defaultScaleDownFactor
+
+	if metrics.ScaleUpThreshold != "" {
+		sut, err := strconv.ParseFloat(metrics.ScaleUpThreshold, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpThreshold cannot be parsed into a float64")
+		}
+		scaleUpThreshold = sut
+	}
+	if metrics.ScaleDownThreshold != "" {
+		sdt, err := strconv.ParseFloat(metrics.ScaleDownThreshold, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownThreshold cannot be parsed into a float64")
+		}
+
+		scaleDownThreshold = sdt
+	}
+
+	scaleUpAdjustment := metrics.ScaleUpAdjustment
+	if scaleUpAdjustment != 0 {
+		if metrics.ScaleUpAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpAdjustment cannot be lower than 0")
+		}
+
+		if metrics.ScaleUpFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleUpAdjustment and scaleUpFactor cannot be specified together")
+		}
+	} else if metrics.ScaleUpFactor != "" {
+		suf, err := strconv.ParseFloat(metrics.ScaleUpFactor, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpFactor cannot be parsed into a float64")
+		}
+		scaleUpFactor = suf
+	}
+
+	scaleDownAdjustment := metrics.ScaleDownAdjustment
+	if scaleDownAdjustment != 0 {
+		if metrics.ScaleDownAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownAdjustment cannot be lower than 0")
+		}
+
+		if metrics.ScaleDownFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleDownAdjustment and scaleDownFactor cannot be specified together")
+		}
+	} else if metrics.ScaleDownFactor != "" {
+		sdf, err := strconv.ParseFloat(metrics.ScaleDownFactor, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownFactor cannot be parsed into a float64")
+		}
+		scaleDownFactor = sdf
+	}
+
+	// return the list of runners in namespace. Horizontal Runner Autoscaler should only be responsible for scaling resources in its own ns.
+	var runnerList v1alpha1.RunnerList
+	if err := r.List(ctx, &runnerList, client.InNamespace(rd.Namespace)); err != nil {
+		return nil, err
+	}
+	runnerMap := make(map[string]struct{})
+	for _, items := range runnerList.Items {
+		runnerMap[items.Name] = struct{}{}
+	}
+
+	var (
+		enterprise   = rd.Spec.Template.Spec.Enterprise
+		organization = rd.Spec.Template.Spec.Organization
+		repository   = rd.Spec.Template.Spec.Repository
+	)
+
+	// ListRunners will return all runners managed by GitHub - not restricted to ns
+	runners, err := r.GitHubClient.ListRunners(
+		ctx,
+		enterprise,
+		organization,
+		repository)
+	if err != nil {
+		return nil, err
+	}
+	numRunners := len(runnerList.Items)
+	numRunnersBusy := 0
+	for _, runner := range runners {
+		if _, ok := runnerMap[*runner.Name]; ok && runner.GetBusy() {
+			numRunnersBusy++
+		}
+	}
+
+	var desiredReplicas int
+	fractionBusy := float64(numRunnersBusy) / float64(numRunners)
+	if fractionBusy >= scaleUpThreshold {
+		if scaleUpAdjustment > 0 {
+			desiredReplicas = numRunners + scaleUpAdjustment
+		} else {
+			desiredReplicas = int(math.Ceil(float64(numRunners) * scaleUpFactor))
+		}
+	} else if fractionBusy < scaleDownThreshold {
+		if scaleDownAdjustment > 0 {
+			desiredReplicas = numRunners - scaleDownAdjustment
+		} else {
+			desiredReplicas = int(float64(numRunners) * scaleDownFactor)
+		}
+	} else {
+		desiredReplicas = *rd.Spec.Replicas
+	}
+
+	if desiredReplicas < minReplicas {
+		desiredReplicas = minReplicas
+	} else if desiredReplicas > maxReplicas {
+		desiredReplicas = maxReplicas
+	}
+
+	r.Log.V(1).Info(
+		"Calculated desired replicas",
+		"computed_replicas_desired", desiredReplicas,
+		"spec_replicas_min", minReplicas,
+		"spec_replicas_max", maxReplicas,
+		"current_replicas", rd.Spec.Replicas,
+		"num_runners", numRunners,
+		"num_runners_busy", numRunnersBusy,
+		"namespace", hra.Namespace,
+		"runner_deployment", rd.Name,
+		"horizontal_runner_autoscaler", hra.Name,
+		"enterprise", enterprise,
+		"organization", organization,
+		"repository", repository,
+	)
+
+	rd.Status.Replicas = &desiredReplicas
+	replicas := desiredReplicas
+
+	return &replicas, nil
+}

controllers/autoscaling_test.go

@@ -0,0 +1,445 @@
+package controllers
+
+import (
+	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+func newGithubClient(server *httptest.Server) *github.Client {
+	c := github.Config{
+		Token: "token",
+	}
+	client, err := c.NewClient()
+	if err != nil {
+		panic(err)
+	}
+
+	baseURL, err := url.Parse(server.URL + "/")
+	if err != nil {
+		panic(err)
+	}
+	client.Client.BaseURL = baseURL
+
+	return client
+}
+
+func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
+	intPtr := func(v int) *int {
+		return &v
+	}
+
+	metav1Now := metav1.Now()
+	testcases := []struct {
+		repo         string
+		org          string
+		fixed        *int
+		max          *int
+		min          *int
+		sReplicas    *int
+		sTime        *metav1.Time
+		workflowRuns string
+		workflowJobs map[int]string
+		want         int
+		err          string
+	}{
+		// Legacy functionality
+		// 3 demanded, max at 3
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			sReplicas:    intPtr(3),
+			sTime:        &metav1Now,
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 3 demanded, max at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(2),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 2 demanded, min at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 1
+		{
+			repo:         "test/valid",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// 1 demanded, min at 1
+		{
+			repo:         "test/valid",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// fixed at 3
+		{
+			repo:         "test/valid",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			fixed:        intPtr(3),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
+	}
+
+	for i := range testcases {
+		tc := testcases[i]
+
+		log := zap.New(func(o *zap.Options) {
+			o.Development = true
+		})
+
+		scheme := runtime.NewScheme()
+		_ = clientgoscheme.AddToScheme(scheme)
+		_ = v1alpha1.AddToScheme(scheme)
+
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			server := fake.NewServer(
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns),
+				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
+				fake.WithListRunnersResponse(200, fake.RunnersListBody),
+			)
+			defer server.Close()
+			client := newGithubClient(server)
+
+			h := &HorizontalRunnerAutoscalerReconciler{
+				Log:          log,
+				GitHubClient: client,
+				Scheme:       scheme,
+			}
+
+			rd := v1alpha1.RunnerDeployment{
+				TypeMeta: metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testrd",
+				},
+				Spec: v1alpha1.RunnerDeploymentSpec{
+					Template: v1alpha1.RunnerTemplate{
+						Spec: v1alpha1.RunnerSpec{
+							Repository: tc.repo,
+						},
+					},
+					Replicas: tc.fixed,
+				},
+				Status: v1alpha1.RunnerDeploymentStatus{
+					Replicas: tc.sReplicas,
+				},
+			}
+
+			hra := v1alpha1.HorizontalRunnerAutoscaler{
+				Spec: v1alpha1.HorizontalRunnerAutoscalerSpec{
+					MaxReplicas: tc.max,
+					MinReplicas: tc.min,
+				},
+				Status: v1alpha1.HorizontalRunnerAutoscalerStatus{
+					DesiredReplicas:            tc.sReplicas,
+					LastSuccessfulScaleOutTime: tc.sTime,
+				},
+			}
+
+			got, err := h.computeReplicas(rd, hra)
+			if err != nil {
+				if tc.err == "" {
+					t.Fatalf("unexpected error: expected none, got %v", err)
+				} else if err.Error() != tc.err {
+					t.Fatalf("unexpected error: expected %v, got %v", tc.err, err)
+				}
+				return
+			}
+
+			if got == nil {
+				t.Fatalf("unexpected value of rs.Spec.Replicas: nil")
+			}
+
+			if *got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, *got)
+			}
+		})
+	}
+}
+
+func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
+	intPtr := func(v int) *int {
+		return &v
+	}
+
+	metav1Now := metav1.Now()
+	testcases := []struct {
+		repos        []string
+		org          string
+		fixed        *int
+		max          *int
+		min          *int
+		sReplicas    *int
+		sTime        *metav1.Time
+		workflowRuns string
+		workflowJobs map[int]string
+		want         int
+		err          string
+	}{
+		// 3 demanded, max at 3
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			sReplicas:    intPtr(3),
+			sTime:        &metav1Now,
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 3 demanded, max at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(2),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 2 demanded, min at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 1
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// 1 demanded, min at 1
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// fixed at 3
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			fixed:        intPtr(1),
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// org runner, fixed at 3
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			fixed:        intPtr(1),
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// org runner, 1 demanded, min at 1, no repos
+		{
+			org:          "test",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			err:          "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
+		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
+	}
+
+	for i := range testcases {
+		tc := testcases[i]
+
+		log := zap.New(func(o *zap.Options) {
+			o.Development = true
+		})
+
+		scheme := runtime.NewScheme()
+		_ = clientgoscheme.AddToScheme(scheme)
+		_ = v1alpha1.AddToScheme(scheme)
+
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			server := fake.NewServer(
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns),
+				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
+				fake.WithListRunnersResponse(200, fake.RunnersListBody),
+			)
+			defer server.Close()
+			client := newGithubClient(server)
+
+			h := &HorizontalRunnerAutoscalerReconciler{
+				Log:          log,
+				Scheme:       scheme,
+				GitHubClient: client,
+			}
+
+			rd := v1alpha1.RunnerDeployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testrd",
+				},
+				Spec: v1alpha1.RunnerDeploymentSpec{
+					Template: v1alpha1.RunnerTemplate{
+						Spec: v1alpha1.RunnerSpec{
+							Organization: tc.org,
+						},
+					},
+					Replicas: tc.fixed,
+				},
+				Status: v1alpha1.RunnerDeploymentStatus{
+					Replicas: tc.sReplicas,
+				},
+			}
+
+			hra := v1alpha1.HorizontalRunnerAutoscaler{
+				Spec: v1alpha1.HorizontalRunnerAutoscalerSpec{
+					ScaleTargetRef: v1alpha1.ScaleTargetRef{
+						Name: "testrd",
+					},
+					MaxReplicas: tc.max,
+					MinReplicas: tc.min,
+					Metrics: []v1alpha1.MetricSpec{
+						{
+							Type:            v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns,
+							RepositoryNames: tc.repos,
+						},
+					},
+				},
+				Status: v1alpha1.HorizontalRunnerAutoscalerStatus{
+					DesiredReplicas:            tc.sReplicas,
+					LastSuccessfulScaleOutTime: tc.sTime,
+				},
+			}
+
+			got, err := h.computeReplicas(rd, hra)
+			if err != nil {
+				if tc.err == "" {
+					t.Fatalf("unexpected error: expected none, got %v", err)
+				} else if err.Error() != tc.err {
+					t.Fatalf("unexpected error: expected %v, got %v", tc.err, err)
+				}
+				return
+			}
+
+			if got == nil {
+				t.Fatalf("unexpected value of rs.Spec.Replicas: nil, wanted %v", tc.want)
+			}
+
+			if *got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, *got)
+			}
+		})
+	}
+}

controllers/horizontal_runner_autoscaler_webhook.go

@@ -0,0 +1,375 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"net/http"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"time"
+
+	"github.com/go-logr/logr"
+	gogithub "github.com/google/go-github/v33/github"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+const (
+	scaleTargetKey = "scaleTarget"
+)
+
+// HorizontalRunnerAutoscalerGitHubWebhook autoscales a HorizontalRunnerAutoscaler and the RunnerDeployment on each
+// GitHub Webhook received
+type HorizontalRunnerAutoscalerGitHubWebhook struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+
+	// SecretKeyBytes is the byte representation of the Webhook secret token
+	// the administrator is generated and specified in GitHub Web UI.
+	SecretKeyBytes []byte
+
+	// WatchNamespace is the namespace to watch for HorizontalRunnerAutoscaler's to be
+	// scaled on Webhook.
+	// Set to empty for letting it watch for all namespaces.
+	WatchNamespace string
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+	return ctrl.Result{}, nil
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.ResponseWriter, r *http.Request) {
+	var (
+		ok bool
+
+		err error
+	)
+
+	defer func() {
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+
+			if err != nil {
+				msg := err.Error()
+				if written, err := w.Write([]byte(msg)); err != nil {
+					autoscaler.Log.Error(err, "failed writing http error response", "msg", msg, "written", written)
+				}
+			}
+		}
+	}()
+
+	defer func() {
+		if r.Body != nil {
+			r.Body.Close()
+		}
+	}()
+
+	var payload []byte
+
+	if len(autoscaler.SecretKeyBytes) > 0 {
+		payload, err = gogithub.ValidatePayload(r, autoscaler.SecretKeyBytes)
+		if err != nil {
+			autoscaler.Log.Error(err, "error validating request body")
+
+			return
+		}
+	} else {
+		payload, err = ioutil.ReadAll(r.Body)
+		if err != nil {
+			autoscaler.Log.Error(err, "error reading request body")
+
+			return
+		}
+	}
+
+	webhookType := gogithub.WebHookType(r)
+	event, err := gogithub.ParseWebHook(webhookType, payload)
+	if err != nil {
+		var s string
+		if payload != nil {
+			s = string(payload)
+		}
+
+		autoscaler.Log.Error(err, "could not parse webhook", "webhookType", webhookType, "payload", s)
+
+		return
+	}
+
+	var target *ScaleTarget
+
+	autoscaler.Log.Info("processing webhook event", "eventType", webhookType)
+
+	switch e := event.(type) {
+	case *gogithub.PushEvent:
+		target, err = autoscaler.getScaleUpTarget(
+			context.TODO(),
+			*e.Repo.Name,
+			*e.Repo.Organization,
+			autoscaler.MatchPushEvent(e),
+		)
+	case *gogithub.PullRequestEvent:
+		target, err = autoscaler.getScaleUpTarget(
+			context.TODO(),
+			*e.Repo.Name,
+			*e.Repo.Organization.Name,
+			autoscaler.MatchPullRequestEvent(e),
+		)
+	case *gogithub.CheckRunEvent:
+		target, err = autoscaler.getScaleUpTarget(
+			context.TODO(),
+			*e.Repo.Name,
+			*e.Org.Name,
+			autoscaler.MatchCheckRunEvent(e),
+		)
+	case *gogithub.PingEvent:
+		ok = true
+
+		w.WriteHeader(http.StatusOK)
+
+		msg := "pong"
+
+		if written, err := w.Write([]byte(msg)); err != nil {
+			autoscaler.Log.Error(err, "failed writing http response", "msg", msg, "written", written)
+		}
+
+		autoscaler.Log.Info("received ping event")
+
+		return
+	default:
+		autoscaler.Log.Info("unknown event type", "eventType", webhookType)
+
+		return
+	}
+
+	if err != nil {
+		autoscaler.Log.Error(err, "handling check_run event")
+
+		return
+	}
+
+	if target == nil {
+		msg := "no horizontalrunnerautoscaler to scale for this github event"
+
+		autoscaler.Log.Info(msg, "eventType", webhookType)
+
+		ok = true
+
+		w.WriteHeader(http.StatusOK)
+
+		if written, err := w.Write([]byte(msg)); err != nil {
+			autoscaler.Log.Error(err, "failed writing http response", "msg", msg, "written", written)
+		}
+
+		return
+	}
+
+	if err := autoscaler.tryScaleUp(context.TODO(), target); err != nil {
+		autoscaler.Log.Error(err, "could not scale up")
+
+		return
+	}
+
+	ok = true
+
+	w.WriteHeader(http.StatusOK)
+
+	msg := fmt.Sprintf("scaled %s by 1", target.Name)
+
+	autoscaler.Log.Info(msg)
+
+	if written, err := w.Write([]byte(msg)); err != nil {
+		autoscaler.Log.Error(err, "failed writing http response", "msg", msg, "written", written)
+	}
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) findHRAsByKey(ctx context.Context, value string) ([]v1alpha1.HorizontalRunnerAutoscaler, error) {
+	ns := autoscaler.WatchNamespace
+
+	var defaultListOpts []client.ListOption
+
+	if ns != "" {
+		defaultListOpts = append(defaultListOpts, client.InNamespace(ns))
+	}
+
+	var hras []v1alpha1.HorizontalRunnerAutoscaler
+
+	if value != "" {
+		opts := append([]client.ListOption{}, defaultListOpts...)
+		opts = append(opts, client.MatchingFields{scaleTargetKey: value})
+
+		var hraList v1alpha1.HorizontalRunnerAutoscalerList
+
+		if err := autoscaler.List(ctx, &hraList, opts...); err != nil {
+			return nil, err
+		}
+
+		for _, d := range hraList.Items {
+			hras = append(hras, d)
+		}
+	}
+
+	return hras, nil
+}
+
+func matchTriggerConditionAgainstEvent(types []string, eventAction *string) bool {
+	if len(types) == 0 {
+		return true
+	}
+
+	if eventAction == nil {
+		return false
+	}
+
+	for _, tpe := range types {
+		if tpe == *eventAction {
+			return true
+		}
+	}
+
+	return false
+}
+
+type ScaleTarget struct {
+	v1alpha1.HorizontalRunnerAutoscaler
+	v1alpha1.ScaleUpTrigger
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) searchScaleTargets(hras []v1alpha1.HorizontalRunnerAutoscaler, f func(v1alpha1.ScaleUpTrigger) bool) []ScaleTarget {
+	var matched []ScaleTarget
+
+	for _, hra := range hras {
+		if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		for _, scaleUpTrigger := range hra.Spec.ScaleUpTriggers {
+			if !f(scaleUpTrigger) {
+				continue
+			}
+
+			matched = append(matched, ScaleTarget{
+				HorizontalRunnerAutoscaler: hra,
+				ScaleUpTrigger:             scaleUpTrigger,
+			})
+		}
+	}
+
+	return matched
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleTarget(ctx context.Context, name string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) {
+	hras, err := autoscaler.findHRAsByKey(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	targets := autoscaler.searchScaleTargets(hras, f)
+
+	if len(targets) != 1 {
+		return nil, nil
+	}
+
+	return &targets[0], nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTarget(ctx context.Context, repoNameFromWebhook, orgNameFromWebhook string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) {
+	if target, err := autoscaler.getScaleTarget(ctx, repoNameFromWebhook, f); err != nil {
+		return nil, err
+	} else if target != nil {
+		autoscaler.Log.Info("scale up target is repository-wide runners", "repository", repoNameFromWebhook)
+		return target, nil
+	}
+
+	if target, err := autoscaler.getScaleTarget(ctx, orgNameFromWebhook, f); err != nil {
+		return nil, err
+	} else if target != nil {
+		autoscaler.Log.Info("scale up target is organizational runners", "repository", orgNameFromWebhook)
+		return target, nil
+	}
+
+	return nil, nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScaleUp(ctx context.Context, target *ScaleTarget) error {
+	if target == nil {
+		return nil
+	}
+
+	log := autoscaler.Log.WithValues("horizontalrunnerautoscaler", target.HorizontalRunnerAutoscaler.Name)
+
+	copy := target.HorizontalRunnerAutoscaler.DeepCopy()
+
+	amount := 1
+
+	if target.ScaleUpTrigger.Amount > 0 {
+		amount = target.ScaleUpTrigger.Amount
+	}
+
+	copy.Spec.CapacityReservations = append(copy.Spec.CapacityReservations, v1alpha1.CapacityReservation{
+		ExpirationTime: metav1.Time{Time: time.Now().Add(target.ScaleUpTrigger.Duration.Duration)},
+		Replicas:       amount,
+	})
+
+	if err := autoscaler.Client.Update(ctx, copy); err != nil {
+		log.Error(err, "Failed to update horizontalrunnerautoscaler resource")
+
+		return err
+	}
+
+	return nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr ctrl.Manager) error {
+	autoscaler.Recorder = mgr.GetEventRecorderFor("webhookbasedautoscaler")
+
+	if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.HorizontalRunnerAutoscaler{}, scaleTargetKey, func(rawObj runtime.Object) []string {
+		hra := rawObj.(*v1alpha1.HorizontalRunnerAutoscaler)
+
+		if hra.Spec.ScaleTargetRef.Name == "" {
+			return nil
+		}
+
+		var rd v1alpha1.RunnerDeployment
+
+		if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil {
+			return nil
+		}
+
+		return []string{rd.Spec.Template.Spec.Repository, rd.Spec.Template.Spec.Organization}
+	}); err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Complete(autoscaler)
+}

controllers/horizontal_runner_autoscaler_webhook_on_check_run.go

@@ -0,0 +1,32 @@
+package controllers
+
+import (
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchCheckRunEvent(event *github.CheckRunEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+	return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+		g := scaleUpTrigger.GitHubEvent
+
+		if g == nil {
+			return false
+		}
+
+		cr := g.CheckRun
+
+		if cr == nil {
+			return false
+		}
+
+		if !matchTriggerConditionAgainstEvent(cr.Types, event.Action) {
+			return false
+		}
+
+		if cr.Status != "" && (event.CheckRun == nil || event.CheckRun.Status == nil || *event.CheckRun.Status != cr.Status) {
+			return false
+		}
+
+		return true
+	}
+}

controllers/horizontal_runner_autoscaler_webhook_on_pull_request.go

@@ -0,0 +1,32 @@
+package controllers
+
+import (
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPullRequestEvent(event *github.PullRequestEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+	return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+		g := scaleUpTrigger.GitHubEvent
+
+		if g == nil {
+			return false
+		}
+
+		pr := g.PullRequest
+
+		if pr == nil {
+			return false
+		}
+
+		if !matchTriggerConditionAgainstEvent(pr.Types, event.Action) {
+			return false
+		}
+
+		if !matchTriggerConditionAgainstEvent(pr.Branches, event.PullRequest.Base.Ref) {
+			return false
+		}
+
+		return true
+	}
+}

controllers/horizontal_runner_autoscaler_webhook_on_push.go

@@ -0,0 +1,24 @@
+package controllers
+
+import (
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) MatchPushEvent(event *github.PushEvent) func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+	return func(scaleUpTrigger v1alpha1.ScaleUpTrigger) bool {
+		g := scaleUpTrigger.GitHubEvent
+
+		if g == nil {
+			return false
+		}
+
+		push := g.Push
+
+		if push == nil {
+			return false
+		}
+
+		return true
+	}
+}

controllers/horizontal_runner_autoscaler_webhook_test.go

@@ -0,0 +1,245 @@
+package controllers
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/go-logr/logr"
+	"github.com/google/go-github/v33/github"
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"io"
+	"io/ioutil"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"testing"
+)
+
+var (
+	sc = runtime.NewScheme()
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(sc)
+	_ = actionsv1alpha1.AddToScheme(sc)
+}
+
+func TestWebhookCheckRun(t *testing.T) {
+	testServer(t,
+		"check_run",
+		&github.CheckRunEvent{
+			CheckRun: &github.CheckRun{
+				Status: github.String("queued"),
+			},
+			Repo: &github.Repository{
+				Name: github.String("myorg/myrepo"),
+			},
+			Org: &github.Organization{
+				Name: github.String("myorg"),
+			},
+			Action: github.String("created"),
+		},
+		200,
+		"no horizontalrunnerautoscaler to scale for this github event",
+	)
+}
+
+func TestWebhookPullRequest(t *testing.T) {
+	testServer(t,
+		"pull_request",
+		&github.PullRequestEvent{
+			PullRequest: &github.PullRequest{
+				Base: &github.PullRequestBranch{
+					Ref: github.String("main"),
+				},
+			},
+			Repo: &github.Repository{
+				Name: github.String("myorg/myrepo"),
+				Organization: &github.Organization{
+					Name: github.String("myorg"),
+				},
+			},
+			Action: github.String("created"),
+		},
+		200,
+		"no horizontalrunnerautoscaler to scale for this github event",
+	)
+}
+
+func TestWebhookPush(t *testing.T) {
+	testServer(t,
+		"push",
+		&github.PushEvent{
+			Repo: &github.PushEventRepository{
+				Name:         github.String("myrepo"),
+				Organization: github.String("myorg"),
+			},
+		},
+		200,
+		"no horizontalrunnerautoscaler to scale for this github event",
+	)
+}
+
+func TestWebhookPing(t *testing.T) {
+	testServer(t,
+		"ping",
+		&github.PingEvent{
+			Zen: github.String("zen"),
+		},
+		200,
+		"pong",
+	)
+}
+
+func installTestLogger(webhook *HorizontalRunnerAutoscalerGitHubWebhook) *bytes.Buffer {
+	logs := &bytes.Buffer{}
+
+	log := testLogger{
+		name:   "testlog",
+		writer: logs,
+	}
+
+	webhook.Log = &log
+
+	return logs
+}
+
+func testServer(t *testing.T, eventType string, event interface{}, wantCode int, wantBody string) {
+	t.Helper()
+
+	hraWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{}
+
+	var initObjs []runtime.Object
+
+	client := fake.NewFakeClientWithScheme(sc, initObjs...)
+
+	logs := installTestLogger(hraWebhook)
+
+	defer func() {
+		if t.Failed() {
+			t.Logf("diagnostics: %s", logs.String())
+		}
+	}()
+
+	hraWebhook.Client = client
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", hraWebhook.Handle)
+
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	resp, err := sendWebhook(server, eventType, event)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if resp != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	if resp.StatusCode != wantCode {
+		t.Error("status:", resp.StatusCode)
+	}
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(respBody) != wantBody {
+		t.Fatal("body:", string(respBody))
+	}
+}
+
+func sendWebhook(server *httptest.Server, eventType string, event interface{}) (*http.Response, error) {
+	jsonBuf := &bytes.Buffer{}
+	enc := json.NewEncoder(jsonBuf)
+	enc.SetIndent("  ", "")
+	err := enc.Encode(event)
+	if err != nil {
+		return nil, fmt.Errorf("[bug in test] encoding event to json: %+v", err)
+	}
+
+	reqBody := jsonBuf.Bytes()
+
+	u, err := url.Parse(server.URL)
+	if err != nil {
+		return nil, fmt.Errorf("parsing server url: %v", err)
+	}
+
+	req := &http.Request{
+		Method: http.MethodPost,
+		URL:    u,
+		Header: map[string][]string{
+			"X-GitHub-Event": {eventType},
+			"Content-Type":   {"application/json"},
+		},
+		Body: ioutil.NopCloser(bytes.NewBuffer(reqBody)),
+	}
+
+	return http.DefaultClient.Do(req)
+}
+
+// testLogger is a sample logr.Logger that logs in-memory.
+// It's only for testing log outputs.
+type testLogger struct {
+	name      string
+	keyValues map[string]interface{}
+
+	writer io.Writer
+}
+
+var _ logr.Logger = &testLogger{}
+
+func (l *testLogger) Info(msg string, kvs ...interface{}) {
+	fmt.Fprintf(l.writer, "%s] %s\t", l.name, msg)
+	for k, v := range l.keyValues {
+		fmt.Fprintf(l.writer, "%s=%+v ", k, v)
+	}
+	for i := 0; i < len(kvs); i += 2 {
+		fmt.Fprintf(l.writer, "%s=%+v ", kvs[i], kvs[i+1])
+	}
+	fmt.Fprintf(l.writer, "\n")
+}
+
+func (_ *testLogger) Enabled() bool {
+	return true
+}
+
+func (l *testLogger) Error(err error, msg string, kvs ...interface{}) {
+	kvs = append(kvs, "error", err)
+	l.Info(msg, kvs...)
+}
+
+func (l *testLogger) V(_ int) logr.InfoLogger {
+	return l
+}
+
+func (l *testLogger) WithName(name string) logr.Logger {
+	return &testLogger{
+		name:      l.name + "." + name,
+		keyValues: l.keyValues,
+		writer:    l.writer,
+	}
+}
+
+func (l *testLogger) WithValues(kvs ...interface{}) logr.Logger {
+	newMap := make(map[string]interface{}, len(l.keyValues)+len(kvs)/2)
+	for k, v := range l.keyValues {
+		newMap[k] = v
+	}
+	for i := 0; i < len(kvs); i += 2 {
+		newMap[kvs[i].(string)] = kvs[i+1]
+	}
+	return &testLogger{
+		name:      l.name,
+		keyValues: newMap,
+		writer:    l.writer,
+	}
+}

controllers/horizontalrunnerautoscaler_controller.go

@@ -0,0 +1,222 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	"github.com/summerwind/actions-runner-controller/github"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+const (
+	DefaultScaleDownDelay = 10 * time.Minute
+)
+
+// HorizontalRunnerAutoscalerReconciler reconciles a HorizontalRunnerAutoscaler object
+type HorizontalRunnerAutoscalerReconciler struct {
+	client.Client
+	GitHubClient *github.Client
+	Log          logr.Logger
+	Recorder     record.EventRecorder
+	Scheme       *runtime.Scheme
+
+	CacheDuration time.Duration
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	ctx := context.Background()
+	log := r.Log.WithValues("horizontalrunnerautoscaler", req.NamespacedName)
+
+	var hra v1alpha1.HorizontalRunnerAutoscaler
+	if err := r.Get(ctx, req.NamespacedName, &hra); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	var rd v1alpha1.RunnerDeployment
+	if err := r.Get(ctx, types.NamespacedName{
+		Namespace: req.Namespace,
+		Name:      hra.Spec.ScaleTargetRef.Name,
+	}, &rd); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !rd.ObjectMeta.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	var replicas *int
+
+	replicasFromCache := r.getDesiredReplicasFromCache(hra)
+
+	if replicasFromCache != nil {
+		replicas = replicasFromCache
+	} else {
+		var err error
+
+		replicas, err = r.computeReplicas(rd, hra)
+		if err != nil {
+			r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
+
+			log.Error(err, "Could not compute replicas")
+
+			return ctrl.Result{}, err
+		}
+	}
+
+	const defaultReplicas = 1
+
+	currentDesiredReplicas := getIntOrDefault(rd.Spec.Replicas, defaultReplicas)
+	newDesiredReplicas := getIntOrDefault(replicas, defaultReplicas)
+
+	now := time.Now()
+
+	for _, reservation := range hra.Spec.CapacityReservations {
+		if reservation.ExpirationTime.Time.After(now) {
+			newDesiredReplicas += reservation.Replicas
+		}
+	}
+
+	if hra.Spec.MaxReplicas != nil && *hra.Spec.MaxReplicas < newDesiredReplicas {
+		newDesiredReplicas = *hra.Spec.MaxReplicas
+	}
+
+	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
+	if currentDesiredReplicas != newDesiredReplicas {
+		copy := rd.DeepCopy()
+		copy.Spec.Replicas = &newDesiredReplicas
+
+		if err := r.Client.Update(ctx, copy); err != nil {
+			log.Error(err, "Failed to update runnerderployment resource")
+
+			return ctrl.Result{}, err
+		}
+	}
+
+	var updated *v1alpha1.HorizontalRunnerAutoscaler
+
+	if hra.Status.DesiredReplicas == nil || *hra.Status.DesiredReplicas != *replicas {
+		updated = hra.DeepCopy()
+
+		if (hra.Status.DesiredReplicas == nil && *replicas > 1) ||
+			(hra.Status.DesiredReplicas != nil && *replicas > *hra.Status.DesiredReplicas) {
+
+			updated.Status.LastSuccessfulScaleOutTime = &metav1.Time{Time: time.Now()}
+		}
+
+		updated.Status.DesiredReplicas = replicas
+	}
+
+	if replicasFromCache == nil {
+		if updated == nil {
+			updated = hra.DeepCopy()
+		}
+
+		var cacheEntries []v1alpha1.CacheEntry
+
+		for _, ent := range updated.Status.CacheEntries {
+			if ent.ExpirationTime.Before(&metav1.Time{Time: now}) {
+				cacheEntries = append(cacheEntries, ent)
+			}
+		}
+
+		var cacheDuration time.Duration
+
+		if r.CacheDuration > 0 {
+			cacheDuration = r.CacheDuration
+		} else {
+			cacheDuration = 10 * time.Minute
+		}
+
+		updated.Status.CacheEntries = append(updated.Status.CacheEntries, v1alpha1.CacheEntry{
+			Key:            v1alpha1.CacheEntryKeyDesiredReplicas,
+			Value:          *replicas,
+			ExpirationTime: metav1.Time{Time: time.Now().Add(cacheDuration)},
+		})
+	}
+
+	if updated != nil {
+		if err := r.Status().Update(ctx, updated); err != nil {
+			log.Error(err, "Failed to update horizontalrunnerautoscaler status")
+
+			return ctrl.Result{}, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.Recorder = mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller")
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Complete(r)
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+	var computedReplicas *int
+
+	replicas, err := r.determineDesiredReplicas(rd, hra)
+	if err != nil {
+		return nil, err
+	}
+
+	var scaleDownDelay time.Duration
+
+	if hra.Spec.ScaleDownDelaySecondsAfterScaleUp != nil {
+		scaleDownDelay = time.Duration(*hra.Spec.ScaleDownDelaySecondsAfterScaleUp) * time.Second
+	} else {
+		scaleDownDelay = DefaultScaleDownDelay
+	}
+
+	now := time.Now()
+
+	if hra.Status.DesiredReplicas == nil ||
+		*hra.Status.DesiredReplicas < *replicas ||
+		hra.Status.LastSuccessfulScaleOutTime == nil ||
+		hra.Status.LastSuccessfulScaleOutTime.Add(scaleDownDelay).Before(now) {
+
+		computedReplicas = replicas
+	} else {
+		computedReplicas = hra.Status.DesiredReplicas
+	}
+
+	return computedReplicas, nil
+}

controllers/integration_test.go

@@ -0,0 +1,433 @@
+package controllers
+
+import (
+	"context"
+	"github.com/google/go-github/v33/github"
+	github3 "github.com/google/go-github/v33/github"
+	github2 "github.com/summerwind/actions-runner-controller/github"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	"github.com/summerwind/actions-runner-controller/github/fake"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+type testEnvironment struct {
+	Namespace *corev1.Namespace
+	Responses *fake.FixedResponses
+}
+
+var (
+	workflowRunsFor3Replicas = `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`
+	workflowRunsFor1Replicas = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"`
+)
+
+var webhookServer *httptest.Server
+
+var ghClient *github2.Client
+
+var fakeRunnerList *fake.RunnersList
+
+// SetupIntegrationTest will set up a testing environment.
+// This includes:
+// * creating a Namespace to be used during the test
+// * starting all the reconcilers
+// * stopping all the reconcilers after the test ends
+// Call this function at the start of each of your tests.
+func SetupIntegrationTest(ctx context.Context) *testEnvironment {
+	var stopCh chan struct{}
+	ns := &corev1.Namespace{}
+
+	responses := &fake.FixedResponses{}
+	responses.ListRunners = fake.DefaultListRunnersHandler()
+	responses.ListRepositoryWorkflowRuns = &fake.Handler{
+		Status: 200,
+		Body:   workflowRunsFor3Replicas,
+	}
+	fakeRunnerList = fake.NewRunnersList()
+	responses.ListRunners = fakeRunnerList.HandleList()
+	fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses))
+
+	BeforeEach(func() {
+		stopCh = make(chan struct{})
+		*ns = corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)},
+		}
+
+		err := k8sClient.Create(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
+		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
+
+		ghClient = newGithubClient(fakeGithubServer)
+
+		replicasetController := &RunnerReplicaSetReconciler{
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
+		}
+		err = replicasetController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		deploymentsController := &RunnerDeploymentReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerdeployment-controller"),
+		}
+		err = deploymentsController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		client := newGithubClient(fakeGithubServer)
+
+		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
+			Client:        mgr.GetClient(),
+			Scheme:        scheme.Scheme,
+			Log:           logf.Log,
+			GitHubClient:  client,
+			Recorder:      mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+			CacheDuration: 1 * time.Second,
+		}
+		err = autoscalerController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		autoscalerWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+		}
+		err = autoscalerWebhook.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler webhook")
+
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", autoscalerWebhook.Handle)
+
+		webhookServer = httptest.NewServer(mux)
+
+		go func() {
+			defer GinkgoRecover()
+
+			err := mgr.Start(stopCh)
+			Expect(err).NotTo(HaveOccurred(), "failed to start manager")
+		}()
+	})
+
+	AfterEach(func() {
+		close(stopCh)
+
+		fakeGithubServer.Close()
+		webhookServer.Close()
+
+		err := k8sClient.Delete(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
+	})
+
+	return &testEnvironment{Namespace: ns, Responses: responses}
+}
+
+var _ = Context("INTEGRATION: Inside of a new namespace", func() {
+	ctx := context.TODO()
+	env := SetupIntegrationTest(ctx)
+	ns := env.Namespace
+	responses := env.Responses
+
+	Describe("when no existing resources exist", func() {
+
+		It("should create and scale runners", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rs := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								Repository: "test/valid",
+								Image:      "bar",
+								Group:      "baz",
+								Env: []corev1.EnvVar{
+									{Name: "FOO", Value: "FOOVALUE"},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerDeployment resource")
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+			}
+
+			{
+				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+				Eventually(func() error {
+					var rd actionsv1alpha1.RunnerDeployment
+
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
+
+					Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerDeployment resource")
+
+					rd.Spec.Replicas = intPtr(2)
+
+					return k8sClient.Update(ctx, &rd)
+				},
+					time.Second*1, time.Millisecond*500).Should(BeNil())
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
+			}
+
+			// Scale-up to 3 replicas
+			{
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(3),
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
+						Metrics:                           nil,
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									PullRequest: &actionsv1alpha1.PullRequestSpec{
+										Types:    []string{"created"},
+										Branches: []string{"main"},
+									},
+								},
+								Amount:   1,
+								Duration: metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, hra)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test HorizontalRunnerAutoscaler resource")
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(3))
+			}
+
+			{
+				var runnerList actionsv1alpha1.RunnerList
+
+				err := k8sClient.List(ctx, &runnerList, client.InNamespace(ns.Name))
+				if err != nil {
+					logf.Log.Error(err, "list runners")
+				}
+
+				for i, r := range runnerList.Items {
+					fakeRunnerList.Add(&github3.Runner{
+						ID:     github.Int64(int64(i)),
+						Name:   github.String(r.Name),
+						OS:     github.String("linux"),
+						Status: github.String("online"),
+						Busy:   github.Bool(false),
+					})
+				}
+
+				rs, err := ghClient.ListRunners(context.Background(), "", "", "test/valid")
+				Expect(err).NotTo(HaveOccurred(), "verifying list fake runners response")
+				Expect(len(rs)).To(Equal(3), "count of fake list runners")
+			}
+
+			// Scale-down to 1 replica
+			{
+				time.Sleep(time.Second)
+
+				responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas
+
+				var hra actionsv1alpha1.HorizontalRunnerAutoscaler
+
+				err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &hra)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to get test HorizontalRunnerAutoscaler resource")
+
+				hra.Annotations = map[string]string{
+					"force-update": "1",
+				}
+
+				err = k8sClient.Update(ctx, &hra)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to get test HorizontalRunnerAutoscaler resource")
+
+				Eventually(
+					func() int {
+						var runnerSets actionsv1alpha1.RunnerReplicaSetList
+
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runners after HRA force update for scale-down")
+			}
+
+			{
+				resp, err := sendWebhook(webhookServer, "pull_request", &github.PullRequestEvent{
+					PullRequest: &github.PullRequest{
+						Base: &github.PullRequestBranch{
+							Ref: github.String("main"),
+						},
+					},
+					Repo: &github.Repository{
+						Name: github.String("test/valid"),
+						Organization: &github.Organization{
+							Name: github.String("test"),
+						},
+					},
+					Action: github.String("created"),
+				})
+
+				Expect(err).NotTo(HaveOccurred(), "failed to send pull_request event")
+
+				Expect(resp.StatusCode).To(Equal(200))
+			}
+
+			// Scale-up to 2 replicas
+			{
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runner sets after webhook")
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2), "runners after webhook")
+			}
+		})
+	})
+})

controllers/runner_controller.go

@@ -18,13 +18,15 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"reflect"
+	gogithub "github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/hash"
+	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v29/github"
-	"k8s.io/apimachinery/pkg/api/errors"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -34,30 +36,18 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github"
 )
 
 const (
 	containerName = "runner"
 	finalizerName = "runner.actions.summerwind.dev"
+
+	LabelKeyPodTemplateHash = "pod-template-hash"
+
+	retryDelayOnGitHubAPIRateLimitError = 30 * time.Second
 )
 
-type GitHubRunnerList struct {
-	TotalCount int            `json:"total_count"`
-	Runners    []GitHubRunner `json:"runners,omitempty"`
-}
-
-type GitHubRunner struct {
-	ID     int    `json:"id"`
-	Name   string `json:"name"`
-	OS     string `json:"os"`
-	Status string `json:"status"`
-}
-
-type GitHubRegistrationToken struct {
-	Token     string `json:"token"`
-	ExpiresAt string `json:"expires_at"`
-}
-
 // RunnerReconciler reconciles a Runner object
 type RunnerReconciler struct {
 	client.Client
@@ -70,8 +60,11 @@ type RunnerReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	ctx := context.Background()
@@ -82,6 +75,12 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	err := runner.Validate()
+	if err != nil {
+		log.Info("Failed to validate runner spec", "error", err.Error())
+		return ctrl.Result{}, nil
+	}
+
 	if runner.ObjectMeta.DeletionTimestamp.IsZero() {
 		finalizers, added := addFinalizer(runner.ObjectMeta.Finalizers)
 
@@ -100,14 +99,31 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		finalizers, removed := removeFinalizer(runner.ObjectMeta.Finalizers)
 
 		if removed {
-			ok, err := r.unregisterRunner(ctx, runner.Spec.Repository, runner.Name)
-			if err != nil {
-				log.Error(err, "Failed to unregister runner")
-				return ctrl.Result{}, err
-			}
+			if len(runner.Status.Registration.Token) > 0 {
+				ok, err := r.unregisterRunner(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+				if err != nil {
+					if errors.Is(err, &gogithub.RateLimitError{}) {
+						// We log the underlying error when we failed calling GitHub API to list or unregisters,
+						// or the runner is still busy.
+						log.Error(
+							err,
+							fmt.Sprintf(
+								"Failed to unregister runner due to GitHub API rate limits. Delaying retry for %s to avoid excessive GitHub API calls",
+								retryDelayOnGitHubAPIRateLimitError,
+							),
+						)
 
-			if !ok {
-				log.V(1).Info("Runner no longer exists on GitHub")
+						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+					}
+
+					return ctrl.Result{}, err
+				}
+
+				if !ok {
+					log.V(1).Info("Runner no longer exists on GitHub")
+				}
+			} else {
+				log.V(1).Info("Runner was never registered on GitHub")
 			}
 
 			newRunner := runner.DeepCopy()
@@ -118,39 +134,24 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 				return ctrl.Result{}, err
 			}
 
-			log.Info("Removed runner from GitHub", "repository", runner.Spec.Repository)
+			log.Info("Removed runner from GitHub", "repository", runner.Spec.Repository, "organization", runner.Spec.Organization)
 		}
 
 		return ctrl.Result{}, nil
 	}
 
-	if !runner.IsRegisterable() {
-		reg, err := r.newRegistration(ctx, runner.Spec.Repository)
-		if err != nil {
-			r.Recorder.Event(&runner, corev1.EventTypeWarning, "FailedUpdateRegistrationToken", "Updating registration token failed")
-			log.Error(err, "Failed to get new registration token")
-			return ctrl.Result{}, err
-		}
-
-		updated := runner.DeepCopy()
-		updated.Status.Registration = reg
-
-		if err := r.Status().Update(ctx, updated); err != nil {
-			log.Error(err, "Failed to update runner status")
-			return ctrl.Result{}, err
-		}
-
-		r.Recorder.Event(&runner, corev1.EventTypeNormal, "RegistrationTokenUpdated", "Successfully update registration token")
-		log.Info("Updated registration token", "repository", runner.Spec.Repository)
-		return ctrl.Result{}, nil
-	}
-
 	var pod corev1.Pod
 	if err := r.Get(ctx, req.NamespacedName, &pod); err != nil {
-		if !errors.IsNotFound(err) {
+		if !kerrors.IsNotFound(err) {
 			return ctrl.Result{}, err
 		}
 
+		if updated, err := r.updateRegistrationToken(ctx, runner); err != nil {
+			return ctrl.Result{}, err
+		} else if updated {
+			return ctrl.Result{Requeue: true}, nil
+		}
+
 		newPod, err := r.newPod(runner)
 		if err != nil {
 			log.Error(err, "Could not create pod")
@@ -165,7 +166,11 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodCreated", fmt.Sprintf("Created pod '%s'", newPod.Name))
 		log.Info("Created runner pod", "repository", runner.Spec.Repository)
 	} else {
-		if runner.Status.Phase != string(pod.Status.Phase) {
+		// If pod has ended up succeeded we need to restart it
+		// Happens e.g. when dind is in runner and run completes
+		restart := pod.Status.Phase == corev1.PodSucceeded
+
+		if !restart && runner.Status.Phase != string(pod.Status.Phase) {
 			updated := runner.DeepCopy()
 			updated.Status.Phase = string(pod.Status.Phase)
 			updated.Status.Reason = pod.Status.Reason
@@ -180,10 +185,40 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		}
 
 		if !pod.ObjectMeta.DeletionTimestamp.IsZero() {
-			return ctrl.Result{}, err
-		}
+			deletionTimeout := 1 * time.Minute
+			currentTime := time.Now()
+			deletionDidTimeout := currentTime.Sub(pod.DeletionTimestamp.Add(deletionTimeout)) > 0
 
-		restart := false
+			if deletionDidTimeout {
+				log.Info(
+					"Pod failed to delete itself in a timely manner. "+
+						"This is typically the case when a Kubernetes node became unreachable "+
+						"and the kube controller started evicting nodes. Forcefully deleting the pod to not get stuck.",
+					"podDeletionTimestamp", pod.DeletionTimestamp,
+					"currentTime", currentTime,
+					"configuredDeletionTimeout", deletionTimeout,
+				)
+
+				var force int64 = 0
+				// forcefully delete runner as we would otherwise get stuck if the node stays unreachable
+				if err := r.Delete(ctx, &pod, &client.DeleteOptions{GracePeriodSeconds: &force}); err != nil {
+					// probably
+					if !kerrors.IsNotFound(err) {
+						log.Error(err, "Failed to forcefully delete pod resource ...")
+						return ctrl.Result{}, err
+					}
+					// forceful deletion finally succeeded
+					return ctrl.Result{Requeue: true}, nil
+				}
+
+				r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodDeleted", fmt.Sprintf("Forcefully deleted pod '%s'", pod.Name))
+				log.Info("Forcefully deleted runner pod", "repository", runner.Spec.Repository)
+				// give kube manager a little time to forcefully delete the stuck pod
+				return ctrl.Result{RequeueAfter: 3 * time.Second}, err
+			} else {
+				return ctrl.Result{}, err
+			}
+		}
 
 		if pod.Status.Phase == corev1.PodRunning {
 			for _, status := range pod.Status.ContainerStatuses {
@@ -197,22 +232,80 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			}
 		}
 
+		if updated, err := r.updateRegistrationToken(ctx, runner); err != nil {
+			return ctrl.Result{}, err
+		} else if updated {
+			return ctrl.Result{Requeue: true}, nil
+		}
+
 		newPod, err := r.newPod(runner)
 		if err != nil {
 			log.Error(err, "Could not create pod")
 			return ctrl.Result{}, err
 		}
 
-		if pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image {
-			restart = true
-		}
-		if !reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) {
-			restart = true
-		}
-		if !restart {
-			return ctrl.Result{}, err
+		notRegistered := false
+
+		runnerBusy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+		if err != nil {
+			var e *github.RunnerNotFound
+			if errors.As(err, &e) {
+				log.V(1).Info("Failed to check if runner is busy. Either this runner has never been successfully registered to GitHub or it still needs more time.", "runnerName", runner.Name)
+
+				notRegistered = true
+			} else {
+				var e *gogithub.RateLimitError
+				if errors.As(err, &e) {
+					// We log the underlying error when we failed calling GitHub API to list or unregisters,
+					// or the runner is still busy.
+					log.Error(
+						err,
+						fmt.Sprintf(
+							"Failed to check if runner is busy due to Github API rate limit. Retrying in %s to avoid excessive GitHub API calls",
+							retryDelayOnGitHubAPIRateLimitError,
+						),
+					)
+
+					return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+				}
+
+				return ctrl.Result{}, err
+			}
 		}
 
+		// See the `newPod` function called above for more information
+		// about when this hash changes.
+		curHash := pod.Labels[LabelKeyPodTemplateHash]
+		newHash := newPod.Labels[LabelKeyPodTemplateHash]
+
+		if !runnerBusy && curHash != newHash {
+			restart = true
+		}
+
+		registrationTimeout := 10 * time.Minute
+		currentTime := time.Now()
+		registrationDidTimeout := currentTime.Sub(pod.CreationTimestamp.Add(registrationTimeout)) > 0
+
+		if notRegistered && registrationDidTimeout {
+			log.Info(
+				"Runner failed to register itself to GitHub in timely manner. "+
+					"Recreating the pod to see if it resolves the issue. "+
+					"CAUTION: If you see this a lot, you should investigate the root cause. "+
+					"See https://github.com/summerwind/actions-runner-controller/issues/288",
+				"podCreationTimestamp", pod.CreationTimestamp,
+				"currentTime", currentTime,
+				"configuredRegistrationTimeout", registrationTimeout,
+			)
+
+			restart = true
+		}
+
+		// Don't do anything if there's no need to restart the runner
+		if !restart {
+			return ctrl.Result{}, nil
+		}
+
+		// Delete current pod if recreation is needed
 		if err := r.Delete(ctx, &pod); err != nil {
 			log.Error(err, "Failed to delete pod resource")
 			return ctrl.Result{}, err
@@ -225,113 +318,73 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	return ctrl.Result{}, nil
 }
 
-func (r *RunnerReconciler) newRegistration(ctx context.Context, repo string) (v1alpha1.RunnerStatusRegistration, error) {
-	var reg v1alpha1.RunnerStatusRegistration
-
-	rt, err := r.getRegistrationToken(ctx, repo)
-	if err != nil {
-		return reg, err
-	}
-
-	expiresAt, err := time.Parse(time.RFC3339, rt.ExpiresAt)
-	if err != nil {
-		return reg, err
-	}
-
-	reg.Repository = repo
-	reg.Token = rt.Token
-	reg.ExpiresAt = metav1.NewTime(expiresAt)
-
-	return reg, err
-}
-
-func (r *RunnerReconciler) getRegistrationToken(ctx context.Context, repo string) (GitHubRegistrationToken, error) {
-	var regToken GitHubRegistrationToken
-
-	req, err := r.GitHubClient.NewRequest("POST", fmt.Sprintf("/repos/%s/actions/runners/registration-token", repo), nil)
-	if err != nil {
-		return regToken, err
-	}
-
-	res, err := r.GitHubClient.Do(ctx, req, &regToken)
-	if err != nil {
-		return regToken, err
-	}
-
-	if res.StatusCode != 201 {
-		return regToken, fmt.Errorf("unexpected status: %d", res.StatusCode)
-	}
-
-	return regToken, nil
-}
-
-func (r *RunnerReconciler) unregisterRunner(ctx context.Context, repo, name string) (bool, error) {
-	runners, err := r.listRunners(ctx, repo)
+func (r *RunnerReconciler) unregisterRunner(ctx context.Context, enterprise, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, enterprise, org, repo)
 	if err != nil {
 		return false, err
 	}
 
-	id := 0
-	for _, runner := range runners.Runners {
-		if runner.Name == name {
-			id = runner.ID
+	id := int64(0)
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			if runner.GetBusy() {
+				return false, fmt.Errorf("runner is busy")
+			}
+			id = runner.GetID()
 			break
 		}
 	}
 
-	if id == 0 {
+	if id == int64(0) {
 		return false, nil
 	}
 
-	if err := r.removeRunner(ctx, repo, id); err != nil {
+	if err := r.GitHubClient.RemoveRunner(ctx, enterprise, org, repo, id); err != nil {
 		return false, err
 	}
 
 	return true, nil
 }
 
-func (r *RunnerReconciler) listRunners(ctx context.Context, repo string) (GitHubRunnerList, error) {
-	runners := GitHubRunnerList{}
+func (r *RunnerReconciler) updateRegistrationToken(ctx context.Context, runner v1alpha1.Runner) (bool, error) {
+	if runner.IsRegisterable() {
+		return false, nil
+	}
 
-	req, err := r.GitHubClient.NewRequest("GET", fmt.Sprintf("/repos/%s/actions/runners", repo), nil)
+	log := r.Log.WithValues("runner", runner.Name)
+
+	rt, err := r.GitHubClient.GetRegistrationToken(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
 	if err != nil {
-		return runners, err
+		r.Recorder.Event(&runner, corev1.EventTypeWarning, "FailedUpdateRegistrationToken", "Updating registration token failed")
+		log.Error(err, "Failed to get new registration token")
+		return false, err
 	}
 
-	res, err := r.GitHubClient.Do(ctx, req, &runners)
-	if err != nil {
-		return runners, err
+	updated := runner.DeepCopy()
+	updated.Status.Registration = v1alpha1.RunnerStatusRegistration{
+		Organization: runner.Spec.Organization,
+		Repository:   runner.Spec.Repository,
+		Labels:       runner.Spec.Labels,
+		Token:        rt.GetToken(),
+		ExpiresAt:    metav1.NewTime(rt.GetExpiresAt().Time),
 	}
 
-	if res.StatusCode != 200 {
-		return runners, fmt.Errorf("unexpected status: %d", res.StatusCode)
+	if err := r.Status().Update(ctx, updated); err != nil {
+		log.Error(err, "Failed to update runner status")
+		return false, err
 	}
 
-	return runners, nil
-}
+	r.Recorder.Event(&runner, corev1.EventTypeNormal, "RegistrationTokenUpdated", "Successfully update registration token")
+	log.Info("Updated registration token", "repository", runner.Spec.Repository)
 
-func (r *RunnerReconciler) removeRunner(ctx context.Context, repo string, id int) error {
-	req, err := r.GitHubClient.NewRequest("DELETE", fmt.Sprintf("/repos/%s/actions/runners/%d", repo, id), nil)
-	if err != nil {
-		return err
-	}
-
-	res, err := r.GitHubClient.Do(ctx, req, nil)
-	if err != nil {
-		return err
-	}
-
-	if res.StatusCode != 204 {
-		return fmt.Errorf("unexpected status: %d", res.StatusCode)
-	}
-
-	return nil
+	return true, nil
 }
 
 func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	var (
-		privileged bool  = true
-		group      int64 = 0
+		privileged      bool = true
+		dockerdInRunner bool = runner.Spec.DockerdWithinRunnerContainer != nil && *runner.Spec.DockerdWithinRunnerContainer
+		dockerEnabled   bool = runner.Spec.DockerEnabled == nil || *runner.Spec.DockerEnabled
 	)
 
 	runnerImage := runner.Spec.Image
@@ -339,27 +392,98 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		runnerImage = r.RunnerImage
 	}
 
+	workDir := runner.Spec.WorkDir
+	if workDir == "" {
+		workDir = "/runner/_work"
+	}
+
+	runnerImagePullPolicy := runner.Spec.ImagePullPolicy
+	if runnerImagePullPolicy == "" {
+		runnerImagePullPolicy = corev1.PullAlways
+	}
+
 	env := []corev1.EnvVar{
 		{
 			Name:  "RUNNER_NAME",
 			Value: runner.Name,
 		},
+		{
+			Name:  "RUNNER_ORG",
+			Value: runner.Spec.Organization,
+		},
 		{
 			Name:  "RUNNER_REPO",
 			Value: runner.Spec.Repository,
 		},
+		{
+			Name:  "RUNNER_ENTERPRISE",
+			Value: runner.Spec.Enterprise,
+		},
+		{
+			Name:  "RUNNER_LABELS",
+			Value: strings.Join(runner.Spec.Labels, ","),
+		},
+		{
+			Name:  "RUNNER_GROUP",
+			Value: runner.Spec.Group,
+		},
 		{
 			Name:  "RUNNER_TOKEN",
 			Value: runner.Status.Registration.Token,
 		},
+		{
+			Name:  "DOCKERD_IN_RUNNER",
+			Value: fmt.Sprintf("%v", dockerdInRunner),
+		},
+		{
+			Name:  "GITHUB_URL",
+			Value: r.GitHubClient.GithubBaseURL,
+		},
+		{
+			Name:  "RUNNER_WORKDIR",
+			Value: workDir,
+		},
 	}
 
 	env = append(env, runner.Spec.Env...)
+
+	labels := map[string]string{}
+
+	for k, v := range runner.Labels {
+		labels[k] = v
+	}
+
+	// This implies that...
+	//
+	// (1) We recreate the runner pod whenever the runner has changes in:
+	// - metadata.labels (excluding "runner-template-hash" added by the parent RunnerReplicaSet
+	// - metadata.annotations
+	// - metadata.spec (including image, env, organization, repository, group, and so on)
+	// - GithubBaseURL setting of the controller (can be configured via GITHUB_ENTERPRISE_URL)
+	//
+	// (2) We don't recreate the runner pod when there are changes in:
+	// - runner.status.registration.token
+	//   - This token expires and changes hourly, but you don't need to recreate the pod due to that.
+	//     It's the opposite.
+	//     An unexpired token is required only when the runner agent is registering itself on launch.
+	//
+	//     In other words, the registered runner doesn't get invalidated on registration token expiration.
+	//     A registered runner's session and the a registration token seem to have two different and independent
+	//     lifecycles.
+	//
+	//     See https://github.com/summerwind/actions-runner-controller/issues/143 for more context.
+	labels[LabelKeyPodTemplateHash] = hash.FNVHashStringObjects(
+		filterLabels(runner.Labels, LabelKeyRunnerTemplateHash),
+		runner.Annotations,
+		runner.Spec,
+		r.GitHubClient.GithubBaseURL,
+	)
+
 	pod := corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        runner.Name,
 			Namespace:   runner.Namespace,
-			Labels:      runner.Labels,
+			Labels:      labels,
 			Annotations: runner.Annotations,
 		},
 		Spec: corev1.PodSpec{
@@ -368,47 +492,110 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 				{
 					Name:            containerName,
 					Image:           runnerImage,
-					ImagePullPolicy: "Always",
+					ImagePullPolicy: runnerImagePullPolicy,
 					Env:             env,
 					EnvFrom:         runner.Spec.EnvFrom,
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							Name:      "docker",
-							MountPath: "/var/run",
-						},
-					},
 					SecurityContext: &corev1.SecurityContext{
-						RunAsGroup: &group,
+						// Runner need to run privileged if it contains DinD
+						Privileged: runner.Spec.DockerdWithinRunnerContainer,
 					},
 					Resources: runner.Spec.Resources,
 				},
-				{
-					Name:  "docker",
-					Image: r.DockerImage,
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							Name:      "docker",
-							MountPath: "/var/run",
-						},
-					},
-					SecurityContext: &corev1.SecurityContext{
-						Privileged: &privileged,
-					},
-				},
-			},
-			Volumes: []corev1.Volume{
-				corev1.Volume{
-					Name: "docker",
-					VolumeSource: corev1.VolumeSource{
-						EmptyDir: &corev1.EmptyDirVolumeSource{},
-					},
-				},
 			},
 		},
 	}
 
+	if !dockerdInRunner && dockerEnabled {
+		runnerVolumeName := "runner"
+		runnerVolumeMountPath := "/runner"
+
+		pod.Spec.Volumes = []corev1.Volume{
+			{
+				Name: "work",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: runnerVolumeName,
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "certs-client",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+		}
+		pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+			{
+				Name:      "work",
+				MountPath: workDir,
+			},
+			{
+				Name:      runnerVolumeName,
+				MountPath: runnerVolumeMountPath,
+			},
+			{
+				Name:      "certs-client",
+				MountPath: "/certs/client",
+				ReadOnly:  true,
+			},
+		}
+		pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, []corev1.EnvVar{
+			{
+				Name:  "DOCKER_HOST",
+				Value: "tcp://localhost:2376",
+			},
+			{
+				Name:  "DOCKER_TLS_VERIFY",
+				Value: "1",
+			},
+			{
+				Name:  "DOCKER_CERT_PATH",
+				Value: "/certs/client",
+			},
+		}...)
+		pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
+			Name:  "docker",
+			Image: r.DockerImage,
+			VolumeMounts: []corev1.VolumeMount{
+				{
+					Name:      "work",
+					MountPath: workDir,
+				},
+				{
+					Name:      runnerVolumeName,
+					MountPath: runnerVolumeMountPath,
+				},
+				{
+					Name:      "certs-client",
+					MountPath: "/certs/client",
+				},
+			},
+			Env: []corev1.EnvVar{
+				{
+					Name:  "DOCKER_TLS_CERTDIR",
+					Value: "/certs",
+				},
+			},
+			SecurityContext: &corev1.SecurityContext{
+				Privileged: &privileged,
+			},
+			Resources: runner.Spec.DockerdContainerResources,
+		})
+
+	}
+
 	if len(runner.Spec.Containers) != 0 {
 		pod.Spec.Containers = runner.Spec.Containers
+		for i := 0; i < len(pod.Spec.Containers); i++ {
+			if pod.Spec.Containers[i].Name == containerName {
+				pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, env...)
+			}
+		}
 	}
 
 	if len(runner.Spec.VolumeMounts) != 0 {
@@ -416,7 +603,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	}
 
 	if len(runner.Spec.Volumes) != 0 {
-		pod.Spec.Volumes = append(runner.Spec.Volumes, runner.Spec.Volumes...)
+		pod.Spec.Volumes = append(pod.Spec.Volumes, runner.Spec.Volumes...)
 	}
 	if len(runner.Spec.InitContainers) != 0 {
 		pod.Spec.InitContainers = append(pod.Spec.InitContainers, runner.Spec.InitContainers...)

controllers/runnerdeployment_controller.go

@@ -21,6 +21,9 @@ import (
 	"fmt"
 	"hash/fnv"
 	"sort"
+	"time"
+
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/go-logr/logr"
@@ -51,13 +54,15 @@ type RunnerDeploymentReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	ctx := context.Background()
-	log := r.Log.WithValues("runnerreplicaset", req.NamespacedName)
+	log := r.Log.WithValues("runnerdeployment", req.NamespacedName)
 
 	var rd v1alpha1.RunnerDeployment
 	if err := r.Get(ctx, req.NamespacedName, &rd); err != nil {
@@ -93,13 +98,15 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 
 	desiredRS, err := r.newRunnerReplicaSet(rd)
 	if err != nil {
+		r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
+
 		log.Error(err, "Could not create runnerreplicaset")
 
 		return ctrl.Result{}, err
 	}
 
 	if newestSet == nil {
-		if err := r.Client.Create(ctx, &desiredRS); err != nil {
+		if err := r.Client.Create(ctx, desiredRS); err != nil {
 			log.Error(err, "Failed to create runnerreplicaset resource")
 
 			return ctrl.Result{}, err
@@ -115,7 +122,7 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		return ctrl.Result{}, nil
 	}
 
-	desiredTemplateHash, ok := getTemplateHash(&desiredRS)
+	desiredTemplateHash, ok := getTemplateHash(desiredRS)
 	if !ok {
 		log.Info("Failed to get template hash of desired runnerreplicaset resource. It must be in an invalid state. Please manually delete the runnerreplicaset so that it is recreated")
 
@@ -123,18 +130,25 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	}
 
 	if newestTemplateHash != desiredTemplateHash {
-		if err := r.Client.Create(ctx, &desiredRS); err != nil {
+		if err := r.Client.Create(ctx, desiredRS); err != nil {
 			log.Error(err, "Failed to create runnerreplicaset resource")
 
 			return ctrl.Result{}, err
 		}
 
-		return ctrl.Result{}, nil
+		// We requeue in order to clean up old runner replica sets later.
+		// Otherwise, they aren't cleaned up until the next re-sync interval.
+		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
 	}
 
+	const defaultReplicas = 1
+
+	currentDesiredReplicas := getIntOrDefault(newestSet.Spec.Replicas, defaultReplicas)
+	newDesiredReplicas := getIntOrDefault(desiredRS.Spec.Replicas, defaultReplicas)
+
 	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
-	if newestSet.Spec.Replicas != desiredRS.Spec.Replicas {
-		newestSet.Spec.Replicas = desiredRS.Spec.Replicas
+	if currentDesiredReplicas != newDesiredReplicas {
+		newestSet.Spec.Replicas = &newDesiredReplicas
 
 		if err := r.Client.Update(ctx, newestSet); err != nil {
 			log.Error(err, "Failed to update runnerreplicaset resource")
@@ -142,25 +156,60 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 			return ctrl.Result{}, err
 		}
 
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, err
 	}
 
-	for i := range oldSets {
-		rs := oldSets[i]
+	// Do we old runner replica sets that should eventually deleted?
+	if len(oldSets) > 0 {
+		readyReplicas := newestSet.Status.ReadyReplicas
 
-		if err := r.Client.Delete(ctx, &rs); err != nil {
-			log.Error(err, "Failed to delete runner resource")
+		if readyReplicas < currentDesiredReplicas {
+			log.WithValues("runnerreplicaset", types.NamespacedName{
+				Namespace: newestSet.Namespace,
+				Name:      newestSet.Name,
+			}).
+				Info("Waiting until the newest runner replica set to be 100% available")
+
+			return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+		}
+
+		for i := range oldSets {
+			rs := oldSets[i]
+
+			if err := r.Client.Delete(ctx, &rs); err != nil {
+				log.Error(err, "Failed to delete runnerreplicaset resource")
+
+				return ctrl.Result{}, err
+			}
+
+			r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerReplicaSetDeleted", fmt.Sprintf("Deleted runnerreplicaset '%s'", rs.Name))
+
+			log.Info("Deleted runnerreplicaset", "runnerdeployment", rd.ObjectMeta.Name, "runnerreplicaset", rs.Name)
+		}
+	}
+
+	if rd.Spec.Replicas == nil && desiredRS.Spec.Replicas != nil {
+		updated := rd.DeepCopy()
+		updated.Status.Replicas = desiredRS.Spec.Replicas
+
+		if err := r.Status().Update(ctx, updated); err != nil {
+			log.Error(err, "Failed to update runnerdeployment status")
 
 			return ctrl.Result{}, err
 		}
-
-		r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerReplicaSetDeleted", fmt.Sprintf("Deleted runnerreplicaset '%s'", rs.Name))
-		log.Info("Deleted runnerreplicaset", "runnerdeployment", rd.ObjectMeta.Name, "runnerreplicaset", rs.Name)
 	}
 
 	return ctrl.Result{}, nil
 }
 
+func getIntOrDefault(p *int, d int) int {
+	if p == nil {
+		return d
+	}
+
+	return *p
+}
+
 func getTemplateHash(rs *v1alpha1.RunnerReplicaSet) (string, bool) {
 	hash, ok := rs.Labels[LabelKeyRunnerTemplateHash]
 
@@ -207,7 +256,7 @@ func CloneAndAddLabel(labels map[string]string, labelKey, labelValue string) map
 	return newLabels
 }
 
-func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeployment) (v1alpha1.RunnerReplicaSet, error) {
+func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeployment) (*v1alpha1.RunnerReplicaSet, error) {
 	newRSTemplate := *rd.Spec.Template.DeepCopy()
 	templateHash := ComputeHash(&newRSTemplate)
 	// Add template hash label to selector.
@@ -229,10 +278,10 @@ func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeplo
 	}
 
 	if err := ctrl.SetControllerReference(&rd, &rs, r.Scheme); err != nil {
-		return rs, err
+		return &rs, err
 	}
 
-	return rs, nil
+	return &rs, nil
 }
 
 func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {

controllers/runnerreplicaset_controller.go

@@ -18,10 +18,13 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	gogithub "github.com/google/go-github/v33/github"
+	"time"
 
 	"github.com/go-logr/logr"
-	"k8s.io/apimachinery/pkg/api/errors"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -31,24 +34,28 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github"
 )
 
 // RunnerReplicaSetReconciler reconciles a Runner object
 type RunnerReplicaSetReconciler struct {
 	client.Client
-	Log      logr.Logger
-	Recorder record.EventRecorder
-	Scheme   *runtime.Scheme
+	Log          logr.Logger
+	Recorder     record.EventRecorder
+	Scheme       *runtime.Scheme
+	GitHubClient *github.Client
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	ctx := context.Background()
-	log := r.Log.WithValues("runner", req.NamespacedName)
+	log := r.Log.WithValues("runnerreplicaset", req.NamespacedName)
 
 	var rs v1alpha1.RunnerReplicaSet
 	if err := r.Get(ctx, req.NamespacedName, &rs); err != nil {
@@ -61,7 +68,7 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 
 	var allRunners v1alpha1.RunnerList
 	if err := r.List(ctx, &allRunners, client.InNamespace(req.Namespace)); err != nil {
-		if !errors.IsNotFound(err) {
+		if !kerrors.IsNotFound(err) {
 			return ctrl.Result{}, err
 		}
 	}
@@ -95,8 +102,64 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	if available > desired {
 		n := available - desired
 
+		// get runners that are currently not busy
+		var notBusy []v1alpha1.Runner
+		for _, runner := range myRunners {
+			busy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+			if err != nil {
+				notRegistered := false
+
+				var e *github.RunnerNotFound
+				if errors.As(err, &e) {
+					log.V(1).Info("Failed to check if runner is busy. Either this runner has never been successfully registered to GitHub or has not managed yet to, and therefore we prioritize it for deletion", "runnerName", runner.Name)
+					notRegistered = true
+				} else {
+					var e *gogithub.RateLimitError
+					if errors.As(err, &e) {
+						// We log the underlying error when we failed calling GitHub API to list or unregisters,
+						// or the runner is still busy.
+						log.Error(
+							err,
+							fmt.Sprintf(
+								"Failed to check if runner is busy due to GitHub API rate limit. Retrying in %s to avoid excessive GitHub API calls",
+								retryDelayOnGitHubAPIRateLimitError,
+							),
+						)
+
+						return ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+					}
+
+					return ctrl.Result{}, err
+				}
+
+				registrationTimeout := 15 * time.Minute
+				currentTime := time.Now()
+				registrationDidTimeout := currentTime.Sub(runner.CreationTimestamp.Add(registrationTimeout)) > 0
+
+				if notRegistered && registrationDidTimeout {
+					log.Info(
+						"Runner failed to register itself to GitHub in timely manner. "+
+							"Recreating the pod to see if it resolves the issue. "+
+							"CAUTION: If you see this a lot, you should investigate the root cause. "+
+							"See https://github.com/summerwind/actions-runner-controller/issues/288",
+						"runnerCreationTimestamp", runner.CreationTimestamp,
+						"currentTime", currentTime,
+						"configuredRegistrationTimeout", registrationTimeout,
+					)
+
+					notBusy = append(notBusy, runner)
+				}
+			} else if !busy {
+				notBusy = append(notBusy, runner)
+			}
+		}
+
+		if len(notBusy) < n {
+			n = len(notBusy)
+		}
+
 		for i := 0; i < n; i++ {
-			if err := r.Client.Delete(ctx, &myRunners[i]); err != nil {
+			if err := r.Client.Delete(ctx, &notBusy[i]); client.IgnoreNotFound(err) != nil {
 				log.Error(err, "Failed to delete runner resource")
 
 				return ctrl.Result{}, err

controllers/runnerreplicaset_controller_test.go

@@ -3,11 +3,14 @@ package controllers
 import (
 	"context"
 	"math/rand"
+	"net/http/httptest"
 	"time"
 
+	"github.com/google/go-github/v33/github"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -17,6 +20,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+)
+
+var (
+	runnersList *fake.RunnersList
+	server      *httptest.Server
 )
 
 // SetupTest will set up a testing environment.
@@ -41,11 +50,16 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+		ghClient := newGithubClient(server)
+
 		controller := &RunnerReplicaSetReconciler{
-			Client:   mgr.GetClient(),
-			Scheme:   scheme.Scheme,
-			Log:      logf.Log,
-			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -61,6 +75,7 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 	AfterEach(func() {
 		close(stopCh)
 
+		server.Close()
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
 	})
@@ -124,6 +139,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
@@ -155,6 +180,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
@@ -186,6 +221,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(0))

controllers/suite_test.go

@@ -17,6 +17,8 @@ limitations under the License.
 package controllers
 
 import (
+	"github.com/onsi/ginkgo/config"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -43,6 +45,8 @@ var testEnv *envtest.Environment
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
 
+	config.GinkgoConfig.FocusString = os.Getenv("GINKGO_FOCUS")
+
 	RunSpecsWithDefaultAndCustomReporters(t,
 		"Controller Suite",
 		[]Reporter{envtest.NewlineReporter{}})

controllers/utils.go

@@ -0,0 +1,13 @@
+package controllers
+
+func filterLabels(labels map[string]string, filter string) map[string]string {
+	filtered := map[string]string{}
+
+	for k, v := range labels {
+		if k != filter {
+			filtered[k] = v
+		}
+	}
+
+	return filtered
+}

controllers/utils_test.go

@@ -0,0 +1,34 @@
+package controllers
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_filterLabels(t *testing.T) {
+	type args struct {
+		labels map[string]string
+		filter string
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[string]string
+	}{
+		{
+			name: "ok",
+			args: args{
+				labels: map[string]string{LabelKeyRunnerTemplateHash: "abc", LabelKeyPodTemplateHash: "def"},
+				filter: LabelKeyRunnerTemplateHash,
+			},
+			want: map[string]string{LabelKeyPodTemplateHash: "def"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := filterLabels(tt.args.labels, tt.args.filter); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("filterLabels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

github/fake/fake.go

@@ -0,0 +1,211 @@
+package fake
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+)
+
+const (
+	RegistrationToken = "fake-registration-token"
+
+	RunnersListBody = `
+{
+  "total_count": 2,
+  "runners": [
+    {"id": 1, "name": "test1", "os": "linux", "status": "online", "busy": false},
+    {"id": 2, "name": "test2", "os": "linux", "status": "offline", "busy": false}
+  ]
+}
+`
+)
+
+type ListRunnersHandler struct {
+	Status int
+	Body   string
+}
+
+func (h *ListRunnersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.WriteHeader(h.Status)
+	fmt.Fprintf(w, h.Body)
+}
+
+type Handler struct {
+	Status int
+	Body   string
+}
+
+func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.WriteHeader(h.Status)
+	fmt.Fprintf(w, h.Body)
+}
+
+type MapHandler struct {
+	Status int
+	Bodies map[int]string
+}
+
+func (h *MapHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// Parse out int key from URL path
+	key, err := strconv.Atoi(strings.TrimFunc(req.URL.Path, func(r rune) bool { return !unicode.IsNumber(r) }))
+	if err != nil {
+		w.WriteHeader(400)
+	} else if body := h.Bodies[key]; len(body) == 0 {
+		w.WriteHeader(404)
+	} else {
+		w.WriteHeader(h.Status)
+		fmt.Fprintf(w, body)
+	}
+}
+
+type ServerConfig struct {
+	*FixedResponses
+}
+
+// NewServer creates a fake server for running unit tests
+func NewServer(opts ...Option) *httptest.Server {
+	config := ServerConfig{
+		FixedResponses: &FixedResponses{},
+	}
+
+	for _, o := range opts {
+		o(&config)
+	}
+
+	routes := map[string]http.Handler{
+		// For CreateRegistrationToken
+		"/repos/test/valid/actions/runners/registration-token": &Handler{
+			Status: http.StatusCreated,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/repos/test/invalid/actions/runners/registration-token": &Handler{
+			Status: http.StatusOK,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/repos/test/error/actions/runners/registration-token": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+		"/orgs/test/actions/runners/registration-token": &Handler{
+			Status: http.StatusCreated,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/orgs/invalid/actions/runners/registration-token": &Handler{
+			Status: http.StatusOK,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/orgs/error/actions/runners/registration-token": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+		"/enterprises/test/actions/runners/registration-token": &Handler{
+			Status: http.StatusCreated,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/enterprises/invalid/actions/runners/registration-token": &Handler{
+			Status: http.StatusOK,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/enterprises/error/actions/runners/registration-token": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+
+		// For ListRunners
+		"/repos/test/valid/actions/runners": config.FixedResponses.ListRunners,
+		"/repos/test/invalid/actions/runners": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/repos/test/error/actions/runners": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+		"/orgs/test/actions/runners": &Handler{
+			Status: http.StatusOK,
+			Body:   RunnersListBody,
+		},
+		"/orgs/invalid/actions/runners": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/orgs/error/actions/runners": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+		"/enterprises/test/actions/runners": &Handler{
+			Status: http.StatusOK,
+			Body:   RunnersListBody,
+		},
+		"/enterprises/invalid/actions/runners": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/enterprises/error/actions/runners": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+
+		// For RemoveRunner
+		"/repos/test/valid/actions/runners/1": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/repos/test/invalid/actions/runners/1": &Handler{
+			Status: http.StatusOK,
+			Body:   "",
+		},
+		"/repos/test/error/actions/runners/1": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+		"/orgs/test/actions/runners/1": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/orgs/invalid/actions/runners/1": &Handler{
+			Status: http.StatusOK,
+			Body:   "",
+		},
+		"/orgs/error/actions/runners/1": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+		"/enterprises/test/actions/runners/1": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/enterprises/invalid/actions/runners/1": &Handler{
+			Status: http.StatusOK,
+			Body:   "",
+		},
+		"/enterprises/error/actions/runners/1": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
+
+		// For auto-scaling based on the number of queued(pending) workflow runs
+		"/repos/test/valid/actions/runs": config.FixedResponses.ListRepositoryWorkflowRuns,
+
+		// For auto-scaling based on the number of queued(pending) workflow jobs
+		"/repos/test/valid/actions/runs/": config.FixedResponses.ListWorkflowJobs,
+	}
+
+	mux := http.NewServeMux()
+	for path, handler := range routes {
+		mux.Handle(path, handler)
+	}
+
+	return httptest.NewServer(mux)
+}
+
+func DefaultListRunnersHandler() *ListRunnersHandler {
+	return &ListRunnersHandler{
+		Status: http.StatusOK,
+		Body:   RunnersListBody,
+	}
+}

github/fake/options.go

@@ -0,0 +1,44 @@
+package fake
+
+import "net/http"
+
+type FixedResponses struct {
+	ListRepositoryWorkflowRuns *Handler
+	ListWorkflowJobs           *MapHandler
+	ListRunners                http.Handler
+}
+
+type Option func(*ServerConfig)
+
+func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListRepositoryWorkflowRuns = &Handler{
+			Status: status,
+			Body:   body,
+		}
+	}
+}
+
+func WithListWorkflowJobsResponse(status int, bodies map[int]string) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListWorkflowJobs = &MapHandler{
+			Status: status,
+			Bodies: bodies,
+		}
+	}
+}
+
+func WithListRunnersResponse(status int, body string) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListRunners = &ListRunnersHandler{
+			Status: status,
+			Body:   body,
+		}
+	}
+}
+
+func WithFixedResponses(responses *FixedResponses) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses = responses
+	}
+}

github/fake/runners.go

@@ -0,0 +1,74 @@
+package fake
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+
+	"github.com/google/go-github/v33/github"
+	"github.com/gorilla/mux"
+)
+
+type RunnersList struct {
+	runners []*github.Runner
+}
+
+func NewRunnersList() *RunnersList {
+	return &RunnersList{
+		runners: make([]*github.Runner, 0),
+	}
+}
+
+func (r *RunnersList) Add(runner *github.Runner) {
+	if !exists(r.runners, runner) {
+		r.runners = append(r.runners, runner)
+	}
+}
+
+func (r *RunnersList) GetServer() *httptest.Server {
+	router := mux.NewRouter()
+
+	router.Handle("/repos/{owner}/{repo}/actions/runners", r.HandleList())
+	router.Handle("/repos/{owner}/{repo}/actions/runners/{id}", r.handleRemove())
+	router.Handle("/orgs/{org}/actions/runners", r.HandleList())
+	router.Handle("/orgs/{org}/actions/runners/{id}", r.handleRemove())
+
+	return httptest.NewServer(router)
+}
+
+func (r *RunnersList) HandleList() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		j, err := json.Marshal(github.Runners{
+			TotalCount: len(r.runners),
+			Runners:    r.runners,
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		w.WriteHeader(http.StatusOK)
+		w.Write(j)
+	}
+}
+
+func (r *RunnersList) handleRemove() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		vars := mux.Vars(res)
+		for i, runner := range r.runners {
+			if runner.ID != nil && vars["id"] == strconv.FormatInt(*runner.ID, 10) {
+				r.runners = append(r.runners[:i], r.runners[i+1:]...)
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func exists(runners []*github.Runner, runner *github.Runner) bool {
+	for _, r := range runners {
+		if *r.Name == *runner.Name {
+			return true
+		}
+	}
+	return false
+}

github/github.go

@@ -0,0 +1,304 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/bradleyfalzon/ghinstallation"
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/github/metrics"
+	"golang.org/x/oauth2"
+)
+
+// Config contains configuration for Github client
+type Config struct {
+	EnterpriseURL     string `split_words:"true"`
+	AppID             int64  `split_words:"true"`
+	AppInstallationID int64  `split_words:"true"`
+	AppPrivateKey     string `split_words:"true"`
+	Token             string
+}
+
+// Client wraps GitHub client with some additional
+type Client struct {
+	*github.Client
+	regTokens map[string]*github.RegistrationToken
+	mu        sync.Mutex
+	// GithubBaseURL to Github without API suffix.
+	GithubBaseURL string
+}
+
+// NewClient creates a Github Client
+func (c *Config) NewClient() (*Client, error) {
+	var transport http.RoundTripper
+	if len(c.Token) > 0 {
+		transport = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(&oauth2.Token{AccessToken: c.Token})).Transport
+	} else {
+		tr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.AppID, c.AppInstallationID, c.AppPrivateKey)
+		if err != nil {
+			return nil, fmt.Errorf("authentication failed: %v", err)
+		}
+		if len(c.EnterpriseURL) > 0 {
+			githubAPIURL, err := getEnterpriseApiUrl(c.EnterpriseURL)
+			if err != nil {
+				return nil, fmt.Errorf("enterprise url incorrect: %v", err)
+			}
+			tr.BaseURL = githubAPIURL
+		}
+		transport = tr
+	}
+	transport = metrics.Transport{Transport: transport}
+	httpClient := &http.Client{Transport: transport}
+
+	var client *github.Client
+	var githubBaseURL string
+	if len(c.EnterpriseURL) > 0 {
+		var err error
+		client, err = github.NewEnterpriseClient(c.EnterpriseURL, c.EnterpriseURL, httpClient)
+		if err != nil {
+			return nil, fmt.Errorf("enterprise client creation failed: %v", err)
+		}
+		githubBaseURL = fmt.Sprintf("%s://%s%s", client.BaseURL.Scheme, client.BaseURL.Host, strings.TrimSuffix(client.BaseURL.Path, "api/v3/"))
+	} else {
+		client = github.NewClient(httpClient)
+		githubBaseURL = "https://github.com/"
+	}
+
+	return &Client{
+		Client:        client,
+		regTokens:     map[string]*github.RegistrationToken{},
+		mu:            sync.Mutex{},
+		GithubBaseURL: githubBaseURL,
+	}, nil
+}
+
+// GetRegistrationToken returns a registration token tied with the name of repository and runner.
+func (c *Client) GetRegistrationToken(ctx context.Context, enterprise, org, repo, name string) (*github.RegistrationToken, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	key := getRegistrationKey(org, repo, enterprise)
+	rt, ok := c.regTokens[key]
+
+	if ok && rt.GetExpiresAt().After(time.Now()) {
+		return rt, nil
+	}
+
+	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
+
+	if err != nil {
+		return rt, err
+	}
+
+	rt, res, err := c.createRegistrationToken(ctx, enterprise, owner, repo)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create registration token: %v", err)
+	}
+
+	if res.StatusCode != 201 {
+		return nil, fmt.Errorf("unexpected status: %d", res.StatusCode)
+	}
+
+	c.regTokens[key] = rt
+	go func() {
+		c.cleanup()
+	}()
+
+	return rt, nil
+}
+
+// RemoveRunner removes a runner with specified runner ID from repository.
+func (c *Client) RemoveRunner(ctx context.Context, enterprise, org, repo string, runnerID int64) error {
+	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
+
+	if err != nil {
+		return err
+	}
+
+	res, err := c.removeRunner(ctx, enterprise, owner, repo, runnerID)
+
+	if err != nil {
+		return fmt.Errorf("failed to remove runner: %w", err)
+	}
+
+	if res.StatusCode != 204 {
+		return fmt.Errorf("unexpected status: %d", res.StatusCode)
+	}
+
+	return nil
+}
+
+// ListRunners returns a list of runners of specified owner/repository name.
+func (c *Client) ListRunners(ctx context.Context, enterprise, org, repo string) ([]*github.Runner, error) {
+	enterprise, owner, repo, err := getEnterpriseOrganisationAndRepo(enterprise, org, repo)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var runners []*github.Runner
+
+	opts := github.ListOptions{PerPage: 10}
+	for {
+		list, res, err := c.listRunners(ctx, enterprise, owner, repo, &opts)
+
+		if err != nil {
+			return runners, fmt.Errorf("failed to list runners: %w", err)
+		}
+
+		runners = append(runners, list.Runners...)
+		if res.NextPage == 0 {
+			break
+		}
+		opts.Page = res.NextPage
+	}
+
+	return runners, nil
+}
+
+// cleanup removes expired registration tokens.
+func (c *Client) cleanup() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	for key, rt := range c.regTokens {
+		if rt.GetExpiresAt().Before(time.Now()) {
+			delete(c.regTokens, key)
+		}
+	}
+}
+
+// wrappers for github functions (switch between enterprise/organization/repository mode)
+// so the calling functions don't need to switch and their code is a bit cleaner
+
+func (c *Client) createRegistrationToken(ctx context.Context, enterprise, org, repo string) (*github.RegistrationToken, *github.Response, error) {
+	if len(repo) > 0 {
+		return c.Client.Actions.CreateRegistrationToken(ctx, org, repo)
+	}
+	if len(org) > 0 {
+		return c.Client.Actions.CreateOrganizationRegistrationToken(ctx, org)
+	}
+	return c.Client.Enterprise.CreateRegistrationToken(ctx, enterprise)
+}
+
+func (c *Client) removeRunner(ctx context.Context, enterprise, org, repo string, runnerID int64) (*github.Response, error) {
+	if len(repo) > 0 {
+		return c.Client.Actions.RemoveRunner(ctx, org, repo, runnerID)
+	}
+	if len(org) > 0 {
+		return c.Client.Actions.RemoveOrganizationRunner(ctx, org, runnerID)
+	}
+	return c.Client.Enterprise.RemoveRunner(ctx, enterprise, runnerID)
+}
+
+func (c *Client) listRunners(ctx context.Context, enterprise, org, repo string, opts *github.ListOptions) (*github.Runners, *github.Response, error) {
+	if len(repo) > 0 {
+		return c.Client.Actions.ListRunners(ctx, org, repo, opts)
+	}
+	if len(org) > 0 {
+		return c.Client.Actions.ListOrganizationRunners(ctx, org, opts)
+	}
+	return c.Client.Enterprise.ListRunners(ctx, enterprise, opts)
+}
+
+func (c *Client) ListRepositoryWorkflowRuns(ctx context.Context, user string, repoName string) ([]*github.WorkflowRun, error) {
+	c.Client.Actions.ListRepositoryWorkflowRuns(ctx, user, repoName, nil)
+
+	var workflowRuns []*github.WorkflowRun
+
+	opts := github.ListWorkflowRunsOptions{
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+
+	for {
+		list, res, err := c.Client.Actions.ListRepositoryWorkflowRuns(ctx, user, repoName, &opts)
+
+		if err != nil {
+			return workflowRuns, fmt.Errorf("failed to list workflow runs: %v", err)
+		}
+
+		workflowRuns = append(workflowRuns, list.WorkflowRuns...)
+		if res.NextPage == 0 {
+			break
+		}
+		opts.Page = res.NextPage
+	}
+
+	return workflowRuns, nil
+}
+
+// Validates enterprise, organisation and repo arguments. Both are optional, but at least one should be specified
+func getEnterpriseOrganisationAndRepo(enterprise, org, repo string) (string, string, string, error) {
+	if len(repo) > 0 {
+		owner, repository, err := splitOwnerAndRepo(repo)
+		return "", owner, repository, err
+	}
+	if len(org) > 0 {
+		return "", org, "", nil
+	}
+	if len(enterprise) > 0 {
+		return enterprise, "", "", nil
+	}
+	return "", "", "", fmt.Errorf("enterprise, organization and repository are all empty")
+}
+
+func getRegistrationKey(org, repo, enterprise string) string {
+	return fmt.Sprintf("org=%s,repo=%s,enterprise=%s", org, repo, enterprise)
+}
+
+func splitOwnerAndRepo(repo string) (string, string, error) {
+	chunk := strings.Split(repo, "/")
+	if len(chunk) != 2 {
+		return "", "", fmt.Errorf("invalid repository name: '%s'", repo)
+	}
+	return chunk[0], chunk[1], nil
+}
+
+func getEnterpriseApiUrl(baseURL string) (string, error) {
+	baseEndpoint, err := url.Parse(baseURL)
+	if err != nil {
+		return "", err
+	}
+	if !strings.HasSuffix(baseEndpoint.Path, "/") {
+		baseEndpoint.Path += "/"
+	}
+	if !strings.HasSuffix(baseEndpoint.Path, "/api/v3/") &&
+		!strings.HasPrefix(baseEndpoint.Host, "api.") &&
+		!strings.Contains(baseEndpoint.Host, ".api.") {
+		baseEndpoint.Path += "api/v3/"
+	}
+
+	// Trim trailing slash, otherwise there's double slash added to token endpoint
+	return fmt.Sprintf("%s://%s%s", baseEndpoint.Scheme, baseEndpoint.Host, strings.TrimSuffix(baseEndpoint.Path, "/")), nil
+}
+
+type RunnerNotFound struct {
+	runnerName string
+}
+
+func (e *RunnerNotFound) Error() string {
+	return fmt.Sprintf("runner %q not found", e.runnerName)
+}
+
+func (r *Client) IsRunnerBusy(ctx context.Context, enterprise, org, repo, name string) (bool, error) {
+	runners, err := r.ListRunners(ctx, enterprise, org, repo)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, &RunnerNotFound{runnerName: name}
+}

github/github_test.go

@@ -0,0 +1,154 @@
+package github
+
+import (
+	"context"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+)
+
+var server *httptest.Server
+
+func newTestClient() *Client {
+	c := Config{
+		Token: "token",
+	}
+	client, err := c.NewClient()
+	if err != nil {
+		panic(err)
+	}
+
+	baseURL, err := url.Parse(server.URL + "/")
+	if err != nil {
+		panic(err)
+	}
+	client.Client.BaseURL = baseURL
+
+	return client
+}
+
+func TestMain(m *testing.M) {
+	res := &fake.FixedResponses{
+		ListRunners: fake.DefaultListRunnersHandler(),
+	}
+	server = fake.NewServer(fake.WithFixedResponses(res))
+	defer server.Close()
+	m.Run()
+}
+
+func TestGetRegistrationToken(t *testing.T) {
+	tests := []struct {
+		enterprise string
+		org        string
+		repo       string
+		token      string
+		err        bool
+	}{
+		{enterprise: "", org: "", repo: "test/valid", token: fake.RegistrationToken, err: false},
+		{enterprise: "", org: "", repo: "test/invalid", token: "", err: true},
+		{enterprise: "", org: "", repo: "test/error", token: "", err: true},
+		{enterprise: "", org: "test", repo: "", token: fake.RegistrationToken, err: false},
+		{enterprise: "", org: "invalid", repo: "", token: "", err: true},
+		{enterprise: "", org: "error", repo: "", token: "", err: true},
+		{enterprise: "test", org: "", repo: "", token: fake.RegistrationToken, err: false},
+		{enterprise: "invalid", org: "", repo: "", token: "", err: true},
+		{enterprise: "error", org: "", repo: "", token: "", err: true},
+	}
+
+	client := newTestClient()
+	for i, tt := range tests {
+		rt, err := client.GetRegistrationToken(context.Background(), tt.enterprise, tt.org, tt.repo, "test")
+		if !tt.err && err != nil {
+			t.Errorf("[%d] unexpected error: %v", i, err)
+		}
+		if tt.token != rt.GetToken() {
+			t.Errorf("[%d] unexpected token: %v", i, rt.GetToken())
+		}
+	}
+}
+
+func TestListRunners(t *testing.T) {
+	tests := []struct {
+		enterprise string
+		org        string
+		repo       string
+		length     int
+		err        bool
+	}{
+		{enterprise: "", org: "", repo: "test/valid", length: 2, err: false},
+		{enterprise: "", org: "", repo: "test/invalid", length: 0, err: true},
+		{enterprise: "", org: "", repo: "test/error", length: 0, err: true},
+		{enterprise: "", org: "test", repo: "", length: 2, err: false},
+		{enterprise: "", org: "invalid", repo: "", length: 0, err: true},
+		{enterprise: "", org: "error", repo: "", length: 0, err: true},
+		{enterprise: "test", org: "", repo: "", length: 2, err: false},
+		{enterprise: "invalid", org: "", repo: "", length: 0, err: true},
+		{enterprise: "error", org: "", repo: "", length: 0, err: true},
+	}
+
+	client := newTestClient()
+	for i, tt := range tests {
+		runners, err := client.ListRunners(context.Background(), tt.enterprise, tt.org, tt.repo)
+		if !tt.err && err != nil {
+			t.Errorf("[%d] unexpected error: %v", i, err)
+		}
+		if tt.length != len(runners) {
+			t.Errorf("[%d] unexpected runners list: %v", i, runners)
+		}
+	}
+}
+
+func TestRemoveRunner(t *testing.T) {
+	tests := []struct {
+		enterprise string
+		org        string
+		repo       string
+		err        bool
+	}{
+		{enterprise: "", org: "", repo: "test/valid", err: false},
+		{enterprise: "", org: "", repo: "test/invalid", err: true},
+		{enterprise: "", org: "", repo: "test/error", err: true},
+		{enterprise: "", org: "test", repo: "", err: false},
+		{enterprise: "", org: "invalid", repo: "", err: true},
+		{enterprise: "", org: "error", repo: "", err: true},
+		{enterprise: "test", org: "", repo: "", err: false},
+		{enterprise: "invalid", org: "", repo: "", err: true},
+		{enterprise: "error", org: "", repo: "", err: true},
+	}
+
+	client := newTestClient()
+	for i, tt := range tests {
+		err := client.RemoveRunner(context.Background(), tt.enterprise, tt.org, tt.repo, int64(1))
+		if !tt.err && err != nil {
+			t.Errorf("[%d] unexpected error: %v", i, err)
+		}
+	}
+}
+
+func TestCleanup(t *testing.T) {
+	token := "token"
+
+	client := newTestClient()
+	client.regTokens = map[string]*github.RegistrationToken{
+		"active": &github.RegistrationToken{
+			Token:     &token,
+			ExpiresAt: &github.Timestamp{Time: time.Now().Add(time.Hour * 1)},
+		},
+		"expired": &github.RegistrationToken{
+			Token:     &token,
+			ExpiresAt: &github.Timestamp{Time: time.Now().Add(-time.Hour * 1)},
+		},
+	}
+
+	client.cleanup()
+	if _, ok := client.regTokens["active"]; !ok {
+		t.Errorf("active token was accidentally removed")
+	}
+	if _, ok := client.regTokens["expired"]; ok {
+		t.Errorf("expired token still exists")
+	}
+}

github/metrics/transport.go

@@ -0,0 +1,63 @@
+// Package metrics provides monitoring of the GitHub related metrics.
+//
+// This depends on the metrics exporter of kubebuilder.
+// See https://book.kubebuilder.io/reference/metrics.html for details.
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+func init() {
+	metrics.Registry.MustRegister(metricRateLimit, metricRateLimitRemaining)
+}
+
+var (
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+	metricRateLimit = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit",
+			Help: "The maximum number of requests you're permitted to make per hour",
+		},
+	)
+	metricRateLimitRemaining = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit_remaining",
+			Help: "The number of requests remaining in the current rate limit window",
+		},
+	)
+)
+
+const (
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+	headerRateLimit          = "X-RateLimit-Limit"
+	headerRateLimitRemaining = "X-RateLimit-Remaining"
+)
+
+// Transport wraps a transport with metrics monitoring
+type Transport struct {
+	Transport http.RoundTripper
+}
+
+func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := t.Transport.RoundTrip(req)
+	if resp != nil {
+		parseResponse(resp)
+	}
+	return resp, err
+}
+
+func parseResponse(resp *http.Response) {
+	rateLimit, err := strconv.Atoi(resp.Header.Get(headerRateLimit))
+	if err == nil {
+		metricRateLimit.Set(float64(rateLimit))
+	}
+	rateLimitRemaining, err := strconv.Atoi(resp.Header.Get(headerRateLimitRemaining))
+	if err == nil {
+		metricRateLimitRemaining.Set(float64(rateLimitRemaining))
+	}
+}

go.mod

@@ -1,22 +1,22 @@
 module github.com/summerwind/actions-runner-controller
 
-go 1.13
+go 1.15
 
 require (
-	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v0.1.0
-	github.com/google/go-github v17.0.0+incompatible
-	github.com/google/go-github/v29 v29.0.3
+	github.com/google/go-github/v33 v33.0.1-0.20210204004227-319dcffb518a
+	github.com/gorilla/mux v1.8.0
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.5.0
-	github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
+	github.com/prometheus/client_golang v0.9.2
+	github.com/stretchr/testify v1.4.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
 	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
 	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
+	k8s.io/utils v0.0.0-20190801114015-581e00157fb1
 	sigs.k8s.io/controller-runtime v0.4.0
 )

go.sum

@@ -18,10 +18,6 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
@@ -120,12 +116,10 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
-github.com/google/go-github/v29 v29.0.3 h1:IktKCTwU//aFHnpA+2SLIi7Oo9uhAzgsdZNbcAqhgdc=
-github.com/google/go-github/v29 v29.0.3/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
+github.com/google/go-github/v33 v33.0.1-0.20210204004227-319dcffb518a h1:Z9Nzq8ntvvXCLnFGOkzzcD8HDOzOo+obuwE5oK85vNQ=
+github.com/google/go-github/v33 v33.0.1-0.20210204004227-319dcffb518a/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@@ -141,6 +135,8 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC
 github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
 github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -162,6 +158,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -237,6 +235,7 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
@@ -343,8 +342,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=

hack/make-env.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+COMMIT=$(git rev-parse HEAD)
+TAG=$(git describe --exact-match --abbrev=0 --tags "${COMMIT}" 2> /dev/null || true)
+BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | sed -e 's/[^a-zA-Z0-9+=._:/-]*//g' || true)
+VERSION=""
+
+if [ -z "$TAG" ]; then
+  [[ -n "$BRANCH" ]] && VERSION="${BRANCH}-"
+	VERSION="${VERSION}${COMMIT:0:8}"
+else
+	VERSION=$TAG
+fi
+
+if [ -n "$(git diff --shortstat 2> /dev/null | tail -n1)" ]; then
+    VERSION="${VERSION}-dirty"
+fi
+
+export VERSION

hash/fnv.go

@@ -0,0 +1,17 @@
+package hash
+
+import (
+	"fmt"
+	"hash/fnv"
+	"k8s.io/apimachinery/pkg/util/rand"
+)
+
+func FNVHashStringObjects(objs ...interface{}) string {
+	hash := fnv.New32a()
+
+	for _, obj := range objs {
+		DeepHashObject(hash, obj)
+	}
+
+	return rand.SafeEncodeString(fmt.Sprint(hash.Sum32()))
+}

hash/hash.go

@@ -0,0 +1,25 @@
+// Copyright 2015 The Kubernetes Authors.
+// hash.go is copied from kubernetes's pkg/util/hash.go
+// See https://github.com/kubernetes/kubernetes/blob/e1c617a88ec286f5f6cb2589d6ac562d095e1068/pkg/util/hash/hash.go#L25-L37
+
+package hash
+
+import (
+	"hash"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+// DeepHashObject writes specified object to hash using the spew library
+// which follows pointers and prints actual values of the nested objects
+// ensuring the hash does not change when a pointer changes.
+func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) {
+	hasher.Reset()
+	printer := spew.ConfigState{
+		Indent:         " ",
+		SortKeys:       true,
+		DisableMethods: true,
+		SpewKeys:       true,
+	}
+	printer.Fprintf(hasher, "%#v", objectToWrite)
+}

main.go

@@ -17,15 +17,15 @@ limitations under the License.
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
-	"github.com/google/go-github/v29/github"
+	"github.com/kelseyhightower/envconfig"
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"github.com/summerwind/actions-runner-controller/controllers"
-	"golang.org/x/oauth2"
+	"github.com/summerwind/actions-runner-controller/github"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@@ -35,8 +35,8 @@ import (
 )
 
 const (
-	defaultRunnerImage = "summerwind/actions-runner:v2.165.2"
-	defaultDockerImage = "docker:19.03.6-dind"
+	defaultRunnerImage = "summerwind/actions-runner:latest"
+	defaultDockerImage = "docker:dind"
 )
 
 var (
@@ -53,44 +53,53 @@ func init() {
 
 func main() {
 	var (
+		err      error
+		ghClient *github.Client
+
 		metricsAddr          string
 		enableLeaderElection bool
+		syncPeriod           time.Duration
 
 		runnerImage string
 		dockerImage string
-		ghToken     string
 	)
 
+	var c github.Config
+	err = envconfig.Process("github", &c)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error: Environment variable read failed.")
+	}
+
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&runnerImage, "runner-image", defaultRunnerImage, "The image name of self-hosted runner container.")
 	flag.StringVar(&dockerImage, "docker-image", defaultDockerImage, "The image name of docker sidecar container.")
-	flag.StringVar(&ghToken, "github-token", "", "The access token of GitHub.")
+	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
+	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
+	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
+	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
+	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.Parse()
 
-	if ghToken == "" {
-		ghToken = os.Getenv("GITHUB_TOKEN")
-	}
-	if ghToken == "" {
-		fmt.Fprintln(os.Stderr, "Error: GitHub access token must be specified.")
+	logger := zap.New(func(o *zap.Options) {
+		o.Development = true
+	})
+
+	ghClient, err = c.NewClient()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error: Client creation failed.", err)
 		os.Exit(1)
 	}
 
-	tc := oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: ghToken},
-	))
-	ghClient := github.NewClient(tc)
-
-	ctrl.SetLogger(zap.New(func(o *zap.Options) {
-		o.Development = true
-	}))
+	ctrl.SetLogger(logger)
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
 		Port:               9443,
+		SyncPeriod:         &syncPeriod,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
@@ -112,9 +121,10 @@ func main() {
 	}
 
 	runnerSetReconciler := &controllers.RunnerReplicaSetReconciler{
-		Client: mgr.GetClient(),
-		Log:    ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
-		Scheme: mgr.GetScheme(),
+		Client:       mgr.GetClient(),
+		Log:          ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
+		Scheme:       mgr.GetScheme(),
+		GitHubClient: ghClient,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {
@@ -132,6 +142,32 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "RunnerDeployment")
 		os.Exit(1)
 	}
+
+	horizontalRunnerAutoscaler := &controllers.HorizontalRunnerAutoscalerReconciler{
+		Client:        mgr.GetClient(),
+		Log:           ctrl.Log.WithName("controllers").WithName("HorizontalRunnerAutoscaler"),
+		Scheme:        mgr.GetScheme(),
+		GitHubClient:  ghClient,
+		CacheDuration: syncPeriod - 10*time.Second,
+	}
+
+	if err = horizontalRunnerAutoscaler.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "HorizontalRunnerAutoscaler")
+		os.Exit(1)
+	}
+
+	if err = (&actionsv1alpha1.Runner{}).SetupWebhookWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "Runner")
+		os.Exit(1)
+	}
+	if err = (&actionsv1alpha1.RunnerDeployment{}).SetupWebhookWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "RunnerDeployment")
+		os.Exit(1)
+	}
+	if err = (&actionsv1alpha1.RunnerReplicaSet{}).SetupWebhookWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create webhook", "webhook", "RunnerReplicaSet")
+		os.Exit(1)
+	}
 	// +kubebuilder:scaffold:builder
 
 	setupLog.Info("starting manager")

runner/Dockerfile

@@ -1,27 +1,90 @@
 FROM ubuntu:18.04
 
-ARG RUNNER_VERSION
-ARG DOCKER_VERSION
+ARG TARGETPLATFORM
+ARG RUNNER_VERSION=2.274.2
+ARG DOCKER_VERSION=19.03.12
 
-RUN apt update \
-  && apt install curl ca-certificates -y --no-install-recommends \
-  && curl -L -o docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz \
+RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false)
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update -y \
+  && apt install -y software-properties-common \
+  && add-apt-repository -y ppa:git-core/ppa \
+  && apt update -y \
+  && apt install -y --no-install-recommends \
+  build-essential \
+  curl \
+  ca-certificates \
+  dnsutils \
+  ftp \
+  git \
+  iproute2 \
+  iputils-ping \
+  jq \
+  libunwind8 \
+  locales \
+  netcat \
+  openssh-client \
+  parallel \
+  rsync \
+  shellcheck \
+  sudo \
+  telnet \
+  time \
+  tzdata \
+  unzip \
+  upx \
+  wget \
+  zip \
+  zstd \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} \
+  && chmod +x /usr/local/bin/dumb-init
+
+# Docker download supports arm64 as aarch64 & amd64 as x86_64
+RUN set -vx; \
+  export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \
+  && curl -L -o docker.tgz https://download.docker.com/linux/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz \
   && tar zxvf docker.tgz \
   && install -o root -g root -m 755 docker/docker /usr/local/bin/docker \
   && rm -rf docker docker.tgz \
-  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \
-  && chmod +x /usr/local/bin/dumb-init \
-  && adduser --disabled-password --gecos "" --uid 1000 runner
+  && adduser --disabled-password --gecos "" --uid 1000 runner \
+  && groupadd docker \
+  && usermod -aG sudo runner \
+  && usermod -aG docker runner \
+  && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
 
-RUN mkdir -p /runner \
-  && cd /runner \
-  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+ENV RUNNER_ASSETS_DIR=/runnertmp
+
+# Runner download supports amd64 as x64. Externalstmp is needed for making mount points work inside DinD.
+#
+# libyaml-dev is required for ruby/setup-ruby action.
+# It is installed after installdependencies.sh and before removing /var/lib/apt/lists
+# to avoid rerunning apt-update on its own.
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
+  && mkdir -p "$RUNNER_ASSETS_DIR" \
+  && cd "$RUNNER_ASSETS_DIR" \
+  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
   && tar xzf ./runner.tar.gz \
   && rm runner.tar.gz \
-  && ./bin/installdependencies.sh
+  && ./bin/installdependencies.sh \
+  && mv ./externals ./externalstmp \
+  && apt-get install -y libyaml-dev \
+  && rm -rf /var/lib/apt/lists/*
 
-COPY entrypoint.sh /runner
+RUN echo AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache > .env \
+  && mkdir /opt/hostedtoolcache \
+  && chgrp runner /opt/hostedtoolcache \
+  && chmod g+rwx /opt/hostedtoolcache
 
-USER runner:runner
+COPY entrypoint.sh /
+COPY --chown=runner:docker patched $RUNNER_ASSETS_DIR/patched
+
+USER runner
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
-CMD ["/runner/entrypoint.sh"]
+CMD ["/entrypoint.sh"]

runner/Makefile

@@ -1,11 +1,50 @@
 NAME ?= summerwind/actions-runner
+DIND_RUNNER_NAME ?= ${NAME}-dind
+TAG ?= latest
 
-RUNNER_VERSION ?= 2.165.2
-DOCKER_VERSION ?= 19.03.6
+RUNNER_VERSION ?= 2.274.2
+DOCKER_VERSION ?= 19.03.12
+
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
 
 docker-build:
-	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:latest -t ${NAME}:v${RUNNER_VERSION} .
+	docker build --build-arg TARGETPLATFORM=amd64 --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} .
+	docker build --build-arg TARGETPLATFORM=amd64 --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${DIND_RUNNER_NAME}:${TAG} -f dindrunner.Dockerfile .
+
 
 docker-push:
-	docker push ${NAME}:latest
-	docker push ${NAME}:v${RUNNER_VERSION}
+	docker push ${NAME}:${TAG}
+	docker push ${DIND_RUNNER_NAME}:${TAG}
+
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:latest" \
+		-f Dockerfile \
+		. ${PUSH_ARG}
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${DIND_RUNNER_NAME}:latest" \
+		-f dindrunner.Dockerfile \
+		. ${PUSH_ARG}