Fix permission error with the default setup since v0.9.4 (#142)

Fixes #138

.github/workflows/build-runner.yml

@@ -1,22 +1,78 @@
 on:
+  pull_request:
+    branches:
+      - '**'
+    paths:
+      - 'runner/**'
+      - .github/workflows/build-runner.yml
   push:
     branches:
-    - master
+      - master
     paths:
-    - 'runner/**'
-
+      - runner/patched/*
+      - runner/Dockerfile
+      - runner/entrypoint.sh
+      - .github/workflows/build-runner.yml
+name: Runner
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: Build runner
+    name: Build
+    env:
+      RUNNER_VERSION: 2.273.5
+      DOCKER_VERSION: 19.03.12
+      DOCKERHUB_USERNAME: ${{ github.repository_owner }}
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Build container image
-      run: make docker-build
-      working-directory: runner
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
-      working-directory: runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: latest
+
+      - name: Build Container Image
+        working-directory: runner
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:latest \
+            -f Dockerfile .
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:latest \
+            -f Dockerfile.dindrunner .
+
+      - name: Login to GitHub Docker Registry
+        run: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin
+        if: ${{ github.event_name == 'push' }}
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build and Push Container Image
+        working-directory: runner
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:latest \
+            -f Dockerfile . --push
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:latest \
+            -f Dockerfile.dindrunner . --push

.github/workflows/release.yml

@@ -7,27 +7,45 @@ jobs:
     runs-on: ubuntu-latest
     name: Release
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Install tools
-      run: |
-        curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
-        tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz
-        sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
-        curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
-        sudo mv kustomize /usr/local/bin
-        curl -L -O https://github.com/tcnksm/ghr/releases/download/v0.13.0/ghr_v0.13.0_linux_amd64.tar.gz
-        tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
-        sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
-    - name: Set version
-      run: echo "::set-env name=VERSION::$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')"
-    - name: Upload artifacts
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: make github-release
-    - name: Build container image
-      run: make docker-build
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install tools
+        run: |
+          curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
+          tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz
+          sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
+          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
+          sudo mv kustomize /usr/local/bin
+          curl -L -O https://github.com/tcnksm/ghr/releases/download/v0.13.0/ghr_v0.13.0_linux_amd64.tar.gz
+          tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
+          sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
+
+      - name: Set version
+        run: echo "::set-env name=VERSION::$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')"
+
+      - name: Upload artifacts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make github-release
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: latest
+
+      - name: Login to GitHub Docker Registry
+        run: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build Container Image
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+        run: |
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-controller:${{ env.VERSION }} \
+            -f Dockerfile . --push

.github/workflows/test.yaml

@@ -1,15 +1,16 @@
+name: CI
+
 on:
-  push:
+  pull_request:
     branches:
-    - master
+      - master
     paths-ignore:
     - 'runner/**'
-    - '.github/**'
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
-    name: Build
+    name: Test
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -20,9 +21,7 @@ jobs:
         sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
     - name: Run tests
       run: make test
-    - name: Build container image
-      run: make docker-build
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
+    - name: Verify manifests are up-to-date
+      run: |
+        make manifests
+        git diff --exit-code

Dockerfile

@@ -1,28 +1,37 @@
 # Build the manager binary
 FROM golang:1.13 as builder
 
+ARG TARGETPLATFORM
+
 WORKDIR /workspace
+
+ENV GO111MODULE=on \
+  CGO_ENABLED=0
+
 # Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
+COPY go.mod go.sum ./
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
-COPY api/ api/
-COPY controllers/ controllers/
-COPY github/ github/
+COPY . .
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
+RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
+  export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
+  GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \
+  go build -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
+
 WORKDIR /
+
 COPY --from=builder /workspace/manager .
+
 USER nonroot:nonroot
 
 ENTRYPOINT ["/manager"]

Makefile

@@ -1,5 +1,8 @@
 NAME ?= summerwind/actions-runner-controller
 VERSION ?= latest
+# From https://github.com/VictoriaMetrics/operator/pull/44
+YAML_DROP=$(YQ) delete --inplace
+YAML_DROP_PREFIX=spec.validation.openAPIV3Schema.properties.spec.properties
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
@@ -11,6 +14,23 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
+
 all: manager
 
 # Run tests
@@ -39,7 +59,9 @@ deploy: manifests
 	kustomize build config/default | kubectl apply -f -
 
 # Generate manifests e.g. CRD, RBAC etc.
-manifests: controller-gen
+manifests: manifests-118 fix118
+
+manifests-118: controller-gen
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 # Run go fmt against code
@@ -50,6 +72,22 @@ fmt:
 vet:
 	go vet ./...
 
+# workaround for CRD issue with k8s 1.18 & controller-gen
+# ref: https://github.com/kubernetes/kubernetes/issues/91395
+fix118: yq
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).ephemeralContainers.items.properties
+
 # Generate code
 generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
@@ -62,6 +100,18 @@ docker-build: test
 docker-push:
 	docker push ${NAME}:${VERSION}
 
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:${VERSION}" \
+		-f Dockerfile \
+		. ${PUSH_ARG}
+
 # Generate the release manifest file
 release: manifests
 	cd config/manager && kustomize edit set image controller=${NAME}:${VERSION}
@@ -76,15 +126,34 @@ github-release: release
 # download controller-gen if necessary
 controller-gen:
 ifeq (, $(shell which controller-gen))
+ifeq (, $(wildcard $(GOBIN)/controller-gen))
 	@{ \
 	set -e ;\
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
+endif
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 else
 CONTROLLER_GEN=$(shell which controller-gen)
 endif
+
+# find or download yq
+# download yq if necessary
+# Use always go-version to get consistent line wraps etc.
+yq:
+ifeq (, $(wildcard $(GOBIN)/yq))
+	echo "Downloading yq"
+	@{ \
+	set -e ;\
+	YQ_TMP_DIR=$$(mktemp -d) ;\
+	cd $$YQ_TMP_DIR ;\
+	go mod init tmp ;\
+	go get github.com/mikefarah/yq/v3@3.4.0 ;\
+	rm -rf $$YQ_TMP_DIR ;\
+	}
+endif
+YQ=$(GOBIN)/yq

README.md

@@ -22,7 +22,7 @@ $ kubectl apply -f https://github.com/summerwind/actions-runner-controller/relea
 
 ## Setting up authentication with GitHub API
 
-There are two ways for actions-runner-controller to authenticate with the the GitHub API:
+There are two ways for actions-runner-controller to authenticate with the GitHub API:
 
 1. Using GitHub App.
 2. Using Personal Access Token.
@@ -135,7 +135,7 @@ The runner you created has been registered to your repository.
 
 <img width="756" alt="Actions tab in your repository settings" src="https://user-images.githubusercontent.com/230145/73618667-8cbf9700-466c-11ea-80b6-c67e6d3f70e7.png">
 
-Now your can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
+Now you can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
 
 ### Organization Runners
 
@@ -191,7 +191,7 @@ example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 
 #### Autoscaling
 
-`RunnerDeployment` can scale number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
+`RunnerDeployment` can scale the number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
 
 In the below example, `actions-runner` checks for pending workflow runs for each sync period, and scale to e.g. 3 if there're 3 pending jobs at sync time.
 
@@ -250,6 +250,27 @@ spec:
     repositoryNames:
     - summerwind/actions-runner-controller
 ```
+## Runner with DinD
+
+When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
+
+```yaml
+# dindrunnerdeployment.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-dindrunnerdeploy
+spec:
+  replicas: 2
+  template:
+    spec:
+      image: summerwind/actions-runner-dind
+      dockerdWithinRunnerContainer: true
+      repository: mumoshu/actions-runner-controller-ci
+      env: []
+```
+
+This also helps with resources, as you don't need to give resources separately to docker and runner.
 
 ## Additional tweaks
 
@@ -283,6 +304,17 @@ spec:
         requests:
           cpu: "2.0"
           memory: "4Gi"
+      # If set to true, runner pod container only 1 container that's expected to be able to run docker, too.
+      # image summerwind/actions-runner-dind or custom one should be used with true -value
+      dockerdWithinRunnerContainer: false
+      # Valid if dockerdWithinRunnerContainer is not true
+      dockerdContainerResources:
+        limits:
+          cpu: "4.0"
+          memory: "8Gi"
+        requests:
+          cpu: "2.0"
+          memory: "4Gi"
       sidecarContainers:
         - name: mysql
           image: mysql:5.7
@@ -328,9 +360,9 @@ jobs:
     runs-on: custom-runner
 ```
 
-Note that if you specify `self-hosted` in your worlflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
+Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
 
-## Softeware installed in the runner image
+## Software installed in the runner image
 
 The GitHub hosted runners include a large amount of pre-installed software packages. For Ubuntu 18.04, this list can be found at https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md
 
@@ -364,3 +396,15 @@ spec:
   repository: summerwind/actions-runner-controller
   image: YOUR_CUSTOM_DOCKER_IMAGE
 ```
+
+# Alternatives
+
+The following is a list of alternative solutions that may better fit you depending on your use-case:
+
+- https://github.com/evryfs/github-actions-runner-operator/
+
+Although the situation can change over time, as of writing this sentence, the benefits of using `actions-runner-controller` over the alternatives are:
+
+- `actions-runner-controller` has the ability to autoscale runners based on number of pending/progressing jobs (#99)
+- `actions-runner-controller` is able to gracefully stop runners (#103)
+- `actions-runner-controller` has ARM support

api/v1alpha1/runner_types.go

@@ -39,6 +39,8 @@ type RunnerSpec struct {
 	// +optional
 	Containers []corev1.Container `json:"containers,omitempty"`
 	// +optional
+	DockerdContainerResources corev1.ResourceRequirements `json:"dockerdContainerResources,omitempty"`
+	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 	// +optional
 	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
@@ -77,6 +79,8 @@ type RunnerSpec struct {
 	EphemeralContainers []corev1.EphemeralContainer `json:"ephemeralContainers,omitempty"`
 	// +optional
 	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
+	// +optional
+	DockerdWithinRunnerContainer *bool `json:"dockerdWithinRunnerContainer,omitempty"`
 }
 
 // ValidateRepository validates repository field.

api/v1alpha1/zz_generated.deepcopy.go

@@ -435,6 +435,7 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.DockerdContainerResources.DeepCopyInto(&out.DockerdContainerResources)
 	in.Resources.DeepCopyInto(&out.Resources)
 	if in.VolumeMounts != nil {
 		in, out := &in.VolumeMounts, &out.VolumeMounts
@@ -524,6 +525,11 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 		*out = new(int64)
 		**out = **in
 	}
+	if in.DockerdWithinRunnerContainer != nil {
+		in, out := &in.DockerdWithinRunnerContainer, &out.DockerdWithinRunnerContainer
+		*out = new(bool)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSpec.

config/certmanager/certificate.yaml

@@ -1,7 +1,7 @@
 # The following manifests contain a self-signed issuer CR and a certificate CR.
 # More document can be found at https://docs.cert-manager.io
 # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: selfsigned-issuer
@@ -9,7 +9,7 @@ metadata:
 spec:
   selfSigned: {}
 ---
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: horizontalrunnerautoscalers.actions.summerwind.dev
 spec:

config/crd/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - bases/actions.summerwind.dev_runners.yaml
 - bases/actions.summerwind.dev_runnerreplicasets.yaml
 - bases/actions.summerwind.dev_runnerdeployments.yaml
+- bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/default/kustomization.yaml

@@ -50,7 +50,7 @@ vars:
   objref:
     kind: Certificate
     group: cert-manager.io
-    version: v1alpha2
+    version: v1
     name: serving-cert # this name should match the one in certificate.yaml
   fieldref:
     fieldpath: metadata.namespace
@@ -58,7 +58,7 @@ vars:
   objref:
     kind: Certificate
     group: cert-manager.io
-    version: v1alpha2
+    version: v1
     name: serving-cert # this name should match the one in certificate.yaml
 - name: SERVICE_NAMESPACE # namespace of the service
   objref:

config/rbac/role.yaml

@@ -18,6 +18,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -38,6 +50,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -58,6 +82,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerreplicasets/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -78,6 +114,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -105,3 +153,15 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

controllers/autoscaling.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"strings"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 )
 
 func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
@@ -44,6 +45,38 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp
 	}
 
 	var total, inProgress, queued, completed, unknown int
+	type callback func()
+	listWorkflowJobs := func(user string, repoName string, runID int64, fallback_cb callback) {
+		if runID == 0 {
+			fallback_cb()
+			return
+		}
+		jobs, _, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, nil)
+		if err != nil {
+			r.Log.Error(err, "Error listing workflow jobs")
+			fallback_cb()
+		} else if len(jobs.Jobs) == 0 {
+			fallback_cb()
+		} else {
+			for _, job := range jobs.Jobs {
+				switch job.GetStatus() {
+				case "completed":
+					// We add a case for `completed` so it is not counted in `unknown`.
+					// And we do not increment the counter for completed because
+					// that counter only refers to workflows. The reason for
+					// this is because we do not get a list of jobs for
+					// completed workflows in order to keep the number of API
+					// calls to a minimum.
+				case "in_progress":
+					inProgress++
+				case "queued":
+					queued++
+				default:
+					unknown++
+				}
+			}
+		}
+	}
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
@@ -52,20 +85,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp
 			return nil, err
 		}
 
-		for _, r := range list.WorkflowRuns {
+		for _, run := range list.WorkflowRuns {
 			total++
 
 			// In May 2020, there are only 3 statuses.
 			// Follow the below links for more details:
 			// - https://developer.github.com/v3/actions/workflow-runs/#list-repository-workflow-runs
 			// - https://developer.github.com/v3/checks/runs/#create-a-check-run
-			switch r.GetStatus() {
+			switch run.GetStatus() {
 			case "completed":
 				completed++
 			case "in_progress":
-				inProgress++
+				listWorkflowJobs(user, repoName, run.GetID(), func() { inProgress++ })
 			case "queued":
-				queued++
+				listWorkflowJobs(user, repoName, run.GetID(), func() { queued++ })
 			default:
 				unknown++
 			}

controllers/autoscaling_test.go

@@ -2,16 +2,17 @@ package controllers
 
 import (
 	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"github.com/summerwind/actions-runner-controller/github"
 	"github.com/summerwind/actions-runner-controller/github/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"net/http/httptest"
-	"net/url"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	"testing"
 )
 
 func newGithubClient(server *httptest.Server) *github.Client {
@@ -44,9 +45,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		sReplicas    *int
 		sTime        *metav1.Time
 		workflowRuns string
+		workflowJobs map[int]string
 		want         int
 		err          string
 	}{
+		// Legacy functionality
 		// 3 demanded, max at 3
 		{
 			repo:         "test/valid",
@@ -122,6 +125,21 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
 			want:         3,
 		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
 	}
 
 	for i := range testcases {
@@ -136,7 +154,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
 			defer server.Close()
 			client := newGithubClient(server)
 
@@ -211,6 +229,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		sReplicas    *int
 		sTime        *metav1.Time
 		workflowRuns string
+		workflowJobs map[int]string
 		want         int
 		err          string
 	}{
@@ -316,6 +335,22 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
 			err:          "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
 		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
 	}
 
 	for i := range testcases {
@@ -330,7 +365,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
 			defer server.Close()
 			client := newGithubClient(server)
 

controllers/horizontalrunnerautoscaler_controller.go

@@ -50,6 +50,7 @@ type HorizontalRunnerAutoscalerReconciler struct {
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 

controllers/integration_test.go

@@ -2,9 +2,10 @@ package controllers
 
 import (
 	"context"
-	"github.com/summerwind/actions-runner-controller/github/fake"
 	"time"
 
+	"github.com/summerwind/actions-runner-controller/github/fake"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -19,18 +20,32 @@ import (
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
 )
 
+type testEnvironment struct {
+	Namespace *corev1.Namespace
+	Responses *fake.FixedResponses
+}
+
+var (
+	workflowRunsFor3Replicas = `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`
+	workflowRunsFor1Replicas = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"`
+)
+
 // SetupIntegrationTest will set up a testing environment.
 // This includes:
 // * creating a Namespace to be used during the test
 // * starting all the reconcilers
 // * stopping all the reconcilers after the test ends
 // Call this function at the start of each of your tests.
-func SetupIntegrationTest(ctx context.Context) *corev1.Namespace {
+func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 	var stopCh chan struct{}
 	ns := &corev1.Namespace{}
 
-	workflowRuns := `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`
-	server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, workflowRuns))
+	responses := &fake.FixedResponses{}
+	responses.ListRepositoryWorkflowRuns = &fake.Handler{
+		Status: 200,
+		Body:   workflowRunsFor3Replicas,
+	}
+	fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses))
 
 	BeforeEach(func() {
 		stopCh = make(chan struct{})
@@ -44,11 +59,16 @@ func SetupIntegrationTest(ctx context.Context) *corev1.Namespace {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+		ghClient := newGithubClient(server)
+
 		replicasetController := &RunnerReplicaSetReconciler{
-			Client:   mgr.GetClient(),
-			Scheme:   scheme.Scheme,
-			Log:      logf.Log,
-			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
 		}
 		err = replicasetController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -62,7 +82,7 @@ func SetupIntegrationTest(ctx context.Context) *corev1.Namespace {
 		err = deploymentsController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
 
-		client := newGithubClient(server)
+		client := newGithubClient(fakeGithubServer)
 
 		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
 			Client:       mgr.GetClient(),
@@ -85,18 +105,20 @@ func SetupIntegrationTest(ctx context.Context) *corev1.Namespace {
 	AfterEach(func() {
 		close(stopCh)
 
-		server.Close()
+		fakeGithubServer.Close()
 
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
 	})
 
-	return ns
+	return &testEnvironment{Namespace: ns, Responses: responses}
 }
 
 var _ = Context("Inside of a new namespace", func() {
 	ctx := context.TODO()
-	ns := SetupIntegrationTest(ctx)
+	env := SetupIntegrationTest(ctx)
+	ns := env.Namespace
+	responses := env.Responses
 
 	Describe("when no existing resources exist", func() {
 
@@ -199,8 +221,9 @@ var _ = Context("Inside of a new namespace", func() {
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
 			}
 
+			// Scale-up to 3 replicas
 			{
-				rs := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      name,
 						Namespace: ns.Name,
@@ -216,9 +239,9 @@ var _ = Context("Inside of a new namespace", func() {
 					},
 				}
 
-				err := k8sClient.Create(ctx, rs)
+				err := k8sClient.Create(ctx, hra)
 
-				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerDeployment resource")
+				Expect(err).NotTo(HaveOccurred(), "failed to create test HorizontalRunnerAutoscaler resource")
 
 				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
 
@@ -249,6 +272,43 @@ var _ = Context("Inside of a new namespace", func() {
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(3))
 			}
+
+			// Scale-down to 1 replica
+			{
+				responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas
+
+				var hra actionsv1alpha1.HorizontalRunnerAutoscaler
+
+				err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &hra)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to get test HorizontalRunnerAutoscaler resource")
+
+				hra.Annotations = map[string]string{
+					"force-update": "1",
+				}
+
+				err = k8sClient.Update(ctx, &hra)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to get test HorizontalRunnerAutoscaler resource")
+
+				Eventually(
+					func() int {
+						var runnerSets actionsv1alpha1.RunnerReplicaSetList
+
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+			}
 		})
 	})
 })

controllers/runner_controller.go

@@ -53,8 +53,10 @@ type RunnerReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
@@ -165,7 +167,11 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodCreated", fmt.Sprintf("Created pod '%s'", newPod.Name))
 		log.Info("Created runner pod", "repository", runner.Spec.Repository)
 	} else {
-		if runner.Status.Phase != string(pod.Status.Phase) {
+		// If pod has ended up succeeded we need to restart it
+		// Happens e.g. when dind is in runner and run completes
+		restart := pod.Status.Phase == corev1.PodSucceeded
+
+		if !restart && runner.Status.Phase != string(pod.Status.Phase) {
 			updated := runner.DeepCopy()
 			updated.Status.Phase = string(pod.Status.Phase)
 			updated.Status.Reason = pod.Status.Reason
@@ -183,8 +189,6 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, err
 		}
 
-		restart := false
-
 		if pod.Status.Phase == corev1.PodRunning {
 			for _, status := range pod.Status.ContainerStatuses {
 				if status.Name != containerName {
@@ -203,12 +207,16 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, err
 		}
 
-		if pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image {
-			restart = true
-		}
-		if !reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) {
+		runnerBusy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+		if err != nil {
+			log.Error(err, "Failed to check if runner is busy")
+			return ctrl.Result{}, nil
+		}
+
+		if !runnerBusy && (!reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) || pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image) {
 			restart = true
 		}
+
 		if !restart {
 			return ctrl.Result{}, err
 		}
@@ -225,6 +233,21 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	return ctrl.Result{}, nil
 }
 
+func (r *RunnerReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, fmt.Errorf("runner not found")
+}
+
 func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name string) (bool, error) {
 	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
 	if err != nil {
@@ -234,6 +257,9 @@ func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name
 	id := int64(0)
 	for _, runner := range runners {
 		if runner.GetName() == name {
+			if runner.GetBusy() {
+				return false, fmt.Errorf("runner is busy")
+			}
 			id = runner.GetID()
 			break
 		}
@@ -252,8 +278,8 @@ func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name
 
 func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	var (
-		privileged bool  = true
-		group      int64 = 0
+		privileged      bool = true
+		dockerdInRunner bool = runner.Spec.DockerdWithinRunnerContainer != nil && *runner.Spec.DockerdWithinRunnerContainer
 	)
 
 	runnerImage := runner.Spec.Image
@@ -287,6 +313,10 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 			Name:  "RUNNER_TOKEN",
 			Value: runner.Status.Registration.Token,
 		},
+		{
+			Name:  "DOCKERD_IN_RUNNER",
+			Value: fmt.Sprintf("%v", dockerdInRunner),
+		},
 	}
 
 	env = append(env, runner.Spec.Env...)
@@ -306,58 +336,68 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 					ImagePullPolicy: runnerImagePullPolicy,
 					Env:             env,
 					EnvFrom:         runner.Spec.EnvFrom,
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							Name:      "work",
-							MountPath: "/runner/_work",
-						},
-						{
-							Name:      "docker",
-							MountPath: "/var/run",
-						},
-					},
 					SecurityContext: &corev1.SecurityContext{
-						RunAsGroup: &group,
+						// Runner need to run privileged if it contains DinD
+						Privileged: runner.Spec.DockerdWithinRunnerContainer,
 					},
 					Resources: runner.Spec.Resources,
 				},
-				{
-					Name:  "docker",
-					Image: r.DockerImage,
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							Name:      "work",
-							MountPath: "/runner/_work",
-						},
-						{
-							Name:      "docker",
-							MountPath: "/var/run",
-						},
-					},
-					SecurityContext: &corev1.SecurityContext{
-						Privileged: &privileged,
-					},
-				},
-			},
-			Volumes: []corev1.Volume{
-				{
-					Name: "work",
-					VolumeSource: corev1.VolumeSource{
-						EmptyDir: &corev1.EmptyDirVolumeSource{},
-					},
-				},
-				{
-					Name: "docker",
-					VolumeSource: corev1.VolumeSource{
-						EmptyDir: &corev1.EmptyDirVolumeSource{},
-					},
-				},
 			},
 		},
 	}
 
+	if !dockerdInRunner {
+		pod.Spec.Volumes = []corev1.Volume{
+			{
+				Name: "work",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "docker",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+		}
+		pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+			{
+				Name:      "work",
+				MountPath: "/runner/_work",
+			},
+			{
+				Name:      "docker",
+				MountPath: "/var/run",
+			},
+		}
+		pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
+			Name:  "docker",
+			Image: r.DockerImage,
+			VolumeMounts: []corev1.VolumeMount{
+				{
+					Name:      "work",
+					MountPath: "/runner/_work",
+				},
+				{
+					Name:      "docker",
+					MountPath: "/var/run",
+				},
+			},
+			SecurityContext: &corev1.SecurityContext{
+				Privileged: &privileged,
+			},
+		})
+
+	}
+
 	if len(runner.Spec.Containers) != 0 {
 		pod.Spec.Containers = runner.Spec.Containers
+		for i := 0; i < len(pod.Spec.Containers); i++ {
+			if pod.Spec.Containers[i].Name == containerName {
+				pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, env...)
+			}
+		}
 	}
 
 	if len(runner.Spec.VolumeMounts) != 0 {

controllers/runnerdeployment_controller.go

@@ -54,6 +54,7 @@ type RunnerDeploymentReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch

controllers/runnerreplicaset_controller.go

@@ -31,17 +31,20 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github"
 )
 
 // RunnerReplicaSetReconciler reconciles a Runner object
 type RunnerReplicaSetReconciler struct {
 	client.Client
-	Log      logr.Logger
-	Recorder record.EventRecorder
-	Scheme   *runtime.Scheme
+	Log          logr.Logger
+	Recorder     record.EventRecorder
+	Scheme       *runtime.Scheme
+	GitHubClient *github.Client
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
@@ -96,8 +99,25 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	if available > desired {
 		n := available - desired
 
+		// get runners that are currently not busy
+		var notBusy []v1alpha1.Runner
+		for _, runner := range myRunners {
+			busy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+			if err != nil {
+				log.Error(err, "Failed to check if runner is busy")
+				return ctrl.Result{}, err
+			}
+			if !busy {
+				notBusy = append(notBusy, runner)
+			}
+		}
+
+		if len(notBusy) < n {
+			n = len(notBusy)
+		}
+
 		for i := 0; i < n; i++ {
-			if err := r.Client.Delete(ctx, &myRunners[i]); err != nil {
+			if err := r.Client.Delete(ctx, &notBusy[i]); err != nil {
 				log.Error(err, "Failed to delete runner resource")
 
 				return ctrl.Result{}, err
@@ -166,3 +186,19 @@ func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&v1alpha1.Runner{}).
 		Complete(r)
 }
+
+func (r *RunnerReplicaSetReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
+	r.Log.Info("runners", "github", runners)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, fmt.Errorf("runner not found")
+}

controllers/runnerreplicaset_controller_test.go

@@ -3,11 +3,14 @@ package controllers
 import (
 	"context"
 	"math/rand"
+	"net/http/httptest"
 	"time"
 
+	"github.com/google/go-github/v32/github"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -17,6 +20,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+)
+
+var (
+	runnersList *fake.RunnersList
+	server      *httptest.Server
 )
 
 // SetupTest will set up a testing environment.
@@ -41,11 +50,16 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+		ghClient := newGithubClient(server)
+
 		controller := &RunnerReplicaSetReconciler{
-			Client:   mgr.GetClient(),
-			Scheme:   scheme.Scheme,
-			Log:      logf.Log,
-			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -61,6 +75,7 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 	AfterEach(func() {
 		close(stopCh)
 
+		server.Close()
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
 	})
@@ -124,6 +139,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
@@ -155,6 +180,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
@@ -186,6 +221,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(0))

github/fake/fake.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"time"
+	"unicode"
 )
 
 const (
@@ -14,118 +17,144 @@ const (
 {
   "total_count": 2,
   "runners": [
-    {"id": 1, "name": "test1", "os": "linux", "status": "online"},
-    {"id": 2, "name": "test2", "os": "linux", "status": "offline"}
+    {"id": 1, "name": "test1", "os": "linux", "status": "online", "busy": false},
+    {"id": 2, "name": "test2", "os": "linux", "status": "offline", "busy": false}
   ]
 }
 `
 )
 
-type handler struct {
+type Handler struct {
 	Status int
 	Body   string
 }
 
-func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	w.WriteHeader(h.Status)
 	fmt.Fprintf(w, h.Body)
 }
 
+type MapHandler struct {
+	Status int
+	Bodies map[int]string
+}
+
+func (h *MapHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// Parse out int key from URL path
+	key, err := strconv.Atoi(strings.TrimFunc(req.URL.Path, func(r rune) bool { return !unicode.IsNumber(r) }))
+	if err != nil {
+		w.WriteHeader(400)
+	} else if body := h.Bodies[key]; len(body) == 0 {
+		w.WriteHeader(404)
+	} else {
+		w.WriteHeader(h.Status)
+		fmt.Fprintf(w, body)
+	}
+}
+
+type ServerConfig struct {
+	*FixedResponses
+}
+
 // NewServer creates a fake server for running unit tests
 func NewServer(opts ...Option) *httptest.Server {
-	var responses FixedResponses
-
-	for _, o := range opts {
-		o(&responses)
+	config := ServerConfig{
+		FixedResponses: &FixedResponses{},
 	}
 
-	routes := map[string]handler{
+	for _, o := range opts {
+		o(&config)
+	}
+
+	routes := map[string]http.Handler{
 		// For CreateRegistrationToken
-		"/repos/test/valid/actions/runners/registration-token": handler{
+		"/repos/test/valid/actions/runners/registration-token": &Handler{
 			Status: http.StatusCreated,
 			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
 		},
-		"/repos/test/invalid/actions/runners/registration-token": handler{
+		"/repos/test/invalid/actions/runners/registration-token": &Handler{
 			Status: http.StatusOK,
 			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
 		},
-		"/repos/test/error/actions/runners/registration-token": handler{
+		"/repos/test/error/actions/runners/registration-token": &Handler{
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
-		"/orgs/test/actions/runners/registration-token": handler{
+		"/orgs/test/actions/runners/registration-token": &Handler{
 			Status: http.StatusCreated,
 			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
 		},
-		"/orgs/invalid/actions/runners/registration-token": handler{
+		"/orgs/invalid/actions/runners/registration-token": &Handler{
 			Status: http.StatusOK,
 			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
 		},
-		"/orgs/error/actions/runners/registration-token": handler{
+		"/orgs/error/actions/runners/registration-token": &Handler{
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
 
 		// For ListRunners
-		"/repos/test/valid/actions/runners": handler{
+		"/repos/test/valid/actions/runners": &Handler{
 			Status: http.StatusOK,
 			Body:   RunnersListBody,
 		},
-		"/repos/test/invalid/actions/runners": handler{
+		"/repos/test/invalid/actions/runners": &Handler{
 			Status: http.StatusNoContent,
 			Body:   "",
 		},
-		"/repos/test/error/actions/runners": handler{
+		"/repos/test/error/actions/runners": &Handler{
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
-		"/orgs/test/actions/runners": handler{
+		"/orgs/test/actions/runners": &Handler{
 			Status: http.StatusOK,
 			Body:   RunnersListBody,
 		},
-		"/orgs/invalid/actions/runners": handler{
+		"/orgs/invalid/actions/runners": &Handler{
 			Status: http.StatusNoContent,
 			Body:   "",
 		},
-		"/orgs/error/actions/runners": handler{
+		"/orgs/error/actions/runners": &Handler{
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
 
 		// For RemoveRunner
-		"/repos/test/valid/actions/runners/1": handler{
+		"/repos/test/valid/actions/runners/1": &Handler{
 			Status: http.StatusNoContent,
 			Body:   "",
 		},
-		"/repos/test/invalid/actions/runners/1": handler{
+		"/repos/test/invalid/actions/runners/1": &Handler{
 			Status: http.StatusOK,
 			Body:   "",
 		},
-		"/repos/test/error/actions/runners/1": handler{
+		"/repos/test/error/actions/runners/1": &Handler{
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
-		"/orgs/test/actions/runners/1": handler{
+		"/orgs/test/actions/runners/1": &Handler{
 			Status: http.StatusNoContent,
 			Body:   "",
 		},
-		"/orgs/invalid/actions/runners/1": handler{
+		"/orgs/invalid/actions/runners/1": &Handler{
 			Status: http.StatusOK,
 			Body:   "",
 		},
-		"/orgs/error/actions/runners/1": handler{
+		"/orgs/error/actions/runners/1": &Handler{
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
 
 		// For auto-scaling based on the number of queued(pending) workflow runs
-		"/repos/test/valid/actions/runs": responses.listRepositoryWorkflowRuns.handler(),
+		"/repos/test/valid/actions/runs": config.FixedResponses.ListRepositoryWorkflowRuns,
+
+		// For auto-scaling based on the number of queued(pending) workflow jobs
+		"/repos/test/valid/actions/runs/": config.FixedResponses.ListWorkflowJobs,
 	}
 
 	mux := http.NewServeMux()
 	for path, handler := range routes {
-		h := handler
-		mux.Handle(path, &h)
+		mux.Handle(path, handler)
 	}
 
 	return httptest.NewServer(mux)

github/fake/options.go

@@ -1,28 +1,32 @@
 package fake
 
 type FixedResponses struct {
-	listRepositoryWorkflowRuns FixedResponse
+	ListRepositoryWorkflowRuns *Handler
+	ListWorkflowJobs           *MapHandler
 }
 
-type FixedResponse struct {
-	Status int
-	Body   string
-}
-
-func (r FixedResponse) handler() handler {
-	return handler{
-		Status: r.Status,
-		Body:   r.Body,
-	}
-}
-
-type Option func(responses *FixedResponses)
+type Option func(*ServerConfig)
 
 func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
-	return func(r *FixedResponses) {
-		r.listRepositoryWorkflowRuns = FixedResponse{
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListRepositoryWorkflowRuns = &Handler{
 			Status: status,
 			Body:   body,
 		}
 	}
 }
+
+func WithListWorkflowJobsResponse(status int, bodies map[int]string) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListWorkflowJobs = &MapHandler{
+			Status: status,
+			Bodies: bodies,
+		}
+	}
+}
+
+func WithFixedResponses(responses *FixedResponses) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses = responses
+	}
+}

github/fake/runners.go

@@ -0,0 +1,74 @@
+package fake
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+
+	"github.com/google/go-github/v32/github"
+	"github.com/gorilla/mux"
+)
+
+type RunnersList struct {
+	runners []*github.Runner
+}
+
+func NewRunnersList() *RunnersList {
+	return &RunnersList{
+		runners: make([]*github.Runner, 0),
+	}
+}
+
+func (r *RunnersList) Add(runner *github.Runner) {
+	if !exists(r.runners, runner) {
+		r.runners = append(r.runners, runner)
+	}
+}
+
+func (r *RunnersList) GetServer() *httptest.Server {
+	router := mux.NewRouter()
+
+	router.Handle("/repos/{owner}/{repo}/actions/runners", r.handleList())
+	router.Handle("/repos/{owner}/{repo}/actions/runners/{id}", r.handleRemove())
+	router.Handle("/orgs/{org}/actions/runners", r.handleList())
+	router.Handle("/orgs/{org}/actions/runners/{id}", r.handleRemove())
+
+	return httptest.NewServer(router)
+}
+
+func (r *RunnersList) handleList() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		j, err := json.Marshal(github.Runners{
+			TotalCount: len(r.runners),
+			Runners:    r.runners,
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		w.WriteHeader(http.StatusOK)
+		w.Write(j)
+	}
+}
+
+func (r *RunnersList) handleRemove() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		vars := mux.Vars(res)
+		for i, runner := range r.runners {
+			if runner.ID != nil && vars["id"] == strconv.FormatInt(*runner.ID, 10) {
+				r.runners = append(r.runners[:i], r.runners[i+1:]...)
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func exists(runners []*github.Runner, runner *github.Runner) bool {
+	for _, r := range runners {
+		if *r.Name == *runner.Name {
+			return true
+		}
+	}
+	return false
+}

github/github.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/bradleyfalzon/ghinstallation"
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"golang.org/x/oauth2"
 )
 
@@ -85,7 +85,7 @@ func (c *Client) GetRegistrationToken(ctx context.Context, org, repo, name strin
 	return rt, nil
 }
 
-// RemoveRunner removes a runner with specified runner ID from repocitory.
+// RemoveRunner removes a runner with specified runner ID from repository.
 func (c *Client) RemoveRunner(ctx context.Context, org, repo string, runnerID int64) error {
 	owner, repo, err := getOwnerAndRepo(org, repo)
 
@@ -121,7 +121,7 @@ func (c *Client) ListRunners(ctx context.Context, org, repo string) ([]*github.R
 		list, res, err := c.listRunners(ctx, owner, repo, &opts)
 
 		if err != nil {
-			return runners, fmt.Errorf("failed to remove runner: %v", err)
+			return runners, fmt.Errorf("failed to list runners: %v", err)
 		}
 
 		runners = append(runners, list.Runners...)

github/github_beta.go

@@ -10,7 +10,7 @@ import (
 	"net/url"
 	"reflect"
 
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"github.com/google/go-querystring/query"
 )
 

github/github_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"github.com/summerwind/actions-runner-controller/github/fake"
 )
 
@@ -41,9 +41,9 @@ func TestGetRegistrationToken(t *testing.T) {
 		token string
 		err   bool
 	}{
-		{org: "test", repo: "valid", token: fake.RegistrationToken, err: false},
-		{org: "test", repo: "invalid", token: "", err: true},
-		{org: "test", repo: "error", token: "", err: true},
+		{org: "", repo: "test/valid", token: fake.RegistrationToken, err: false},
+		{org: "", repo: "test/invalid", token: "", err: true},
+		{org: "", repo: "test/error", token: "", err: true},
 		{org: "test", repo: "", token: fake.RegistrationToken, err: false},
 		{org: "invalid", repo: "", token: "", err: true},
 		{org: "error", repo: "", token: "", err: true},
@@ -68,9 +68,9 @@ func TestListRunners(t *testing.T) {
 		length int
 		err    bool
 	}{
-		{org: "test", repo: "valid", length: 2, err: false},
-		{org: "test", repo: "invalid", length: 0, err: true},
-		{org: "test", repo: "error", length: 0, err: true},
+		{org: "", repo: "test/valid", length: 2, err: false},
+		{org: "", repo: "test/invalid", length: 0, err: true},
+		{org: "", repo: "test/error", length: 0, err: true},
 		{org: "test", repo: "", length: 2, err: false},
 		{org: "invalid", repo: "", length: 0, err: true},
 		{org: "error", repo: "", length: 0, err: true},
@@ -94,9 +94,9 @@ func TestRemoveRunner(t *testing.T) {
 		repo string
 		err  bool
 	}{
-		{org: "test", repo: "valid", err: false},
-		{org: "test", repo: "invalid", err: true},
-		{org: "test", repo: "error", err: true},
+		{org: "", repo: "test/valid", err: false},
+		{org: "", repo: "test/invalid", err: true},
+		{org: "", repo: "test/error", err: true},
 		{org: "test", repo: "", err: false},
 		{org: "invalid", repo: "", err: true},
 		{org: "error", repo: "", err: true},

go.mod

@@ -6,8 +6,9 @@ require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v0.1.0
-	github.com/google/go-github/v31 v31.0.0
+	github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04
 	github.com/google/go-querystring v1.0.0
+	github.com/gorilla/mux v1.8.0
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.5.0
 	github.com/stretchr/testify v1.4.0 // indirect
@@ -15,6 +16,6 @@ require (
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
 	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
 	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
-	k8s.io/klog v0.4.0
+	k8s.io/utils v0.0.0-20190801114015-581e00157fb1
 	sigs.k8s.io/controller-runtime v0.4.0
 )

go.sum

@@ -116,11 +116,10 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
-github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo=
-github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM=
+github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04 h1:wEYk2h/GwOhImcVjiTIceP88WxVbXw2F+ARYUQMEsfg=
+github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@@ -136,6 +135,8 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC
 github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
 github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=

main.go

@@ -158,9 +158,10 @@ func main() {
 	}
 
 	runnerSetReconciler := &controllers.RunnerReplicaSetReconciler{
-		Client: mgr.GetClient(),
-		Log:    ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
-		Scheme: mgr.GetScheme(),
+		Client:       mgr.GetClient(),
+		Log:          ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
+		Scheme:       mgr.GetScheme(),
+		GitHubClient: ghClient,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {

runner/Dockerfile

@@ -1,7 +1,8 @@
 FROM ubuntu:18.04
 
-ARG RUNNER_VERSION
-ARG DOCKER_VERSION
+ARG TARGETPLATFORM
+ARG RUNNER_VERSION=2.272.0
+ARG DOCKER_VERSION=19.03.12
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update -y \
@@ -9,53 +10,65 @@ RUN apt update -y \
   && add-apt-repository -y ppa:git-core/ppa \
   && apt update -y \
   && apt install -y --no-install-recommends \
-     build-essential \
-     curl \
-     ca-certificates \
-     dnsutils \
-     ftp \
-     git \
-     iproute2 \
-     iputils-ping \
-     jq \
-     libunwind8 \
-     locales \
-     netcat \
-     openssh-client \
-     parallel \
-     rsync \
-     shellcheck \
-     sudo \
-     telnet \
-     time \
-     tzdata \
-     unzip \
-     upx \
-     wget \
-     zip \
-     zstd \
+  build-essential \
+  curl \
+  ca-certificates \
+  dnsutils \
+  ftp \
+  git \
+  iproute2 \
+  iputils-ping \
+  jq \
+  libunwind8 \
+  locales \
+  netcat \
+  openssh-client \
+  parallel \
+  rsync \
+  shellcheck \
+  sudo \
+  telnet \
+  time \
+  tzdata \
+  unzip \
+  upx \
+  wget \
+  zip \
+  zstd \
   && rm -rf /var/lib/apt/lists/*
 
-RUN curl -L -o docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz \
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} \
+  && chmod +x /usr/local/bin/dumb-init
+
+# Docker download supports arm64 as aarch64 & amd64 as x86_64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \
+  && curl -L -o docker.tgz https://download.docker.com/linux/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz \
   && tar zxvf docker.tgz \
   && install -o root -g root -m 755 docker/docker /usr/local/bin/docker \
   && rm -rf docker docker.tgz \
-  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \
-  && chmod +x /usr/local/bin/dumb-init \
   && adduser --disabled-password --gecos "" --uid 1000 runner \
+  && groupadd docker \
   && usermod -aG sudo runner \
+  && usermod -aG docker runner \
   && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
 
-RUN mkdir -p /runner \
+# Runner download supports amd64 as x64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
+  && mkdir -p /runner \
   && cd /runner \
-  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
   && tar xzf ./runner.tar.gz \
   && rm runner.tar.gz \
   && ./bin/installdependencies.sh \
   && rm -rf /var/lib/apt/lists/*
 
 COPY entrypoint.sh /runner
+COPY patched /runner/patched
 
-USER runner:runner
+USER runner
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
 CMD ["/runner/entrypoint.sh"]

runner/Dockerfile.dindrunner

@@ -0,0 +1,99 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+# Dev + DinD dependencies
+RUN apt update \
+    && apt install -y software-properties-common \
+    && add-apt-repository -y ppa:git-core/ppa \
+    && apt install -y \
+    build-essential \
+    curl \
+    ca-certificates \
+    dnsutils \
+    ftp \
+    git \
+    iproute2 \
+    iptables \
+    iputils-ping \
+    jq \
+    libunwind8 \
+    locales \
+    netcat \
+    openssh-client \
+    parallel \
+    rsync \
+    shellcheck \
+    sudo \
+    supervisor \
+    telnet \
+    time \
+    tzdata \
+    unzip \
+    upx \
+    wget \
+    zip \
+    zstd \
+    && rm -rf /var/lib/apt/list/*
+
+# Runner user
+RUN adduser --disabled-password --gecos "" --uid 1000 runner \
+    && groupadd docker \
+    && usermod -aG sudo runner \
+    && usermod -aG docker runner \
+    && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
+
+ARG TARGETPLATFORM
+ARG RUNNER_VERSION=2.272.0
+ARG DOCKER_CHANNEL=stable
+ARG DOCKER_VERSION=19.03.13
+ARG DEBUG=false
+
+# Docker installation
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
+    && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \
+	&& if ! curl -L -o docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz"; then \
+		echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${ARCH}'"; \
+		exit 1; \
+	fi; \
+	tar --extract \
+		--file docker.tgz \
+		--strip-components 1 \
+		--directory /usr/local/bin/ \
+	; \
+	rm docker.tgz; \
+	dockerd --version; \
+	docker --version
+
+# Runner download supports amd64 as x64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
+    && mkdir -p /runner \
+     && cd /runner \
+    && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
+    && tar xzf ./runner.tar.gz \
+    && rm runner.tar.gz \
+    && ./bin/installdependencies.sh \
+    && rm -rf /var/lib/apt/lists/*
+
+
+COPY modprobe startup.sh /usr/local/bin/
+COPY supervisor/ /etc/supervisor/conf.d/
+COPY logger.sh /opt/bash-utils/logger.sh
+COPY entrypoint.sh /usr/local/bin/
+
+RUN chmod +x /usr/local/bin/startup.sh /usr/local/bin/entrypoint.sh /usr/local/bin/modprobe
+
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} \
+    && chmod +x /usr/local/bin/dumb-init
+
+VOLUME /var/lib/docker
+
+COPY patched /runner/patched
+
+# No group definition, as that makes it harder to run docker.
+USER runner
+
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
+CMD ["startup.sh"]

runner/Makefile

@@ -1,11 +1,52 @@
 NAME ?= summerwind/actions-runner
+DIND_RUNNER_NAME ?= ${NAME}-dind
+TAG ?= latest
 
-RUNNER_VERSION ?= 2.272.0
+RUNNER_VERSION ?= 2.273.5
 DOCKER_VERSION ?= 19.03.12
 
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
+
 docker-build:
-	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:latest -t ${NAME}:v${RUNNER_VERSION} .
+	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} -t ${NAME}:v${RUNNER_VERSION} .
+	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${DIND_RUNNER_NAME}:${TAG} -t ${DIND_RUNNER_NAME}:v${RUNNER_VERSION} -f Dockerfile.dindrunner .
+
 
 docker-push:
-	docker push ${NAME}:latest
+	docker push ${NAME}:${TAG}
 	docker push ${NAME}:v${RUNNER_VERSION}
+	docker push ${DIND_RUNNER_NAME}:${TAG}
+	docker push ${DIND_RUNNER_NAME}:v${RUNNER_VERSION}
+
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:latest" \
+		-f Dockerfile \
+		. ${PUSH_ARG}
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${DIND_RUNNER_NAME}:latest" \
+		-f Dockerfile.dindrunner \
+		. ${PUSH_ARG}

runner/entrypoint.sh

@@ -28,5 +28,11 @@ fi
 cd /runner
 ./config.sh --unattended --replace --name "${RUNNER_NAME}" --url "https://github.com/${ATTACH}" --token "${RUNNER_TOKEN}" ${LABEL_ARG}
 
+for f in runsvc.sh RunnerService.js; do
+  diff {bin,patched}/${f} || :
+  sudo mv bin/${f}{,.bak}
+  sudo mv {patched,bin}/${f}
+done
+
 unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN
-exec ./run.sh --once
+exec ./bin/runsvc.sh --once

runner/logger.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Logger from this post http://www.cubicrace.com/2016/03/log-tracing-mechnism-for-shell-scripts.html
+
+function INFO(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [INFO] [${0}] $msg"
+}
+
+
+function DEBUG(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [DEBUG] [${0}] $msg"
+}
+
+function ERROR(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [ERROR]  $msg"
+}

runner/modprobe

@@ -0,0 +1,20 @@
+ #!/bin/sh
+set -eu
+
+# "modprobe" without modprobe
+# https://twitter.com/lucabruno/status/902934379835662336
+
+# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
+
+# Docker often uses "modprobe -va foo bar baz"
+# so we ignore modules that start with "-"
+for module; do
+	if [ "${module#-}" = "$module" ]; then
+		ip link show "$module" || true
+		lsmod | grep "$module" || true
+	fi
+done
+
+# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
+export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
+exec modprobe "$@"

runner/patched/RunnerService.js

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+// Copyright (c) GitHub. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+var childProcess = require("child_process");
+var path = require("path")
+
+var supported = ['linux', 'darwin']
+
+if (supported.indexOf(process.platform) == -1) {
+    console.log('Unsupported platform: ' + process.platform);
+    console.log('Supported platforms are: ' + supported.toString());
+    process.exit(1);
+}
+
+var stopping = false;
+var listener = null;
+
+var runService = function() {
+    var listenerExePath = path.join(__dirname, '../bin/Runner.Listener');
+    var interactive = process.argv[2] === "interactive";
+
+    if(!stopping) {
+        try {
+            if (interactive) {
+                console.log('Starting Runner listener interactively');
+                listener = childProcess.spawn(listenerExePath, ['run'].concat(process.argv.slice(3)), { env: process.env });
+            } else {
+                console.log('Starting Runner listener with startup type: service');
+                listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'].concat(process.argv.slice(2)), { env: process.env });
+            }
+
+            console.log('Started listener process');
+
+            listener.stdout.on('data', (data) => {
+                process.stdout.write(data.toString('utf8'));
+            });
+
+            listener.stderr.on('data', (data) => {
+                process.stdout.write(data.toString('utf8'));
+            });
+
+            listener.on('close', (code) => {
+                console.log(`Runner listener exited with error code ${code}`);
+
+                if (code === 0) {
+                    console.log('Runner listener exit with 0 return code, stop the service, no retry needed.');
+                    stopping = true;
+                } else if (code === 1) {
+                    console.log('Runner listener exit with terminated error, stop the service, no retry needed.');
+                    stopping = true;
+                } else if (code === 2) {
+                    console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.');
+                } else if (code === 3) {
+                    console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.');
+                } else {
+                    console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.');
+                }
+
+                if(!stopping) {
+                    setTimeout(runService, 5000);
+                }
+            });
+
+        } catch(ex) {
+            console.log(ex);
+        }
+    }
+}
+
+runService();
+console.log('Started running service');
+
+var gracefulShutdown = function(code) {
+    console.log('Shutting down runner listener');
+    stopping = true;
+    if (listener) {
+        console.log('Sending SIGINT to runner listener to stop');
+        listener.kill('SIGINT');
+
+        // TODO wait for 30 seconds and send a SIGKILL
+    }
+}
+
+process.on('SIGINT', () => {
+    gracefulShutdown(0);
+});
+
+process.on('SIGTERM', () => {
+    gracefulShutdown(0);
+});

runner/patched/runsvc.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# convert SIGTERM signal to SIGINT
+# for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
+trap 'kill -INT $PID' TERM INT
+
+if [ -f ".path" ]; then
+    # configure
+    export PATH=`cat .path`
+    echo ".path=${PATH}"
+fi
+
+# insert anything to setup env when running as a service
+
+# run the host process which keep the listener alive
+./externals/node12/bin/node ./bin/RunnerService.js $* &
+PID=$!
+wait $PID
+trap - TERM INT
+wait $PID

runner/startup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+source /opt/bash-utils/logger.sh
+
+function wait_for_process () {
+    local max_time_wait=30
+    local process_name="$1"
+    local waited_sec=0
+    while ! pgrep "$process_name" >/dev/null && ((waited_sec < max_time_wait)); do
+        INFO "Process $process_name is not running yet. Retrying in 1 seconds"
+        INFO "Waited $waited_sec seconds of $max_time_wait seconds"
+        sleep 1
+        ((waited_sec=waited_sec+1))
+        if ((waited_sec >= max_time_wait)); then
+            return 1
+        fi
+    done
+    return 0
+}
+
+INFO "Starting supervisor"
+sudo /usr/bin/supervisord -n >> /dev/null 2>&1 &
+
+INFO "Waiting for processes to be running"
+processes=(dockerd)
+
+for process in "${processes[@]}"; do
+    wait_for_process "$process"
+    if [ $? -ne 0 ]; then
+        ERROR "$process is not running after max time"
+        exit 1
+    else 
+        INFO "$process is running"
+    fi
+done
+
+# Wait processes to be running
+entrypoint.sh

runner/supervisor/dockerd.conf

@@ -0,0 +1,6 @@
+[program:dockerd]
+command=/usr/local/bin/dockerd
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/dockerd.err.log
+stdout_logfile=/var/log/dockerd.out.log