Added support for other than public GitHub URL (#146)

Refactoring a bit

.github/workflows/build-runner.yml

@@ -1,22 +1,79 @@
 on:
+  pull_request:
+    branches:
+      - '**'
+    paths:
+      - 'runner/**'
+      - .github/workflows/build-runner.yml
   push:
     branches:
       - master
     paths:
-    - 'runner/**'
-
+      - runner/patched/*
+      - runner/Dockerfile
+      - runner/dindrunner.Dockerfile
+      - runner/entrypoint.sh
+      - .github/workflows/build-runner.yml
+name: Runner
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: Build runner
+    name: Build
+    env:
+      RUNNER_VERSION: 2.273.5
+      DOCKER_VERSION: 19.03.12
+      DOCKERHUB_USERNAME: ${{ github.repository_owner }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-    - name: Build container image
-      run: make docker-build
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: latest
+
+      - name: Build Container Image
         working-directory: runner
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:latest \
+            -f Dockerfile .
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:latest \
+            -f dindrunner.Dockerfile .
+
+      - name: Login to GitHub Docker Registry
+        run: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin
+        if: ${{ github.event_name == 'push' }}
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build and Push Container Image
         working-directory: runner
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner:latest \
+            -f Dockerfile . --push
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:v${RUNNER_VERSION} \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-dind:latest \
+            -f dindrunner.Dockerfile . --push

.github/workflows/release.yml

@@ -9,6 +9,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+
       - name: Install tools
         run: |
           curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
@@ -19,15 +20,32 @@ jobs:
           curl -L -O https://github.com/tcnksm/ghr/releases/download/v0.13.0/ghr_v0.13.0_linux_amd64.tar.gz
           tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
           sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
+
       - name: Set version
         run: echo "::set-env name=VERSION::$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')"
+
       - name: Upload artifacts
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: make github-release
-    - name: Build container image
-      run: make docker-build
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: latest
+
+      - name: Login to GitHub Docker Registry
+        run: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build Container Image
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+        run: |
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-controller:${{ env.VERSION }} \
+            -f Dockerfile . --push

.github/workflows/test.yaml

@@ -1,15 +1,16 @@
+name: CI
+
 on:
-  push:
+  pull_request:
     branches:
       - master
     paths-ignore:
     - 'runner/**'
-    - '.github/**'
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
-    name: Build
+    name: Test
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -20,9 +21,7 @@ jobs:
         sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
     - name: Run tests
       run: make test
-    - name: Build container image
-      run: make docker-build
-    - name: Docker Login
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
-    - name: Push container image
-      run: make docker-push
+    - name: Verify manifests are up-to-date
+      run: |
+        make manifests
+        git diff --exit-code

.github/workflows/wip.yml

@@ -0,0 +1,35 @@
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - "runner/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: release-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: latest
+
+      - name: Login to GitHub Docker Registry
+        run: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build Container Image
+        env:
+          DOCKERHUB_USERNAME: ${{ github.repository_owner }}
+        run: |
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --tag ${DOCKERHUB_USERNAME}/actions-runner-controller:latest \
+            -f Dockerfile . --push

Dockerfile

@@ -1,28 +1,37 @@
 # Build the manager binary
 FROM golang:1.13 as builder
 
+ARG TARGETPLATFORM
+
 WORKDIR /workspace
+
+ENV GO111MODULE=on \
+  CGO_ENABLED=0
+
 # Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
+COPY go.mod go.sum ./
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
-COPY api/ api/
-COPY controllers/ controllers/
-COPY github/ github/
+COPY . .
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
+RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
+  export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
+  GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \
+  go build -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
+
 WORKDIR /
+
 COPY --from=builder /workspace/manager .
+
 USER nonroot:nonroot
 
 ENTRYPOINT ["/manager"]

Makefile

@@ -1,5 +1,8 @@
 NAME ?= summerwind/actions-runner-controller
 VERSION ?= latest
+# From https://github.com/VictoriaMetrics/operator/pull/44
+YAML_DROP=$(YQ) delete --inplace
+YAML_DROP_PREFIX=spec.validation.openAPIV3Schema.properties.spec.properties
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
@@ -11,6 +14,23 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
+
 all: manager
 
 # Run tests
@@ -39,7 +59,9 @@ deploy: manifests
 	kustomize build config/default | kubectl apply -f -
 
 # Generate manifests e.g. CRD, RBAC etc.
-manifests: controller-gen
+manifests: manifests-118 fix118
+
+manifests-118: controller-gen
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 # Run go fmt against code
@@ -50,6 +72,22 @@ fmt:
 vet:
 	go vet ./...
 
+# workaround for CRD issue with k8s 1.18 & controller-gen
+# ref: https://github.com/kubernetes/kubernetes/issues/91395
+fix118: yq
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml $(YAML_DROP_PREFIX).template.properties.spec.properties.ephemeralContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).containers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).initContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).sidecarContainers.items.properties
+	$(YAML_DROP) config/crd/bases/actions.summerwind.dev_runners.yaml $(YAML_DROP_PREFIX).ephemeralContainers.items.properties
+
 # Generate code
 generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
@@ -62,6 +100,18 @@ docker-build: test
 docker-push:
 	docker push ${NAME}:${VERSION}
 
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:${VERSION}" \
+		-f Dockerfile \
+		. ${PUSH_ARG}
+
 # Generate the release manifest file
 release: manifests
 	cd config/manager && kustomize edit set image controller=${NAME}:${VERSION}
@@ -76,15 +126,34 @@ github-release: release
 # download controller-gen if necessary
 controller-gen:
 ifeq (, $(shell which controller-gen))
+ifeq (, $(wildcard $(GOBIN)/controller-gen))
 	@{ \
 	set -e ;\
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
+endif
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 else
 CONTROLLER_GEN=$(shell which controller-gen)
 endif
+
+# find or download yq
+# download yq if necessary
+# Use always go-version to get consistent line wraps etc.
+yq:
+ifeq (, $(wildcard $(GOBIN)/yq))
+	echo "Downloading yq"
+	@{ \
+	set -e ;\
+	YQ_TMP_DIR=$$(mktemp -d) ;\
+	cd $$YQ_TMP_DIR ;\
+	go mod init tmp ;\
+	go get github.com/mikefarah/yq/v3@3.4.0 ;\
+	rm -rf $$YQ_TMP_DIR ;\
+	}
+endif
+YQ=$(GOBIN)/yq

README.md

@@ -20,9 +20,19 @@ Install the custom resource and actions-runner-controller itself. This will crea
 $ kubectl apply -f https://github.com/summerwind/actions-runner-controller/releases/latest/download/actions-runner-controller.yaml
 ```
 
+### Github Enterprise support
+
+If you use either Github Enterprise Cloud or Server (and have recent enought version supporting Actions), you can use **actions-runner-controller**  with those, too. Authentication works same way as with public Github (repo and organization level). 
+
+```
+$ kubectl set env deploy controller-manager -c manager GITHUB_ENTERPRISE_URL=<GHEC/S URL>
+```
+
+[Enterprise level](https://docs.github.com/en/enterprise-server@2.22/actions/hosting-your-own-runners/adding-self-hosted-runners#adding-a-self-hosted-runner-to-an-enterprise) runners are not working yet as there's no API definition for those.
+
 ## Setting up authentication with GitHub API
 
-There are two ways for actions-runner-controller to authenticate with the the GitHub API:
+There are two ways for actions-runner-controller to authenticate with the GitHub API:
 
 1. Using GitHub App.
 2. Using Personal Access Token.
@@ -135,7 +145,7 @@ The runner you created has been registered to your repository.
 
 <img width="756" alt="Actions tab in your repository settings" src="https://user-images.githubusercontent.com/230145/73618667-8cbf9700-466c-11ea-80b6-c67e6d3f70e7.png">
 
-Now your can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
+Now you can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
 
 ### Organization Runners
 
@@ -191,7 +201,7 @@ example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 
 #### Autoscaling
 
-`RunnerDeployment` can scale number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
+`RunnerDeployment` can scale the number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
 
 In the below example, `actions-runner` checks for pending workflow runs for each sync period, and scale to e.g. 3 if there're 3 pending jobs at sync time.
 
@@ -250,6 +260,27 @@ spec:
     repositoryNames:
     - summerwind/actions-runner-controller
 ```
+## Runner with DinD
+
+When using default runner, runner pod starts up 2 containers: runner and DinD (Docker-in-Docker). This might create issues if there's `LimitRange` set to namespace.
+
+```yaml
+# dindrunnerdeployment.yaml
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-dindrunnerdeploy
+spec:
+  replicas: 2
+  template:
+    spec:
+      image: summerwind/actions-runner-dind
+      dockerdWithinRunnerContainer: true
+      repository: mumoshu/actions-runner-controller-ci
+      env: []
+```
+
+This also helps with resources, as you don't need to give resources separately to docker and runner.
 
 ## Additional tweaks
 
@@ -283,6 +314,17 @@ spec:
         requests:
           cpu: "2.0"
           memory: "4Gi"
+      # If set to true, runner pod container only 1 container that's expected to be able to run docker, too.
+      # image summerwind/actions-runner-dind or custom one should be used with true -value
+      dockerdWithinRunnerContainer: false
+      # Valid if dockerdWithinRunnerContainer is not true
+      dockerdContainerResources:
+        limits:
+          cpu: "4.0"
+          memory: "8Gi"
+        requests:
+          cpu: "2.0"
+          memory: "4Gi"
       sidecarContainers:
         - name: mysql
           image: mysql:5.7
@@ -328,9 +370,9 @@ jobs:
     runs-on: custom-runner
 ```
 
-Note that if you specify `self-hosted` in your worlflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
+Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
 
-## Softeware installed in the runner image
+## Software installed in the runner image
 
 The GitHub hosted runners include a large amount of pre-installed software packages. For Ubuntu 18.04, this list can be found at https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md
 
@@ -364,3 +406,15 @@ spec:
   repository: summerwind/actions-runner-controller
   image: YOUR_CUSTOM_DOCKER_IMAGE
 ```
+
+# Alternatives
+
+The following is a list of alternative solutions that may better fit you depending on your use-case:
+
+- https://github.com/evryfs/github-actions-runner-operator/
+
+Although the situation can change over time, as of writing this sentence, the benefits of using `actions-runner-controller` over the alternatives are:
+
+- `actions-runner-controller` has the ability to autoscale runners based on number of pending/progressing jobs (#99)
+- `actions-runner-controller` is able to gracefully stop runners (#103)
+- `actions-runner-controller` has ARM support

api/v1alpha1/runner_types.go

@@ -39,6 +39,8 @@ type RunnerSpec struct {
 	// +optional
 	Containers []corev1.Container `json:"containers,omitempty"`
 	// +optional
+	DockerdContainerResources corev1.ResourceRequirements `json:"dockerdContainerResources,omitempty"`
+	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 	// +optional
 	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
@@ -77,6 +79,8 @@ type RunnerSpec struct {
 	EphemeralContainers []corev1.EphemeralContainer `json:"ephemeralContainers,omitempty"`
 	// +optional
 	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
+	// +optional
+	DockerdWithinRunnerContainer *bool `json:"dockerdWithinRunnerContainer,omitempty"`
 }
 
 // ValidateRepository validates repository field.

api/v1alpha1/zz_generated.deepcopy.go

@@ -435,6 +435,7 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.DockerdContainerResources.DeepCopyInto(&out.DockerdContainerResources)
 	in.Resources.DeepCopyInto(&out.Resources)
 	if in.VolumeMounts != nil {
 		in, out := &in.VolumeMounts, &out.VolumeMounts
@@ -524,6 +525,11 @@ func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) {
 		*out = new(int64)
 		**out = **in
 	}
+	if in.DockerdWithinRunnerContainer != nil {
+		in, out := &in.DockerdWithinRunnerContainer, &out.DockerdWithinRunnerContainer
+		*out = new(bool)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSpec.

config/certmanager/certificate.yaml

@@ -1,7 +1,7 @@
 # The following manifests contain a self-signed issuer CR and a certificate CR.
 # More document can be found at https://docs.cert-manager.io
 # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: selfsigned-issuer
@@ -9,7 +9,7 @@ metadata:
 spec:
   selfSigned: {}
 ---
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: horizontalrunnerautoscalers.actions.summerwind.dev
 spec:

config/default/kustomization.yaml

@@ -50,7 +50,7 @@ vars:
   objref:
     kind: Certificate
     group: cert-manager.io
-    version: v1alpha2
+    version: v1
     name: serving-cert # this name should match the one in certificate.yaml
   fieldref:
     fieldpath: metadata.namespace
@@ -58,7 +58,7 @@ vars:
   objref:
     kind: Certificate
     group: cert-manager.io
-    version: v1alpha2
+    version: v1
     name: serving-cert # this name should match the one in certificate.yaml
 - name: SERVICE_NAMESPACE # namespace of the service
   objref:

config/rbac/role.yaml

@@ -18,6 +18,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -38,6 +50,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -58,6 +82,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerreplicasets/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -78,6 +114,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -105,3 +153,15 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

controllers/autoscaling.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"strings"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 )
 
 func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
@@ -44,6 +45,38 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp
 	}
 
 	var total, inProgress, queued, completed, unknown int
+	type callback func()
+	listWorkflowJobs := func(user string, repoName string, runID int64, fallback_cb callback) {
+		if runID == 0 {
+			fallback_cb()
+			return
+		}
+		jobs, _, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, nil)
+		if err != nil {
+			r.Log.Error(err, "Error listing workflow jobs")
+			fallback_cb()
+		} else if len(jobs.Jobs) == 0 {
+			fallback_cb()
+		} else {
+			for _, job := range jobs.Jobs {
+				switch job.GetStatus() {
+				case "completed":
+					// We add a case for `completed` so it is not counted in `unknown`.
+					// And we do not increment the counter for completed because
+					// that counter only refers to workflows. The reason for
+					// this is because we do not get a list of jobs for
+					// completed workflows in order to keep the number of API
+					// calls to a minimum.
+				case "in_progress":
+					inProgress++
+				case "queued":
+					queued++
+				default:
+					unknown++
+				}
+			}
+		}
+	}
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
@@ -52,20 +85,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp
 			return nil, err
 		}
 
-		for _, r := range list.WorkflowRuns {
+		for _, run := range list.WorkflowRuns {
 			total++
 
 			// In May 2020, there are only 3 statuses.
 			// Follow the below links for more details:
 			// - https://developer.github.com/v3/actions/workflow-runs/#list-repository-workflow-runs
 			// - https://developer.github.com/v3/checks/runs/#create-a-check-run
-			switch r.GetStatus() {
+			switch run.GetStatus() {
 			case "completed":
 				completed++
 			case "in_progress":
-				inProgress++
+				listWorkflowJobs(user, repoName, run.GetID(), func() { inProgress++ })
 			case "queued":
-				queued++
+				listWorkflowJobs(user, repoName, run.GetID(), func() { queued++ })
 			default:
 				unknown++
 			}

controllers/autoscaling_test.go

@@ -2,20 +2,24 @@ package controllers
 
 import (
 	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"github.com/summerwind/actions-runner-controller/github"
 	"github.com/summerwind/actions-runner-controller/github/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"net/http/httptest"
-	"net/url"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	"testing"
 )
 
 func newGithubClient(server *httptest.Server) *github.Client {
-	client, err := github.NewClientWithAccessToken("token")
+	c := github.Config{
+		Token: "token",
+	}
+	client, err := c.NewClient()
 	if err != nil {
 		panic(err)
 	}
@@ -44,9 +48,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		sReplicas    *int
 		sTime        *metav1.Time
 		workflowRuns string
+		workflowJobs map[int]string
 		want         int
 		err          string
 	}{
+		// Legacy functionality
 		// 3 demanded, max at 3
 		{
 			repo:         "test/valid",
@@ -122,6 +128,21 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
 			want:         3,
 		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
 	}
 
 	for i := range testcases {
@@ -136,7 +157,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
 			defer server.Close()
 			client := newGithubClient(server)
 
@@ -211,6 +232,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		sReplicas    *int
 		sTime        *metav1.Time
 		workflowRuns string
+		workflowJobs map[int]string
 		want         int
 		err          string
 	}{
@@ -316,6 +338,22 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
 			err:          "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
 		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
 	}
 
 	for i := range testcases {
@@ -330,7 +368,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
 			defer server.Close()
 			client := newGithubClient(server)
 

controllers/horizontalrunnerautoscaler_controller.go

@@ -50,6 +50,7 @@ type HorizontalRunnerAutoscalerReconciler struct {
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 

controllers/integration_test.go

@@ -2,9 +2,10 @@ package controllers
 
 import (
 	"context"
-	"github.com/summerwind/actions-runner-controller/github/fake"
 	"time"
 
+	"github.com/summerwind/actions-runner-controller/github/fake"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -44,7 +45,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		Status: 200,
 		Body:   workflowRunsFor3Replicas,
 	}
-	server := fake.NewServer(fake.WithFixedResponses(responses))
+	fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses))
 
 	BeforeEach(func() {
 		stopCh = make(chan struct{})
@@ -58,11 +59,16 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+		ghClient := newGithubClient(server)
+
 		replicasetController := &RunnerReplicaSetReconciler{
 			Client:       mgr.GetClient(),
 			Scheme:       scheme.Scheme,
 			Log:          logf.Log,
 			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
 		}
 		err = replicasetController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -76,7 +82,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		err = deploymentsController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
 
-		client := newGithubClient(server)
+		client := newGithubClient(fakeGithubServer)
 
 		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
 			Client:       mgr.GetClient(),
@@ -99,7 +105,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 	AfterEach(func() {
 		close(stopCh)
 
-		server.Close()
+		fakeGithubServer.Close()
 
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")

controllers/runner_controller.go

@@ -53,8 +53,10 @@ type RunnerReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
@@ -165,7 +167,11 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		r.Recorder.Event(&runner, corev1.EventTypeNormal, "PodCreated", fmt.Sprintf("Created pod '%s'", newPod.Name))
 		log.Info("Created runner pod", "repository", runner.Spec.Repository)
 	} else {
-		if runner.Status.Phase != string(pod.Status.Phase) {
+		// If pod has ended up succeeded we need to restart it
+		// Happens e.g. when dind is in runner and run completes
+		restart := pod.Status.Phase == corev1.PodSucceeded
+
+		if !restart && runner.Status.Phase != string(pod.Status.Phase) {
 			updated := runner.DeepCopy()
 			updated.Status.Phase = string(pod.Status.Phase)
 			updated.Status.Reason = pod.Status.Reason
@@ -183,8 +189,6 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, err
 		}
 
-		restart := false
-
 		if pod.Status.Phase == corev1.PodRunning {
 			for _, status := range pod.Status.ContainerStatuses {
 				if status.Name != containerName {
@@ -203,12 +207,16 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, err
 		}
 
-		if pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image {
-			restart = true
-		}
-		if !reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) {
+		runnerBusy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+		if err != nil {
+			log.Error(err, "Failed to check if runner is busy")
+			return ctrl.Result{}, nil
+		}
+
+		if !runnerBusy && (!reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) || pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image) {
 			restart = true
 		}
+
 		if !restart {
 			return ctrl.Result{}, err
 		}
@@ -225,6 +233,21 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	return ctrl.Result{}, nil
 }
 
+func (r *RunnerReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, fmt.Errorf("runner not found")
+}
+
 func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name string) (bool, error) {
 	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
 	if err != nil {
@@ -234,6 +257,9 @@ func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name
 	id := int64(0)
 	for _, runner := range runners {
 		if runner.GetName() == name {
+			if runner.GetBusy() {
+				return false, fmt.Errorf("runner is busy")
+			}
 			id = runner.GetID()
 			break
 		}
@@ -253,7 +279,7 @@ func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name
 func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	var (
 		privileged      bool = true
-		group      int64 = 0
+		dockerdInRunner bool = runner.Spec.DockerdWithinRunnerContainer != nil && *runner.Spec.DockerdWithinRunnerContainer
 	)
 
 	runnerImage := runner.Spec.Image
@@ -287,6 +313,14 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 			Name:  "RUNNER_TOKEN",
 			Value: runner.Status.Registration.Token,
 		},
+		{
+			Name:  "DOCKERD_IN_RUNNER",
+			Value: fmt.Sprintf("%v", dockerdInRunner),
+		},
+		{
+			Name:  "GITHUB_URL",
+			Value: r.GitHubClient.GithubBaseURL,
+		},
 	}
 
 	env = append(env, runner.Spec.Env...)
@@ -306,7 +340,32 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 					ImagePullPolicy: runnerImagePullPolicy,
 					Env:             env,
 					EnvFrom:         runner.Spec.EnvFrom,
-					VolumeMounts: []corev1.VolumeMount{
+					SecurityContext: &corev1.SecurityContext{
+						// Runner need to run privileged if it contains DinD
+						Privileged: runner.Spec.DockerdWithinRunnerContainer,
+					},
+					Resources: runner.Spec.Resources,
+				},
+			},
+		},
+	}
+
+	if !dockerdInRunner {
+		pod.Spec.Volumes = []corev1.Volume{
+			{
+				Name: "work",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+			{
+				Name: "docker",
+				VolumeSource: corev1.VolumeSource{
+					EmptyDir: &corev1.EmptyDirVolumeSource{},
+				},
+			},
+		}
+		pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
 			{
 				Name:      "work",
 				MountPath: "/runner/_work",
@@ -315,13 +374,8 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 				Name:      "docker",
 				MountPath: "/var/run",
 			},
-					},
-					SecurityContext: &corev1.SecurityContext{
-						RunAsGroup: &group,
-					},
-					Resources: runner.Spec.Resources,
-				},
-				{
+		}
+		pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
 			Name:  "docker",
 			Image: r.DockerImage,
 			VolumeMounts: []corev1.VolumeMount{
@@ -337,27 +391,17 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 			SecurityContext: &corev1.SecurityContext{
 				Privileged: &privileged,
 			},
-				},
-			},
-			Volumes: []corev1.Volume{
-				{
-					Name: "work",
-					VolumeSource: corev1.VolumeSource{
-						EmptyDir: &corev1.EmptyDirVolumeSource{},
-					},
-				},
-				{
-					Name: "docker",
-					VolumeSource: corev1.VolumeSource{
-						EmptyDir: &corev1.EmptyDirVolumeSource{},
-					},
-				},
-			},
-		},
+		})
+
 	}
 
 	if len(runner.Spec.Containers) != 0 {
 		pod.Spec.Containers = runner.Spec.Containers
+		for i := 0; i < len(pod.Spec.Containers); i++ {
+			if pod.Spec.Containers[i].Name == containerName {
+				pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, env...)
+			}
+		}
 	}
 
 	if len(runner.Spec.VolumeMounts) != 0 {

controllers/runnerdeployment_controller.go

@@ -54,6 +54,7 @@ type RunnerDeploymentReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch

controllers/runnerreplicaset_controller.go

@@ -31,6 +31,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github"
 )
 
 // RunnerReplicaSetReconciler reconciles a Runner object
@@ -39,9 +40,11 @@ type RunnerReplicaSetReconciler struct {
 	Log          logr.Logger
 	Recorder     record.EventRecorder
 	Scheme       *runtime.Scheme
+	GitHubClient *github.Client
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
@@ -96,8 +99,25 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	if available > desired {
 		n := available - desired
 
+		// get runners that are currently not busy
+		var notBusy []v1alpha1.Runner
+		for _, runner := range myRunners {
+			busy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+			if err != nil {
+				log.Error(err, "Failed to check if runner is busy")
+				return ctrl.Result{}, err
+			}
+			if !busy {
+				notBusy = append(notBusy, runner)
+			}
+		}
+
+		if len(notBusy) < n {
+			n = len(notBusy)
+		}
+
 		for i := 0; i < n; i++ {
-			if err := r.Client.Delete(ctx, &myRunners[i]); err != nil {
+			if err := r.Client.Delete(ctx, &notBusy[i]); err != nil {
 				log.Error(err, "Failed to delete runner resource")
 
 				return ctrl.Result{}, err
@@ -166,3 +186,19 @@ func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&v1alpha1.Runner{}).
 		Complete(r)
 }
+
+func (r *RunnerReplicaSetReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
+	r.Log.Info("runners", "github", runners)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, fmt.Errorf("runner not found")
+}

controllers/runnerreplicaset_controller_test.go

@@ -3,11 +3,14 @@ package controllers
 import (
 	"context"
 	"math/rand"
+	"net/http/httptest"
 	"time"
 
+	"github.com/google/go-github/v32/github"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -17,6 +20,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+)
+
+var (
+	runnersList *fake.RunnersList
+	server      *httptest.Server
 )
 
 // SetupTest will set up a testing environment.
@@ -41,11 +50,16 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+		ghClient := newGithubClient(server)
+
 		controller := &RunnerReplicaSetReconciler{
 			Client:       mgr.GetClient(),
 			Scheme:       scheme.Scheme,
 			Log:          logf.Log,
 			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -61,6 +75,7 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 	AfterEach(func() {
 		close(stopCh)
 
+		server.Close()
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
 	})
@@ -124,6 +139,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
@@ -155,6 +180,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
@@ -186,6 +221,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(0))

github/fake/fake.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"time"
+	"unicode"
 )
 
 const (
@@ -14,8 +17,8 @@ const (
 {
   "total_count": 2,
   "runners": [
-    {"id": 1, "name": "test1", "os": "linux", "status": "online"},
-    {"id": 2, "name": "test2", "os": "linux", "status": "offline"}
+    {"id": 1, "name": "test1", "os": "linux", "status": "online", "busy": false},
+    {"id": 2, "name": "test2", "os": "linux", "status": "offline", "busy": false}
   ]
 }
 `
@@ -31,6 +34,24 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	fmt.Fprintf(w, h.Body)
 }
 
+type MapHandler struct {
+	Status int
+	Bodies map[int]string
+}
+
+func (h *MapHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// Parse out int key from URL path
+	key, err := strconv.Atoi(strings.TrimFunc(req.URL.Path, func(r rune) bool { return !unicode.IsNumber(r) }))
+	if err != nil {
+		w.WriteHeader(400)
+	} else if body := h.Bodies[key]; len(body) == 0 {
+		w.WriteHeader(404)
+	} else {
+		w.WriteHeader(h.Status)
+		fmt.Fprintf(w, body)
+	}
+}
+
 type ServerConfig struct {
 	*FixedResponses
 }
@@ -45,7 +66,7 @@ func NewServer(opts ...Option) *httptest.Server {
 		o(&config)
 	}
 
-	routes := map[string]*Handler{
+	routes := map[string]http.Handler{
 		// For CreateRegistrationToken
 		"/repos/test/valid/actions/runners/registration-token": &Handler{
 			Status: http.StatusCreated,
@@ -126,6 +147,9 @@ func NewServer(opts ...Option) *httptest.Server {
 
 		// For auto-scaling based on the number of queued(pending) workflow runs
 		"/repos/test/valid/actions/runs": config.FixedResponses.ListRepositoryWorkflowRuns,
+
+		// For auto-scaling based on the number of queued(pending) workflow jobs
+		"/repos/test/valid/actions/runs/": config.FixedResponses.ListWorkflowJobs,
 	}
 
 	mux := http.NewServeMux()

github/fake/options.go

@@ -2,6 +2,7 @@ package fake
 
 type FixedResponses struct {
 	ListRepositoryWorkflowRuns *Handler
+	ListWorkflowJobs           *MapHandler
 }
 
 type Option func(*ServerConfig)
@@ -15,6 +16,15 @@ func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
 	}
 }
 
+func WithListWorkflowJobsResponse(status int, bodies map[int]string) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListWorkflowJobs = &MapHandler{
+			Status: status,
+			Bodies: bodies,
+		}
+	}
+}
+
 func WithFixedResponses(responses *FixedResponses) Option {
 	return func(c *ServerConfig) {
 		c.FixedResponses = responses

github/fake/runners.go

@@ -0,0 +1,74 @@
+package fake
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+
+	"github.com/google/go-github/v32/github"
+	"github.com/gorilla/mux"
+)
+
+type RunnersList struct {
+	runners []*github.Runner
+}
+
+func NewRunnersList() *RunnersList {
+	return &RunnersList{
+		runners: make([]*github.Runner, 0),
+	}
+}
+
+func (r *RunnersList) Add(runner *github.Runner) {
+	if !exists(r.runners, runner) {
+		r.runners = append(r.runners, runner)
+	}
+}
+
+func (r *RunnersList) GetServer() *httptest.Server {
+	router := mux.NewRouter()
+
+	router.Handle("/repos/{owner}/{repo}/actions/runners", r.handleList())
+	router.Handle("/repos/{owner}/{repo}/actions/runners/{id}", r.handleRemove())
+	router.Handle("/orgs/{org}/actions/runners", r.handleList())
+	router.Handle("/orgs/{org}/actions/runners/{id}", r.handleRemove())
+
+	return httptest.NewServer(router)
+}
+
+func (r *RunnersList) handleList() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		j, err := json.Marshal(github.Runners{
+			TotalCount: len(r.runners),
+			Runners:    r.runners,
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		w.WriteHeader(http.StatusOK)
+		w.Write(j)
+	}
+}
+
+func (r *RunnersList) handleRemove() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		vars := mux.Vars(res)
+		for i, runner := range r.runners {
+			if runner.ID != nil && vars["id"] == strconv.FormatInt(*runner.ID, 10) {
+				r.runners = append(r.runners[:i], r.runners[i+1:]...)
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func exists(runners []*github.Runner, runner *github.Runner) bool {
+	for _, r := range runners {
+		if *r.Name == *runner.Name {
+			return true
+		}
+	}
+	return false
+}

github/github.go

@@ -9,43 +9,66 @@ import (
 	"time"
 
 	"github.com/bradleyfalzon/ghinstallation"
-	"github.com/google/go-github/v31/github"
+	"github.com/go-logr/logr"
+	"github.com/google/go-github/v32/github"
 	"golang.org/x/oauth2"
 )
 
+// Config contains configuration for Github client
+type Config struct {
+	Log               logr.Logger
+	EnterpriseURL     string `split_words:"true"`
+	AppID             int64  `split_words:"true"`
+	AppInstallationID int64  `split_words:"true"`
+	AppPrivateKey     string `split_words:"true"`
+	Token             string
+}
+
 // Client wraps GitHub client with some additional
 type Client struct {
 	*github.Client
 	regTokens map[string]*github.RegistrationToken
 	mu        sync.Mutex
+	// GithubBaseURL to Github without API suffix.
+	GithubBaseURL string
 }
 
-// NewClient returns a client authenticated as a GitHub App.
-func NewClient(appID, installationID int64, privateKeyPath string) (*Client, error) {
-	tr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, appID, installationID, privateKeyPath)
+func (c *Config) NewClient() (*Client, error) {
+	var (
+		httpClient *http.Client
+		client     *github.Client
+	)
+	githubBaseURL := "https://github.com/"
+	if len(c.Token) > 0 {
+		httpClient = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: c.Token},
+		))
+	} else {
+		tr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.AppID, c.AppInstallationID, c.AppPrivateKey)
 		if err != nil {
+			c.Log.Error(err, "Authentication failed")
 			return nil, fmt.Errorf("authentication failed: %v", err)
 		}
-
-	gh := github.NewClient(&http.Client{Transport: tr})
-
-	return &Client{
-		Client:    gh,
-		regTokens: map[string]*github.RegistrationToken{},
-		mu:        sync.Mutex{},
-	}, nil
+		httpClient = &http.Client{Transport: tr}
 	}
 
-// NewClientWithAccessToken returns a client authenticated with personal access token.
-func NewClientWithAccessToken(token string) (*Client, error) {
-	tc := oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: token},
-	))
+	if len(c.EnterpriseURL) > 0 {
+		var err error
+		client, err = github.NewEnterpriseClient(c.EnterpriseURL, c.EnterpriseURL, httpClient)
+		if err != nil {
+			c.Log.Error(err, "Enterprise client creation failed")
+			return nil, fmt.Errorf("enterprise client creation failed: %v", err)
+		}
+		githubBaseURL = fmt.Sprintf("%s://%s%s", client.BaseURL.Scheme, client.BaseURL.Host, strings.TrimSuffix(client.BaseURL.Path, "api/v3/"))
+	} else {
+		client = github.NewClient(httpClient)
+	}
 
 	return &Client{
-		Client:    github.NewClient(tc),
+		Client:        client,
 		regTokens:     map[string]*github.RegistrationToken{},
 		mu:            sync.Mutex{},
+		GithubBaseURL: githubBaseURL,
 	}, nil
 }
 
@@ -85,7 +108,7 @@ func (c *Client) GetRegistrationToken(ctx context.Context, org, repo, name strin
 	return rt, nil
 }
 
-// RemoveRunner removes a runner with specified runner ID from repocitory.
+// RemoveRunner removes a runner with specified runner ID from repository.
 func (c *Client) RemoveRunner(ctx context.Context, org, repo string, runnerID int64) error {
 	owner, repo, err := getOwnerAndRepo(org, repo)
 
@@ -121,7 +144,7 @@ func (c *Client) ListRunners(ctx context.Context, org, repo string) ([]*github.R
 		list, res, err := c.listRunners(ctx, owner, repo, &opts)
 
 		if err != nil {
-			return runners, fmt.Errorf("failed to remove runner: %v", err)
+			return runners, fmt.Errorf("failed to list runners: %v", err)
 		}
 
 		runners = append(runners, list.Runners...)

github/github_beta.go

@@ -10,7 +10,7 @@ import (
 	"net/url"
 	"reflect"
 
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"github.com/google/go-querystring/query"
 )
 

github/github_test.go

@@ -7,14 +7,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"github.com/summerwind/actions-runner-controller/github/fake"
 )
 
 var server *httptest.Server
 
 func newTestClient() *Client {
-	client, err := NewClientWithAccessToken("token")
+	c := Config{
+		Token: "token",
+	}
+	client, err := c.NewClient()
 	if err != nil {
 		panic(err)
 	}
@@ -41,9 +44,9 @@ func TestGetRegistrationToken(t *testing.T) {
 		token string
 		err   bool
 	}{
-		{org: "test", repo: "valid", token: fake.RegistrationToken, err: false},
-		{org: "test", repo: "invalid", token: "", err: true},
-		{org: "test", repo: "error", token: "", err: true},
+		{org: "", repo: "test/valid", token: fake.RegistrationToken, err: false},
+		{org: "", repo: "test/invalid", token: "", err: true},
+		{org: "", repo: "test/error", token: "", err: true},
 		{org: "test", repo: "", token: fake.RegistrationToken, err: false},
 		{org: "invalid", repo: "", token: "", err: true},
 		{org: "error", repo: "", token: "", err: true},
@@ -68,9 +71,9 @@ func TestListRunners(t *testing.T) {
 		length int
 		err    bool
 	}{
-		{org: "test", repo: "valid", length: 2, err: false},
-		{org: "test", repo: "invalid", length: 0, err: true},
-		{org: "test", repo: "error", length: 0, err: true},
+		{org: "", repo: "test/valid", length: 2, err: false},
+		{org: "", repo: "test/invalid", length: 0, err: true},
+		{org: "", repo: "test/error", length: 0, err: true},
 		{org: "test", repo: "", length: 2, err: false},
 		{org: "invalid", repo: "", length: 0, err: true},
 		{org: "error", repo: "", length: 0, err: true},
@@ -94,9 +97,9 @@ func TestRemoveRunner(t *testing.T) {
 		repo string
 		err  bool
 	}{
-		{org: "test", repo: "valid", err: false},
-		{org: "test", repo: "invalid", err: true},
-		{org: "test", repo: "error", err: true},
+		{org: "", repo: "test/valid", err: false},
+		{org: "", repo: "test/invalid", err: true},
+		{org: "", repo: "test/error", err: true},
 		{org: "test", repo: "", err: false},
 		{org: "invalid", repo: "", err: true},
 		{org: "error", repo: "", err: true},

go.mod

@@ -6,8 +6,10 @@ require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v0.1.0
-	github.com/google/go-github/v31 v31.0.0
+	github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04
 	github.com/google/go-querystring v1.0.0
+	github.com/gorilla/mux v1.8.0
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.5.0
 	github.com/stretchr/testify v1.4.0 // indirect
@@ -15,6 +17,6 @@ require (
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
 	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
 	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
-	k8s.io/klog v0.4.0
+	k8s.io/utils v0.0.0-20190801114015-581e00157fb1
 	sigs.k8s.io/controller-runtime v0.4.0
 )

go.sum

@@ -116,11 +116,10 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
-github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo=
-github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM=
+github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04 h1:wEYk2h/GwOhImcVjiTIceP88WxVbXw2F+ARYUQMEsfg=
+github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@@ -136,6 +135,8 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC
 github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
 github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -157,6 +158,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

main.go

@@ -20,9 +20,9 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strconv"
 	"time"
 
+	"github.com/kelseyhightower/envconfig"
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"github.com/summerwind/actions-runner-controller/controllers"
 	"github.com/summerwind/actions-runner-controller/github"
@@ -62,68 +62,29 @@ func main() {
 
 		runnerImage string
 		dockerImage string
-
-		ghToken             string
-		ghAppID             int64
-		ghAppInstallationID int64
-		ghAppPrivateKey     string
 	)
 
+	var c github.Config
+	err = envconfig.Process("github", &c)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error: Environment variable read failed.")
+	}
+
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&runnerImage, "runner-image", defaultRunnerImage, "The image name of self-hosted runner container.")
 	flag.StringVar(&dockerImage, "docker-image", defaultDockerImage, "The image name of docker sidecar container.")
-	flag.StringVar(&ghToken, "github-token", "", "The personal access token of GitHub.")
-	flag.Int64Var(&ghAppID, "github-app-id", 0, "The application ID of GitHub App.")
-	flag.Int64Var(&ghAppInstallationID, "github-app-installation-id", 0, "The installation ID of GitHub App.")
-	flag.StringVar(&ghAppPrivateKey, "github-app-private-key", "", "The path of a private key file to authenticate as a GitHub App")
+	flag.StringVar(&c.Token, "github-token", c.Token, "The personal access token of GitHub.")
+	flag.Int64Var(&c.AppID, "github-app-id", c.AppID, "The application ID of GitHub App.")
+	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
+	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
 	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.Parse()
 
-	if ghToken == "" {
-		ghToken = os.Getenv("GITHUB_TOKEN")
-	}
-	if ghAppID == 0 {
-		appID, err := strconv.ParseInt(os.Getenv("GITHUB_APP_ID"), 10, 64)
-		if err == nil {
-			ghAppID = appID
-		}
-	}
-	if ghAppInstallationID == 0 {
-		appInstallationID, err := strconv.ParseInt(os.Getenv("GITHUB_APP_INSTALLATION_ID"), 10, 64)
-		if err == nil {
-			ghAppInstallationID = appInstallationID
-		}
-	}
-	if ghAppPrivateKey == "" {
-		ghAppPrivateKey = os.Getenv("GITHUB_APP_PRIVATE_KEY")
-	}
-
-	if ghAppID != 0 {
-		if ghAppInstallationID == 0 {
-			fmt.Fprintln(os.Stderr, "Error: The installation ID must be specified.")
-			os.Exit(1)
-		}
-
-		if ghAppPrivateKey == "" {
-			fmt.Fprintln(os.Stderr, "Error: The path of a private key file must be specified.")
-			os.Exit(1)
-		}
-
-		ghClient, err = github.NewClient(ghAppID, ghAppInstallationID, ghAppPrivateKey)
+	ghClient, err = c.NewClient()
 	if err != nil {
-			fmt.Fprintf(os.Stderr, "Error: Failed to create GitHub client: %v\n", err)
-			os.Exit(1)
-		}
-	} else if ghToken != "" {
-		ghClient, err = github.NewClientWithAccessToken(ghToken)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Error: Failed to create GitHub client: %v\n", err)
-			os.Exit(1)
-		}
-	} else {
-		fmt.Fprintln(os.Stderr, "Error: GitHub App credentials or personal access token must be specified.")
+		fmt.Fprintln(os.Stderr, "Error: Client creation failed.", err)
 		os.Exit(1)
 	}
 
@@ -161,6 +122,7 @@ func main() {
 		Client:       mgr.GetClient(),
 		Log:          ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
 		Scheme:       mgr.GetScheme(),
+		GitHubClient: ghClient,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {

runner/Dockerfile

@@ -1,7 +1,8 @@
 FROM ubuntu:18.04
 
-ARG RUNNER_VERSION
-ARG DOCKER_VERSION
+ARG TARGETPLATFORM
+ARG RUNNER_VERSION=2.272.0
+ARG DOCKER_VERSION=19.03.12
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update -y \
@@ -36,26 +37,38 @@ RUN apt update -y \
   zstd \
   && rm -rf /var/lib/apt/lists/*
 
-RUN curl -L -o docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz \
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} \
+  && chmod +x /usr/local/bin/dumb-init
+
+# Docker download supports arm64 as aarch64 & amd64 as x86_64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \
+  && curl -L -o docker.tgz https://download.docker.com/linux/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz \
   && tar zxvf docker.tgz \
   && install -o root -g root -m 755 docker/docker /usr/local/bin/docker \
   && rm -rf docker docker.tgz \
-  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \
-  && chmod +x /usr/local/bin/dumb-init \
   && adduser --disabled-password --gecos "" --uid 1000 runner \
+  && groupadd docker \
   && usermod -aG sudo runner \
+  && usermod -aG docker runner \
   && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
 
-RUN mkdir -p /runner \
+# Runner download supports amd64 as x64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
+  && mkdir -p /runner \
   && cd /runner \
-  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
   && tar xzf ./runner.tar.gz \
   && rm runner.tar.gz \
   && ./bin/installdependencies.sh \
   && rm -rf /var/lib/apt/lists/*
 
 COPY entrypoint.sh /runner
+COPY patched /runner/patched
 
-USER runner:runner
+USER runner
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
 CMD ["/runner/entrypoint.sh"]

runner/Makefile

@@ -1,11 +1,52 @@
 NAME ?= summerwind/actions-runner
+DIND_RUNNER_NAME ?= ${NAME}-dind
+TAG ?= latest
 
-RUNNER_VERSION ?= 2.272.0
+RUNNER_VERSION ?= 2.273.5
 DOCKER_VERSION ?= 19.03.12
 
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
+
 docker-build:
-	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:latest -t ${NAME}:v${RUNNER_VERSION} .
+	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} -t ${NAME}:v${RUNNER_VERSION} .
+	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${DIND_RUNNER_NAME}:${TAG} -t ${DIND_RUNNER_NAME}:v${RUNNER_VERSION} -f dindrunner.Dockerfile .
+
 
 docker-push:
-	docker push ${NAME}:latest
+	docker push ${NAME}:${TAG}
 	docker push ${NAME}:v${RUNNER_VERSION}
+	docker push ${DIND_RUNNER_NAME}:${TAG}
+	docker push ${DIND_RUNNER_NAME}:v${RUNNER_VERSION}
+
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:latest" \
+		-f Dockerfile \
+		. ${PUSH_ARG}
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${DIND_RUNNER_NAME}:latest" \
+		-f dindrunner.Dockerfile \
+		. ${PUSH_ARG}

runner/dindrunner.Dockerfile

@@ -0,0 +1,100 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+# Dev + DinD dependencies
+RUN apt update \
+    && apt install -y software-properties-common \
+    && add-apt-repository -y ppa:git-core/ppa \
+    && apt install -y \
+    build-essential \
+    curl \
+    ca-certificates \
+    dnsutils \
+    ftp \
+    git \
+    iproute2 \
+    iptables \
+    iputils-ping \
+    jq \
+    libunwind8 \
+    locales \
+    netcat \
+    openssh-client \
+    parallel \
+    rsync \
+    shellcheck \
+    sudo \
+    supervisor \
+    telnet \
+    time \
+    tzdata \
+    unzip \
+    upx \
+    wget \
+    zip \
+    zstd \
+    && rm -rf /var/lib/apt/list/*
+
+# Runner user
+RUN adduser --disabled-password --gecos "" --uid 1000 runner \
+    && groupadd docker \
+    && usermod -aG sudo runner \
+    && usermod -aG docker runner \
+    && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
+
+ARG TARGETPLATFORM
+ARG RUNNER_VERSION=2.272.0
+ARG DOCKER_CHANNEL=stable
+ARG DOCKER_VERSION=19.03.13
+ARG DEBUG=false
+
+# Docker installation
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
+    && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \
+	&& if ! curl -L -o docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz"; then \
+		echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${ARCH}'"; \
+		exit 1; \
+	fi; \
+    echo "Downloaded Docker from https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz"; \
+	tar --extract \
+		--file docker.tgz \
+		--strip-components 1 \
+		--directory /usr/local/bin/ \
+	; \
+	rm docker.tgz; \
+	dockerd --version; \
+	docker --version
+
+# Runner download supports amd64 as x64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
+    && mkdir -p /runner \
+     && cd /runner \
+    && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
+    && tar xzf ./runner.tar.gz \
+    && rm runner.tar.gz \
+    && ./bin/installdependencies.sh \
+    && rm -rf /var/lib/apt/lists/*
+
+
+COPY modprobe startup.sh /usr/local/bin/
+COPY supervisor/ /etc/supervisor/conf.d/
+COPY logger.sh /opt/bash-utils/logger.sh
+COPY entrypoint.sh /usr/local/bin/
+
+RUN chmod +x /usr/local/bin/startup.sh /usr/local/bin/entrypoint.sh /usr/local/bin/modprobe
+
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+    && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} \
+    && chmod +x /usr/local/bin/dumb-init
+
+VOLUME /var/lib/docker
+
+COPY patched /runner/patched
+
+# No group definition, as that makes it harder to run docker.
+USER runner
+
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
+CMD ["startup.sh"]

runner/entrypoint.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 
+if [ -z "${GITHUB_URL}" ]; then
+  echo "Working with public GitHub" 1>&2
+  GITHUB_URL="https://github.com/"
+else
+  length=${#GITHUB_URL}
+  last_char=${GITHUB_URL:length-1:1}
+
+  [[ $last_char != "/" ]] && GITHUB_URL="$GITHUB_URL/"; :
+  echo "Github endpoint URL ${GITHUB_URL}"
+fi
+
 if [ -z "${RUNNER_NAME}" ]; then
   echo "RUNNER_NAME must be set" 1>&2
   exit 1
@@ -26,7 +37,13 @@ if [ -z "${RUNNER_TOKEN}" ]; then
 fi
 
 cd /runner
-./config.sh --unattended --replace --name "${RUNNER_NAME}" --url "https://github.com/${ATTACH}" --token "${RUNNER_TOKEN}" ${LABEL_ARG}
+./config.sh --unattended --replace --name "${RUNNER_NAME}" --url "${GITHUB_URL}${ATTACH}" --token "${RUNNER_TOKEN}" ${LABEL_ARG}
+
+for f in runsvc.sh RunnerService.js; do
+  diff {bin,patched}/${f} || :
+  sudo mv bin/${f}{,.bak}
+  sudo mv {patched,bin}/${f}
+done
 
 unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN
-exec ./run.sh --once
+exec ./bin/runsvc.sh --once

runner/logger.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Logger from this post http://www.cubicrace.com/2016/03/log-tracing-mechnism-for-shell-scripts.html
+
+function INFO(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [INFO] [${0}] $msg"
+}
+
+
+function DEBUG(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [DEBUG] [${0}] $msg"
+}
+
+function ERROR(){
+    local function_name="${FUNCNAME[1]}"
+    local msg="$1"
+    timeAndDate=`date`
+    echo "[$timeAndDate] [ERROR]  $msg"
+}

runner/modprobe

@@ -0,0 +1,20 @@
+ #!/bin/sh
+set -eu
+
+# "modprobe" without modprobe
+# https://twitter.com/lucabruno/status/902934379835662336
+
+# this isn't 100% fool-proof, but it'll have a much higher success rate than simply using the "real" modprobe
+
+# Docker often uses "modprobe -va foo bar baz"
+# so we ignore modules that start with "-"
+for module; do
+	if [ "${module#-}" = "$module" ]; then
+		ip link show "$module" || true
+		lsmod | grep "$module" || true
+	fi
+done
+
+# remove /usr/local/... from PATH so we can exec the real modprobe as a last resort
+export PATH='/usr/sbin:/usr/bin:/sbin:/bin'
+exec modprobe "$@"

runner/patched/RunnerService.js

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+// Copyright (c) GitHub. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+var childProcess = require("child_process");
+var path = require("path")
+
+var supported = ['linux', 'darwin']
+
+if (supported.indexOf(process.platform) == -1) {
+    console.log('Unsupported platform: ' + process.platform);
+    console.log('Supported platforms are: ' + supported.toString());
+    process.exit(1);
+}
+
+var stopping = false;
+var listener = null;
+
+var runService = function() {
+    var listenerExePath = path.join(__dirname, '../bin/Runner.Listener');
+    var interactive = process.argv[2] === "interactive";
+
+    if(!stopping) {
+        try {
+            if (interactive) {
+                console.log('Starting Runner listener interactively');
+                listener = childProcess.spawn(listenerExePath, ['run'].concat(process.argv.slice(3)), { env: process.env });
+            } else {
+                console.log('Starting Runner listener with startup type: service');
+                listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'].concat(process.argv.slice(2)), { env: process.env });
+            }
+
+            console.log('Started listener process');
+
+            listener.stdout.on('data', (data) => {
+                process.stdout.write(data.toString('utf8'));
+            });
+
+            listener.stderr.on('data', (data) => {
+                process.stdout.write(data.toString('utf8'));
+            });
+
+            listener.on('close', (code) => {
+                console.log(`Runner listener exited with error code ${code}`);
+
+                if (code === 0) {
+                    console.log('Runner listener exit with 0 return code, stop the service, no retry needed.');
+                    stopping = true;
+                } else if (code === 1) {
+                    console.log('Runner listener exit with terminated error, stop the service, no retry needed.');
+                    stopping = true;
+                } else if (code === 2) {
+                    console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.');
+                } else if (code === 3) {
+                    console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.');
+                } else {
+                    console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.');
+                }
+
+                if(!stopping) {
+                    setTimeout(runService, 5000);
+                }
+            });
+
+        } catch(ex) {
+            console.log(ex);
+        }
+    }
+}
+
+runService();
+console.log('Started running service');
+
+var gracefulShutdown = function(code) {
+    console.log('Shutting down runner listener');
+    stopping = true;
+    if (listener) {
+        console.log('Sending SIGINT to runner listener to stop');
+        listener.kill('SIGINT');
+
+        // TODO wait for 30 seconds and send a SIGKILL
+    }
+}
+
+process.on('SIGINT', () => {
+    gracefulShutdown(0);
+});
+
+process.on('SIGTERM', () => {
+    gracefulShutdown(0);
+});

runner/patched/runsvc.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# convert SIGTERM signal to SIGINT
+# for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
+trap 'kill -INT $PID' TERM INT
+
+if [ -f ".path" ]; then
+    # configure
+    export PATH=`cat .path`
+    echo ".path=${PATH}"
+fi
+
+# insert anything to setup env when running as a service
+
+# run the host process which keep the listener alive
+./externals/node12/bin/node ./bin/RunnerService.js $* &
+PID=$!
+wait $PID
+trap - TERM INT
+wait $PID

runner/startup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+source /opt/bash-utils/logger.sh
+
+function wait_for_process () {
+    local max_time_wait=30
+    local process_name="$1"
+    local waited_sec=0
+    while ! pgrep "$process_name" >/dev/null && ((waited_sec < max_time_wait)); do
+        INFO "Process $process_name is not running yet. Retrying in 1 seconds"
+        INFO "Waited $waited_sec seconds of $max_time_wait seconds"
+        sleep 1
+        ((waited_sec=waited_sec+1))
+        if ((waited_sec >= max_time_wait)); then
+            return 1
+        fi
+    done
+    return 0
+}
+
+INFO "Starting supervisor"
+sudo /usr/bin/supervisord -n >> /dev/null 2>&1 &
+
+INFO "Waiting for processes to be running"
+processes=(dockerd)
+
+for process in "${processes[@]}"; do
+    wait_for_process "$process"
+    if [ $? -ne 0 ]; then
+        ERROR "$process is not running after max time"
+        exit 1
+    else 
+        INFO "$process is running"
+    fi
+done
+
+# Wait processes to be running
+entrypoint.sh

runner/supervisor/dockerd.conf

@@ -0,0 +1,6 @@
+[program:dockerd]
+command=/usr/local/bin/dockerd
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/dockerd.err.log
+stdout_logfile=/var/log/dockerd.out.log