Fixing Docker Build and Push for Runner Image (#115)

A new image tag for the runner stopped being published on master merges from changes in #86.

This fixes that in the following ways:
- Tests the GH workflow on PRs w/o pushing the images
- Runner and Docker version are moved from Makefile to GH Actions workflow and are passed in as build args
- GHA workflow runs on PRs, and if the workflow file itself is changed (i.e. version bump) or the runner Docker source changes (excluding the Makefile since thats just for local dev)
- Images are pushed on push (i.e. a merge)

.github/workflows/build-runner.yml

@@ -1,22 +1,51 @@
 on:
+  pull_request:
+    branches:
+      - '**'
+    paths:
+      - 'runner/**'
+      - .github/workflows/build-runner.yml
   push:
     branches:
-    - master
+      - master
     paths:
-    - 'runner/**'
+      - runner/patched/*
-
+      - runner/Dockerfile
+      - runner/entrypoint.sh
+      - .github/workflows/build-runner.yml
+name: Runner
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: Build runner
+    name: Build
+    env:
+      RUNNER_VERSION: 2.273.5
+      DOCKER_VERSION: 19.03.12
     steps:
-    - name: Checkout
+      - name: Checkout
-      uses: actions/checkout@v2
+        uses: actions/checkout@v2
-    - name: Build container image
+
-      run: make docker-build
+      - name: Set up Docker Buildx
-      working-directory: runner
+        id: buildx
-    - name: Docker Login
+        uses: crazy-max/ghaction-docker-buildx@v1
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
+        with:
-    - name: Push container image
+          buildx-version: latest
-      run: make docker-push
+
-      working-directory: runner
+      - name: Build Container Image
+        working-directory: runner
+        run: |
+          docker buildx build \
+            --build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+            --build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+            --platform linux/amd64,linux/arm64 \
+            --tag summerwind/actions-runner:v${RUNNER_VERSION} \
+            -f Dockerfile .
+
+      - name: Push Container Image
+        working-directory: runner
+        run: |
+          docker login -u summerwind --password-stdin <<<${{ secrets.DOCKER_ACCESS_TOKEN }}
+          docker push summerwind/actions-runner:v${RUNNER_VERSION}
+          docker tag summerwind/actions-runner:v${RUNNER_VERSION} summerwind/actions-runner:latest
+          docker push summerwind/actions-runner:latest
+        if: ${{ github.event_name == 'push' }}

.github/workflows/release.yml

@@ -7,27 +7,43 @@ jobs:
     runs-on: ubuntu-latest
     name: Release
     steps:
-    - name: Checkout
+      - name: Checkout
-      uses: actions/checkout@v2
+        uses: actions/checkout@v2
-    - name: Install tools
+
-      run: |
+      - name: Install tools
-        curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
+        run: |
-        tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz
+          curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
-        sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
+          tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz
-        curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
+          sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
-        sudo mv kustomize /usr/local/bin
+          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
-        curl -L -O https://github.com/tcnksm/ghr/releases/download/v0.13.0/ghr_v0.13.0_linux_amd64.tar.gz
+          sudo mv kustomize /usr/local/bin
-        tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
+          curl -L -O https://github.com/tcnksm/ghr/releases/download/v0.13.0/ghr_v0.13.0_linux_amd64.tar.gz
-        sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
+          tar zxvf ghr_v0.13.0_linux_amd64.tar.gz
-    - name: Set version
+          sudo mv ghr_v0.13.0_linux_amd64/ghr /usr/local/bin
-      run: echo "::set-env name=VERSION::$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')"
+
-    - name: Upload artifacts
+      - name: Set version
-      env:
+        run: echo "::set-env name=VERSION::$(cat ${GITHUB_EVENT_PATH} | jq -r '.release.tag_name')"
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
-      run: make github-release
+      - name: Upload artifacts
-    - name: Build container image
+        env:
-      run: make docker-build
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    - name: Docker Login
+        run: make github-release
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
-    - name: Push container image
+      - name: Set up Docker Buildx
-      run: make docker-push
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v1
+        with:
+          buildx-version: latest
+
+      - name: Login to GitHub Docker Registry
+        run: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin
+        env:
+          DOCKERHUB_USERNAME: summerwind
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: Build Container Image
+        run: |
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --tag summerwind/actions-runner-controller:${{ env.VERSION }} \
+            -f Dockerfile . --push

.github/workflows/test.yaml

@@ -1,15 +1,16 @@
+name: CI
+
 on:
-  push:
+  pull_request:
     branches:
-    - master
+      - master
     paths-ignore:
     - 'runner/**'
-    - '.github/**'
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
-    name: Build
+    name: Test
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -20,9 +21,7 @@ jobs:
         sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
     - name: Run tests
       run: make test
-    - name: Build container image
+    - name: Verify manifests are up-to-date
-      run: make docker-build
+      run: |
-    - name: Docker Login
+        make manifests
-      run: docker login -u summerwind -p ${{ secrets.DOCKER_ACCESS_TOKEN }}
+        git diff --exit-code
-    - name: Push container image
-      run: make docker-push

Dockerfile

@@ -1,28 +1,37 @@
 # Build the manager binary
 FROM golang:1.13 as builder
 
+ARG TARGETPLATFORM
+
 WORKDIR /workspace
+
+ENV GO111MODULE=on \
+  CGO_ENABLED=0
+
 # Copy the Go Modules manifests
-COPY go.mod go.mod
+COPY go.mod go.sum ./
-COPY go.sum go.sum
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
+COPY . .
-COPY api/ api/
-COPY controllers/ controllers/
-COPY github/ github/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
+RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
+  export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
+  GOARM=$(echo ${TARGETPLATFORM} | cut -d / -f3 | cut -c2-) && \
+  go build -a -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
+
 WORKDIR /
+
 COPY --from=builder /workspace/manager .
+
 USER nonroot:nonroot
 
 ENTRYPOINT ["/manager"]

Makefile

@@ -11,6 +11,23 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
+
 all: manager
 
 # Run tests
@@ -62,6 +79,18 @@ docker-build: test
 docker-push:
 	docker push ${NAME}:${VERSION}
 
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:${VERSION}" \
+		-f Dockerfile \
+		. ${PUSH_ARG}
+
 # Generate the release manifest file
 release: manifests
 	cd config/manager && kustomize edit set image controller=${NAME}:${VERSION}
@@ -81,7 +110,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen

README.md

@@ -22,7 +22,7 @@ $ kubectl apply -f https://github.com/summerwind/actions-runner-controller/relea
 
 ## Setting up authentication with GitHub API
 
-There are two ways for actions-runner-controller to authenticate with the the GitHub API:
+There are two ways for actions-runner-controller to authenticate with the GitHub API:
 
 1. Using GitHub App.
 2. Using Personal Access Token.
@@ -135,7 +135,7 @@ The runner you created has been registered to your repository.
 
 <img width="756" alt="Actions tab in your repository settings" src="https://user-images.githubusercontent.com/230145/73618667-8cbf9700-466c-11ea-80b6-c67e6d3f70e7.png">
 
-Now your can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
+Now you can use your self-hosted runner. See the [official documentation](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-self-hosted-runners-in-a-workflow) on how to run a job with it.
 
 ### Organization Runners
 
@@ -191,7 +191,7 @@ example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 
 #### Autoscaling
 
-`RunnerDeployment` can scale number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
+`RunnerDeployment` can scale the number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
 
 In the below example, `actions-runner` checks for pending workflow runs for each sync period, and scale to e.g. 3 if there're 3 pending jobs at sync time.
 
@@ -328,9 +328,9 @@ jobs:
     runs-on: custom-runner
 ```
 
-Note that if you specify `self-hosted` in your worlflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
+Note that if you specify `self-hosted` in your workflow, then this will run your job on _any_ self-hosted runner, regardless of the labels that they have.
 
-## Softeware installed in the runner image
+## Software installed in the runner image
 
 The GitHub hosted runners include a large amount of pre-installed software packages. For Ubuntu 18.04, this list can be found at https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md
 

config/certmanager/certificate.yaml

@@ -1,7 +1,7 @@
 # The following manifests contain a self-signed issuer CR and a certificate CR.
 # More document can be found at https://docs.cert-manager.io
 # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
   name: selfsigned-issuer
@@ -9,7 +9,7 @@ metadata:
 spec:
   selfSigned: {}
 ---
-apiVersion: cert-manager.io/v1alpha2
+apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: horizontalrunnerautoscalers.actions.summerwind.dev
 spec:

config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: runnerdeployments.actions.summerwind.dev
 spec:
@@ -788,10 +788,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -1256,6 +1260,10 @@ spec:
                               - containerPort
                               type: object
                             type: array
+                            x-kubernetes-list-map-keys:
+                            - containerPort
+                            - protocol
+                            x-kubernetes-list-type: map
                           readinessProbe:
                             description: 'Periodic probe of container service readiness.
                               Container will be removed from service endpoints if
@@ -1381,13 +1389,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -1847,9 +1863,13 @@ spec:
                                       optional for env vars'
                                     type: string
                                   divisor:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
                                     description: Specifies the output format of the
                                       exposed resources, defaults to "1"
-                                    type: string
+                                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                    x-kubernetes-int-or-string: true
                                   resource:
                                     description: 'Required: resource to select'
                                     type: string
@@ -2033,10 +2053,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -2611,13 +2635,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -3127,10 +3159,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -3595,6 +3631,10 @@ spec:
                               - containerPort
                               type: object
                             type: array
+                            x-kubernetes-list-map-keys:
+                            - containerPort
+                            - protocol
+                            x-kubernetes-list-type: map
                           readinessProbe:
                             description: 'Periodic probe of container service readiness.
                               Container will be removed from service endpoints if
@@ -3720,13 +3760,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -4135,13 +4183,21 @@ spec:
                       properties:
                         limits:
                           additionalProperties:
-                            type: string
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
                           description: 'Limits describes the maximum amount of compute
                             resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                           type: object
                         requests:
                           additionalProperties:
-                            type: string
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
                           description: 'Requests describes the minimum amount of compute
                             resources required. If Requests is omitted for a container,
                             it defaults to Limits if that is explicitly specified,
@@ -4385,10 +4441,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -4853,6 +4913,10 @@ spec:
                               - containerPort
                               type: object
                             type: array
+                            x-kubernetes-list-map-keys:
+                            - containerPort
+                            - protocol
+                            x-kubernetes-list-type: map
                           readinessProbe:
                             description: 'Periodic probe of container service readiness.
                               Container will be removed from service endpoints if
@@ -4978,13 +5042,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -5800,10 +5872,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -5826,6 +5902,9 @@ spec:
                                   string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'
                                 type: string
                               sizeLimit:
+                                anyOf:
+                                - type: integer
+                                - type: string
                                 description: 'Total amount of local storage required
                                   for this EmptyDir volume. The size limit is also
                                   applicable for memory medium. The maximum usage
@@ -5834,7 +5913,8 @@ spec:
                                   of memory limits of all containers in a pod. The
                                   default is nil which means that the limit is undefined.
                                   More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'
-                                type: string
+                                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                x-kubernetes-int-or-string: true
                             type: object
                           fc:
                             description: FC represents a Fibre Channel resource that
@@ -6321,10 +6401,14 @@ spec:
                                                       for env vars'
                                                     type: string
                                                   divisor:
+                                                    anyOf:
+                                                    - type: integer
+                                                    - type: string
                                                     description: Specifies the output
                                                       format of the exposed resources,
                                                       defaults to "1"
-                                                    type: string
+                                                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                                    x-kubernetes-int-or-string: true
                                                   resource:
                                                     description: 'Required: resource
                                                       to select'

config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: runnerreplicasets.actions.summerwind.dev
 spec:
@@ -788,10 +788,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -1256,6 +1260,10 @@ spec:
                               - containerPort
                               type: object
                             type: array
+                            x-kubernetes-list-map-keys:
+                            - containerPort
+                            - protocol
+                            x-kubernetes-list-type: map
                           readinessProbe:
                             description: 'Periodic probe of container service readiness.
                               Container will be removed from service endpoints if
@@ -1381,13 +1389,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -1847,9 +1863,13 @@ spec:
                                       optional for env vars'
                                     type: string
                                   divisor:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
                                     description: Specifies the output format of the
                                       exposed resources, defaults to "1"
-                                    type: string
+                                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                    x-kubernetes-int-or-string: true
                                   resource:
                                     description: 'Required: resource to select'
                                     type: string
@@ -2033,10 +2053,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -2611,13 +2635,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -3127,10 +3159,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -3595,6 +3631,10 @@ spec:
                               - containerPort
                               type: object
                             type: array
+                            x-kubernetes-list-map-keys:
+                            - containerPort
+                            - protocol
+                            x-kubernetes-list-type: map
                           readinessProbe:
                             description: 'Periodic probe of container service readiness.
                               Container will be removed from service endpoints if
@@ -3720,13 +3760,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -4135,13 +4183,21 @@ spec:
                       properties:
                         limits:
                           additionalProperties:
-                            type: string
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
                           description: 'Limits describes the maximum amount of compute
                             resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                           type: object
                         requests:
                           additionalProperties:
-                            type: string
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
                           description: 'Requests describes the minimum amount of compute
                             resources required. If Requests is omitted for a container,
                             it defaults to Limits if that is explicitly specified,
@@ -4385,10 +4441,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -4853,6 +4913,10 @@ spec:
                               - containerPort
                               type: object
                             type: array
+                            x-kubernetes-list-map-keys:
+                            - containerPort
+                            - protocol
+                            x-kubernetes-list-type: map
                           readinessProbe:
                             description: 'Periodic probe of container service readiness.
                               Container will be removed from service endpoints if
@@ -4978,13 +5042,21 @@ spec:
                             properties:
                               limits:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Limits describes the maximum amount
                                   of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                                 type: object
                               requests:
                                 additionalProperties:
-                                  type: string
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 description: 'Requests describes the minimum amount
                                   of compute resources required. If Requests is omitted
                                   for a container, it defaults to Limits if that is
@@ -5800,10 +5872,14 @@ spec:
                                             volumes, optional for env vars'
                                           type: string
                                         divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
                                           description: Specifies the output format
                                             of the exposed resources, defaults to
                                             "1"
-                                          type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
                                         resource:
                                           description: 'Required: resource to select'
                                           type: string
@@ -5826,6 +5902,9 @@ spec:
                                   string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'
                                 type: string
                               sizeLimit:
+                                anyOf:
+                                - type: integer
+                                - type: string
                                 description: 'Total amount of local storage required
                                   for this EmptyDir volume. The size limit is also
                                   applicable for memory medium. The maximum usage
@@ -5834,7 +5913,8 @@ spec:
                                   of memory limits of all containers in a pod. The
                                   default is nil which means that the limit is undefined.
                                   More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'
-                                type: string
+                                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                x-kubernetes-int-or-string: true
                             type: object
                           fc:
                             description: FC represents a Fibre Channel resource that
@@ -6321,10 +6401,14 @@ spec:
                                                       for env vars'
                                                     type: string
                                                   divisor:
+                                                    anyOf:
+                                                    - type: integer
+                                                    - type: string
                                                     description: Specifies the output
                                                       format of the exposed resources,
                                                       defaults to "1"
-                                                    type: string
+                                                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                                    x-kubernetes-int-or-string: true
                                                   resource:
                                                     description: 'Required: resource
                                                       to select'

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: runners.actions.summerwind.dev
 spec:
@@ -723,9 +723,13 @@ spec:
                                     optional for env vars'
                                   type: string
                                 divisor:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
                                   description: Specifies the output format of the
                                     exposed resources, defaults to "1"
-                                  type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 resource:
                                   description: 'Required: resource to select'
                                   type: string
@@ -1160,6 +1164,10 @@ spec:
                       - containerPort
                       type: object
                     type: array
+                    x-kubernetes-list-map-keys:
+                    - containerPort
+                    - protocol
+                    x-kubernetes-list-type: map
                   readinessProbe:
                     description: 'Periodic probe of container service readiness. Container
                       will be removed from service endpoints if the probe fails. Cannot
@@ -1282,13 +1290,21 @@ spec:
                     properties:
                       limits:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Limits describes the maximum amount of compute
                           resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                         type: object
                       requests:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Requests describes the minimum amount of compute
                           resources required. If Requests is omitted for a container,
                           it defaults to Limits if that is explicitly specified, otherwise
@@ -1722,9 +1738,13 @@ spec:
                               for env vars'
                             type: string
                           divisor:
+                            anyOf:
+                            - type: integer
+                            - type: string
                             description: Specifies the output format of the exposed
                               resources, defaults to "1"
-                            type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
                           resource:
                             description: 'Required: resource to select'
                             type: string
@@ -1894,9 +1914,13 @@ spec:
                                     optional for env vars'
                                   type: string
                                 divisor:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
                                   description: Specifies the output format of the
                                     exposed resources, defaults to "1"
-                                  type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 resource:
                                   description: 'Required: resource to select'
                                   type: string
@@ -2440,13 +2464,21 @@ spec:
                     properties:
                       limits:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Limits describes the maximum amount of compute
                           resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                         type: object
                       requests:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Requests describes the minimum amount of compute
                           resources required. If Requests is omitted for a container,
                           it defaults to Limits if that is explicitly specified, otherwise
@@ -2930,9 +2962,13 @@ spec:
                                     optional for env vars'
                                   type: string
                                 divisor:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
                                   description: Specifies the output format of the
                                     exposed resources, defaults to "1"
-                                  type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 resource:
                                   description: 'Required: resource to select'
                                   type: string
@@ -3367,6 +3403,10 @@ spec:
                       - containerPort
                       type: object
                     type: array
+                    x-kubernetes-list-map-keys:
+                    - containerPort
+                    - protocol
+                    x-kubernetes-list-type: map
                   readinessProbe:
                     description: 'Periodic probe of container service readiness. Container
                       will be removed from service endpoints if the probe fails. Cannot
@@ -3489,13 +3529,21 @@ spec:
                     properties:
                       limits:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Limits describes the maximum amount of compute
                           resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                         type: object
                       requests:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Requests describes the minimum amount of compute
                           resources required. If Requests is omitted for a container,
                           it defaults to Limits if that is explicitly specified, otherwise
@@ -3883,13 +3931,21 @@ spec:
               properties:
                 limits:
                   additionalProperties:
-                    type: string
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
                   description: 'Limits describes the maximum amount of compute resources
                     allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                   type: object
                 requests:
                   additionalProperties:
-                    type: string
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
                   description: 'Requests describes the minimum amount of compute resources
                     required. If Requests is omitted for a container, it defaults
                     to Limits if that is explicitly specified, otherwise to an implementation-defined
@@ -4118,9 +4174,13 @@ spec:
                                     optional for env vars'
                                   type: string
                                 divisor:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
                                   description: Specifies the output format of the
                                     exposed resources, defaults to "1"
-                                  type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 resource:
                                   description: 'Required: resource to select'
                                   type: string
@@ -4555,6 +4615,10 @@ spec:
                       - containerPort
                       type: object
                     type: array
+                    x-kubernetes-list-map-keys:
+                    - containerPort
+                    - protocol
+                    x-kubernetes-list-type: map
                   readinessProbe:
                     description: 'Periodic probe of container service readiness. Container
                       will be removed from service endpoints if the probe fails. Cannot
@@ -4677,13 +4741,21 @@ spec:
                     properties:
                       limits:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Limits describes the maximum amount of compute
                           resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                         type: object
                       requests:
                         additionalProperties:
-                          type: string
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
                         description: 'Requests describes the minimum amount of compute
                           resources required. If Requests is omitted for a container,
                           it defaults to Limits if that is explicitly specified, otherwise
@@ -5451,9 +5523,13 @@ spec:
                                     optional for env vars'
                                   type: string
                                 divisor:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
                                   description: Specifies the output format of the
                                     exposed resources, defaults to "1"
-                                  type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
                                 resource:
                                   description: 'Required: resource to select'
                                   type: string
@@ -5476,6 +5552,9 @@ spec:
                           More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'
                         type: string
                       sizeLimit:
+                        anyOf:
+                        - type: integer
+                        - type: string
                         description: 'Total amount of local storage required for this
                           EmptyDir volume. The size limit is also applicable for memory
                           medium. The maximum usage on memory medium EmptyDir would
@@ -5483,7 +5562,8 @@ spec:
                           and the sum of memory limits of all containers in a pod.
                           The default is nil which means that the limit is undefined.
                           More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'
-                        type: string
+                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                        x-kubernetes-int-or-string: true
                     type: object
                   fc:
                     description: FC represents a Fibre Channel resource that is attached
@@ -5940,10 +6020,14 @@ spec:
                                               for volumes, optional for env vars'
                                             type: string
                                           divisor:
+                                            anyOf:
+                                            - type: integer
+                                            - type: string
                                             description: Specifies the output format
                                               of the exposed resources, defaults to
                                               "1"
-                                            type: string
+                                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                            x-kubernetes-int-or-string: true
                                           resource:
                                             description: 'Required: resource to select'
                                             type: string

config/default/kustomization.yaml

@@ -50,7 +50,7 @@ vars:
   objref:
     kind: Certificate
     group: cert-manager.io
-    version: v1alpha2
+    version: v1
     name: serving-cert # this name should match the one in certificate.yaml
   fieldref:
     fieldpath: metadata.namespace
@@ -58,7 +58,7 @@ vars:
   objref:
     kind: Certificate
     group: cert-manager.io
-    version: v1alpha2
+    version: v1
     name: serving-cert # this name should match the one in certificate.yaml
 - name: SERVICE_NAMESPACE # namespace of the service
   objref:

config/rbac/role.yaml

@@ -18,6 +18,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -38,6 +50,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerdeployments/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -58,6 +82,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runnerreplicasets/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -78,6 +114,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - runners/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - actions.summerwind.dev
   resources:
@@ -105,3 +153,15 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

controllers/autoscaling.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"strings"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 )
 
 func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
@@ -44,6 +45,38 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp
 	}
 
 	var total, inProgress, queued, completed, unknown int
+	type callback func()
+	listWorkflowJobs := func(user string, repoName string, runID int64, fallback_cb callback) {
+		if runID == 0 {
+			fallback_cb()
+			return
+		}
+		jobs, _, err := r.GitHubClient.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, nil)
+		if err != nil {
+			r.Log.Error(err, "Error listing workflow jobs")
+			fallback_cb()
+		} else if len(jobs.Jobs) == 0 {
+			fallback_cb()
+		} else {
+			for _, job := range jobs.Jobs {
+				switch job.GetStatus() {
+				case "completed":
+					// We add a case for `completed` so it is not counted in `unknown`.
+					// And we do not increment the counter for completed because
+					// that counter only refers to workflows. The reason for
+					// this is because we do not get a list of jobs for
+					// completed workflows in order to keep the number of API
+					// calls to a minimum.
+				case "in_progress":
+					inProgress++
+				case "queued":
+					queued++
+				default:
+					unknown++
+				}
+			}
+		}
+	}
 
 	for _, repo := range repos {
 		user, repoName := repo[0], repo[1]
@@ -52,20 +85,20 @@ func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alp
 			return nil, err
 		}
 
-		for _, r := range list.WorkflowRuns {
+		for _, run := range list.WorkflowRuns {
 			total++
 
 			// In May 2020, there are only 3 statuses.
 			// Follow the below links for more details:
 			// - https://developer.github.com/v3/actions/workflow-runs/#list-repository-workflow-runs
 			// - https://developer.github.com/v3/checks/runs/#create-a-check-run
-			switch r.GetStatus() {
+			switch run.GetStatus() {
 			case "completed":
 				completed++
 			case "in_progress":
-				inProgress++
+				listWorkflowJobs(user, repoName, run.GetID(), func() { inProgress++ })
 			case "queued":
-				queued++
+				listWorkflowJobs(user, repoName, run.GetID(), func() { queued++ })
 			default:
 				unknown++
 			}

controllers/autoscaling_test.go

@@ -2,16 +2,17 @@ package controllers
 
 import (
 	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"github.com/summerwind/actions-runner-controller/github"
 	"github.com/summerwind/actions-runner-controller/github/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
-	"net/http/httptest"
-	"net/url"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	"testing"
 )
 
 func newGithubClient(server *httptest.Server) *github.Client {
@@ -44,9 +45,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		sReplicas    *int
 		sTime        *metav1.Time
 		workflowRuns string
+		workflowJobs map[int]string
 		want         int
 		err          string
 	}{
+		// Legacy functionality
 		// 3 demanded, max at 3
 		{
 			repo:         "test/valid",
@@ -122,6 +125,21 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
 			want:         3,
 		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
 	}
 
 	for i := range testcases {
@@ -136,7 +154,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
 			defer server.Close()
 			client := newGithubClient(server)
 
@@ -211,6 +229,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		sReplicas    *int
 		sTime        *metav1.Time
 		workflowRuns string
+		workflowJobs map[int]string
 		want         int
 		err          string
 	}{
@@ -316,6 +335,22 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
 			err:          "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
 		},
+
+		// Job-level autoscaling
+		// 5 requested from 3 workflows
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(10),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 5,
+		},
 	}
 
 	for i := range testcases {
@@ -330,7 +365,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		_ = v1alpha1.AddToScheme(scheme)
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns), fake.WithListWorkflowJobsResponse(200, tc.workflowJobs))
 			defer server.Close()
 			client := newGithubClient(server)
 

controllers/horizontalrunnerautoscaler_controller.go

@@ -50,6 +50,7 @@ type HorizontalRunnerAutoscalerReconciler struct {
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 

controllers/integration_test.go

@@ -2,9 +2,10 @@ package controllers
 
 import (
 	"context"
-	"github.com/summerwind/actions-runner-controller/github/fake"
 	"time"
 
+	"github.com/summerwind/actions-runner-controller/github/fake"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -44,7 +45,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		Status: 200,
 		Body:   workflowRunsFor3Replicas,
 	}
-	server := fake.NewServer(fake.WithFixedResponses(responses))
+	fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses))
 
 	BeforeEach(func() {
 		stopCh = make(chan struct{})
@@ -58,11 +59,16 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+		ghClient := newGithubClient(server)
+
 		replicasetController := &RunnerReplicaSetReconciler{
-			Client:   mgr.GetClient(),
+			Client:       mgr.GetClient(),
-			Scheme:   scheme.Scheme,
+			Scheme:       scheme.Scheme,
-			Log:      logf.Log,
+			Log:          logf.Log,
-			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
 		}
 		err = replicasetController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -76,7 +82,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		err = deploymentsController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
 
-		client := newGithubClient(server)
+		client := newGithubClient(fakeGithubServer)
 
 		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
 			Client:       mgr.GetClient(),
@@ -99,7 +105,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 	AfterEach(func() {
 		close(stopCh)
 
-		server.Close()
+		fakeGithubServer.Close()
 
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")

controllers/runner_controller.go

@@ -53,8 +53,10 @@ type RunnerReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
 
 func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
@@ -203,12 +205,16 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, err
 		}
 
-		if pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image {
+		runnerBusy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
-			restart = true
+		if err != nil {
-		}
+			log.Error(err, "Failed to check if runner is busy")
-		if !reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) {
+			return ctrl.Result{}, nil
+		}
+
+		if !runnerBusy && (!reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) || pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image) {
 			restart = true
 		}
+
 		if !restart {
 			return ctrl.Result{}, err
 		}
@@ -225,6 +231,21 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	return ctrl.Result{}, nil
 }
 
+func (r *RunnerReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, fmt.Errorf("runner not found")
+}
+
 func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name string) (bool, error) {
 	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
 	if err != nil {
@@ -234,6 +255,9 @@ func (r *RunnerReconciler) unregisterRunner(ctx context.Context, org, repo, name
 	id := int64(0)
 	for _, runner := range runners {
 		if runner.GetName() == name {
+			if runner.GetBusy() {
+				return false, fmt.Errorf("runner is busy")
+			}
 			id = runner.GetID()
 			break
 		}

controllers/runnerdeployment_controller.go

@@ -54,6 +54,7 @@ type RunnerDeploymentReconciler struct {
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch

controllers/runnerreplicaset_controller.go

@@ -31,17 +31,20 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github"
 )
 
 // RunnerReplicaSetReconciler reconciles a Runner object
 type RunnerReplicaSetReconciler struct {
 	client.Client
-	Log      logr.Logger
+	Log          logr.Logger
-	Recorder record.EventRecorder
+	Recorder     record.EventRecorder
-	Scheme   *runtime.Scheme
+	Scheme       *runtime.Scheme
+	GitHubClient *github.Client
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/finalizers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
@@ -96,8 +99,25 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	if available > desired {
 		n := available - desired
 
+		// get runners that are currently not busy
+		var notBusy []v1alpha1.Runner
+		for _, runner := range myRunners {
+			busy, err := r.isRunnerBusy(ctx, runner.Spec.Organization, runner.Spec.Repository, runner.Name)
+			if err != nil {
+				log.Error(err, "Failed to check if runner is busy")
+				return ctrl.Result{}, err
+			}
+			if !busy {
+				notBusy = append(notBusy, runner)
+			}
+		}
+
+		if len(notBusy) < n {
+			n = len(notBusy)
+		}
+
 		for i := 0; i < n; i++ {
-			if err := r.Client.Delete(ctx, &myRunners[i]); err != nil {
+			if err := r.Client.Delete(ctx, &notBusy[i]); err != nil {
 				log.Error(err, "Failed to delete runner resource")
 
 				return ctrl.Result{}, err
@@ -166,3 +186,19 @@ func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&v1alpha1.Runner{}).
 		Complete(r)
 }
+
+func (r *RunnerReplicaSetReconciler) isRunnerBusy(ctx context.Context, org, repo, name string) (bool, error) {
+	runners, err := r.GitHubClient.ListRunners(ctx, org, repo)
+	r.Log.Info("runners", "github", runners)
+	if err != nil {
+		return false, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner.GetBusy(), nil
+		}
+	}
+
+	return false, fmt.Errorf("runner not found")
+}

controllers/runnerreplicaset_controller_test.go

@@ -3,11 +3,14 @@ package controllers
 import (
 	"context"
 	"math/rand"
+	"net/http/httptest"
 	"time"
 
+	"github.com/google/go-github/v32/github"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -17,6 +20,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+)
+
+var (
+	runnersList *fake.RunnersList
+	server      *httptest.Server
 )
 
 // SetupTest will set up a testing environment.
@@ -41,11 +50,16 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+		ghClient := newGithubClient(server)
+
 		controller := &RunnerReplicaSetReconciler{
-			Client:   mgr.GetClient(),
+			Client:       mgr.GetClient(),
-			Scheme:   scheme.Scheme,
+			Scheme:       scheme.Scheme,
-			Log:      logf.Log,
+			Log:          logf.Log,
-			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Recorder:     mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient: ghClient,
 		}
 		err = controller.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
@@ -61,6 +75,7 @@ func SetupTest(ctx context.Context) *corev1.Namespace {
 	AfterEach(func() {
 		close(stopCh)
 
+		server.Close()
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
 	})
@@ -124,6 +139,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
@@ -155,6 +180,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
@@ -186,6 +221,16 @@ var _ = Context("Inside of a new namespace", func() {
 							logf.Log.Error(err, "list runners")
 						}
 
+						for i, runner := range runners.Items {
+							runnersList.Add(&github.Runner{
+								ID:     pointer.Int64Ptr(int64(i) + 1),
+								Name:   pointer.StringPtr(runner.Name),
+								OS:     pointer.StringPtr("linux"),
+								Status: pointer.StringPtr("online"),
+								Busy:   pointer.BoolPtr(false),
+							})
+						}
+
 						return len(runners.Items)
 					},
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(0))

github/fake/fake.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"time"
+	"unicode"
 )
 
 const (
@@ -14,8 +17,8 @@ const (
 {
   "total_count": 2,
   "runners": [
-    {"id": 1, "name": "test1", "os": "linux", "status": "online"},
+    {"id": 1, "name": "test1", "os": "linux", "status": "online", "busy": false},
-    {"id": 2, "name": "test2", "os": "linux", "status": "offline"}
+    {"id": 2, "name": "test2", "os": "linux", "status": "offline", "busy": false}
   ]
 }
 `
@@ -31,6 +34,24 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	fmt.Fprintf(w, h.Body)
 }
 
+type MapHandler struct {
+	Status int
+	Bodies map[int]string
+}
+
+func (h *MapHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// Parse out int key from URL path
+	key, err := strconv.Atoi(strings.TrimFunc(req.URL.Path, func(r rune) bool { return !unicode.IsNumber(r) }))
+	if err != nil {
+		w.WriteHeader(400)
+	} else if body := h.Bodies[key]; len(body) == 0 {
+		w.WriteHeader(404)
+	} else {
+		w.WriteHeader(h.Status)
+		fmt.Fprintf(w, body)
+	}
+}
+
 type ServerConfig struct {
 	*FixedResponses
 }
@@ -45,7 +66,7 @@ func NewServer(opts ...Option) *httptest.Server {
 		o(&config)
 	}
 
-	routes := map[string]*Handler{
+	routes := map[string]http.Handler{
 		// For CreateRegistrationToken
 		"/repos/test/valid/actions/runners/registration-token": &Handler{
 			Status: http.StatusCreated,
@@ -126,6 +147,9 @@ func NewServer(opts ...Option) *httptest.Server {
 
 		// For auto-scaling based on the number of queued(pending) workflow runs
 		"/repos/test/valid/actions/runs": config.FixedResponses.ListRepositoryWorkflowRuns,
+
+		// For auto-scaling based on the number of queued(pending) workflow jobs
+		"/repos/test/valid/actions/runs/": config.FixedResponses.ListWorkflowJobs,
 	}
 
 	mux := http.NewServeMux()

github/fake/options.go

@@ -2,6 +2,7 @@ package fake
 
 type FixedResponses struct {
 	ListRepositoryWorkflowRuns *Handler
+	ListWorkflowJobs           *MapHandler
 }
 
 type Option func(*ServerConfig)
@@ -15,6 +16,15 @@ func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
 	}
 }
 
+func WithListWorkflowJobsResponse(status int, bodies map[int]string) Option {
+	return func(c *ServerConfig) {
+		c.FixedResponses.ListWorkflowJobs = &MapHandler{
+			Status: status,
+			Bodies: bodies,
+		}
+	}
+}
+
 func WithFixedResponses(responses *FixedResponses) Option {
 	return func(c *ServerConfig) {
 		c.FixedResponses = responses

github/fake/runners.go

@@ -0,0 +1,74 @@
+package fake
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+
+	"github.com/google/go-github/v32/github"
+	"github.com/gorilla/mux"
+)
+
+type RunnersList struct {
+	runners []*github.Runner
+}
+
+func NewRunnersList() *RunnersList {
+	return &RunnersList{
+		runners: make([]*github.Runner, 0),
+	}
+}
+
+func (r *RunnersList) Add(runner *github.Runner) {
+	if !exists(r.runners, runner) {
+		r.runners = append(r.runners, runner)
+	}
+}
+
+func (r *RunnersList) GetServer() *httptest.Server {
+	router := mux.NewRouter()
+
+	router.Handle("/repos/{owner}/{repo}/actions/runners", r.handleList())
+	router.Handle("/repos/{owner}/{repo}/actions/runners/{id}", r.handleRemove())
+	router.Handle("/orgs/{org}/actions/runners", r.handleList())
+	router.Handle("/orgs/{org}/actions/runners/{id}", r.handleRemove())
+
+	return httptest.NewServer(router)
+}
+
+func (r *RunnersList) handleList() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		j, err := json.Marshal(github.Runners{
+			TotalCount: len(r.runners),
+			Runners:    r.runners,
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		w.WriteHeader(http.StatusOK)
+		w.Write(j)
+	}
+}
+
+func (r *RunnersList) handleRemove() http.HandlerFunc {
+	return func(w http.ResponseWriter, res *http.Request) {
+		vars := mux.Vars(res)
+		for i, runner := range r.runners {
+			if runner.ID != nil && vars["id"] == strconv.FormatInt(*runner.ID, 10) {
+				r.runners = append(r.runners[:i], r.runners[i+1:]...)
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func exists(runners []*github.Runner, runner *github.Runner) bool {
+	for _, r := range runners {
+		if *r.Name == *runner.Name {
+			return true
+		}
+	}
+	return false
+}

github/github.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/bradleyfalzon/ghinstallation"
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"golang.org/x/oauth2"
 )
 
@@ -85,7 +85,7 @@ func (c *Client) GetRegistrationToken(ctx context.Context, org, repo, name strin
 	return rt, nil
 }
 
-// RemoveRunner removes a runner with specified runner ID from repocitory.
+// RemoveRunner removes a runner with specified runner ID from repository.
 func (c *Client) RemoveRunner(ctx context.Context, org, repo string, runnerID int64) error {
 	owner, repo, err := getOwnerAndRepo(org, repo)
 
@@ -121,7 +121,7 @@ func (c *Client) ListRunners(ctx context.Context, org, repo string) ([]*github.R
 		list, res, err := c.listRunners(ctx, owner, repo, &opts)
 
 		if err != nil {
-			return runners, fmt.Errorf("failed to remove runner: %v", err)
+			return runners, fmt.Errorf("failed to list runners: %v", err)
 		}
 
 		runners = append(runners, list.Runners...)

github/github_beta.go

@@ -10,7 +10,7 @@ import (
 	"net/url"
 	"reflect"
 
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"github.com/google/go-querystring/query"
 )
 

github/github_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/go-github/v31/github"
+	"github.com/google/go-github/v32/github"
 	"github.com/summerwind/actions-runner-controller/github/fake"
 )
 
@@ -41,9 +41,9 @@ func TestGetRegistrationToken(t *testing.T) {
 		token string
 		err   bool
 	}{
-		{org: "test", repo: "valid", token: fake.RegistrationToken, err: false},
+		{org: "", repo: "test/valid", token: fake.RegistrationToken, err: false},
-		{org: "test", repo: "invalid", token: "", err: true},
+		{org: "", repo: "test/invalid", token: "", err: true},
-		{org: "test", repo: "error", token: "", err: true},
+		{org: "", repo: "test/error", token: "", err: true},
 		{org: "test", repo: "", token: fake.RegistrationToken, err: false},
 		{org: "invalid", repo: "", token: "", err: true},
 		{org: "error", repo: "", token: "", err: true},
@@ -68,9 +68,9 @@ func TestListRunners(t *testing.T) {
 		length int
 		err    bool
 	}{
-		{org: "test", repo: "valid", length: 2, err: false},
+		{org: "", repo: "test/valid", length: 2, err: false},
-		{org: "test", repo: "invalid", length: 0, err: true},
+		{org: "", repo: "test/invalid", length: 0, err: true},
-		{org: "test", repo: "error", length: 0, err: true},
+		{org: "", repo: "test/error", length: 0, err: true},
 		{org: "test", repo: "", length: 2, err: false},
 		{org: "invalid", repo: "", length: 0, err: true},
 		{org: "error", repo: "", length: 0, err: true},
@@ -94,9 +94,9 @@ func TestRemoveRunner(t *testing.T) {
 		repo string
 		err  bool
 	}{
-		{org: "test", repo: "valid", err: false},
+		{org: "", repo: "test/valid", err: false},
-		{org: "test", repo: "invalid", err: true},
+		{org: "", repo: "test/invalid", err: true},
-		{org: "test", repo: "error", err: true},
+		{org: "", repo: "test/error", err: true},
 		{org: "test", repo: "", err: false},
 		{org: "invalid", repo: "", err: true},
 		{org: "error", repo: "", err: true},

go.mod

@@ -6,8 +6,9 @@ require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v0.1.0
-	github.com/google/go-github/v31 v31.0.0
+	github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04
 	github.com/google/go-querystring v1.0.0
+	github.com/gorilla/mux v1.8.0
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.5.0
 	github.com/stretchr/testify v1.4.0 // indirect
@@ -15,6 +16,6 @@ require (
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
 	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
 	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
-	k8s.io/klog v0.4.0
+	k8s.io/utils v0.0.0-20190801114015-581e00157fb1
 	sigs.k8s.io/controller-runtime v0.4.0
 )

go.sum

@@ -116,11 +116,10 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
-github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo=
+github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04 h1:wEYk2h/GwOhImcVjiTIceP88WxVbXw2F+ARYUQMEsfg=
-github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM=
+github.com/google/go-github/v32 v32.1.1-0.20200822031813-d57a3a84ba04/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@@ -136,6 +135,8 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC
 github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
 github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=

main.go

@@ -158,9 +158,10 @@ func main() {
 	}
 
 	runnerSetReconciler := &controllers.RunnerReplicaSetReconciler{
-		Client: mgr.GetClient(),
+		Client:       mgr.GetClient(),
-		Log:    ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
+		Log:          ctrl.Log.WithName("controllers").WithName("RunnerReplicaSet"),
-		Scheme: mgr.GetScheme(),
+		Scheme:       mgr.GetScheme(),
+		GitHubClient: ghClient,
 	}
 
 	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {

runner/Dockerfile

@@ -1,7 +1,8 @@
 FROM ubuntu:18.04
 
-ARG RUNNER_VERSION
+ARG TARGETPLATFORM
-ARG DOCKER_VERSION
+ARG RUNNER_VERSION=2.272.0
+ARG DOCKER_VERSION=19.03.12
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update -y \
@@ -9,52 +10,62 @@ RUN apt update -y \
   && add-apt-repository -y ppa:git-core/ppa \
   && apt update -y \
   && apt install -y --no-install-recommends \
-     build-essential \
+  build-essential \
-     curl \
+  curl \
-     ca-certificates \
+  ca-certificates \
-     dnsutils \
+  dnsutils \
-     ftp \
+  ftp \
-     git \
+  git \
-     iproute2 \
+  iproute2 \
-     iputils-ping \
+  iputils-ping \
-     jq \
+  jq \
-     libunwind8 \
+  libunwind8 \
-     locales \
+  locales \
-     netcat \
+  netcat \
-     openssh-client \
+  openssh-client \
-     parallel \
+  parallel \
-     rsync \
+  rsync \
-     shellcheck \
+  shellcheck \
-     sudo \
+  sudo \
-     telnet \
+  telnet \
-     time \
+  time \
-     tzdata \
+  tzdata \
-     unzip \
+  unzip \
-     upx \
+  upx \
-     wget \
+  wget \
-     zip \
+  zip \
-     zstd \
+  zstd \
   && rm -rf /var/lib/apt/lists/*
 
-RUN curl -L -o docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz \
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_${ARCH} \
+  && chmod +x /usr/local/bin/dumb-init
+
+# Docker download supports arm64 as aarch64 & amd64 as x86_64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \
+  && curl -L -o docker.tgz https://download.docker.com/linux/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz \
   && tar zxvf docker.tgz \
   && install -o root -g root -m 755 docker/docker /usr/local/bin/docker \
   && rm -rf docker docker.tgz \
-  && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.2/dumb-init_1.2.2_amd64 \
-  && chmod +x /usr/local/bin/dumb-init \
   && adduser --disabled-password --gecos "" --uid 1000 runner \
   && usermod -aG sudo runner \
   && echo "%sudo   ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers
 
-RUN mkdir -p /runner \
+# Runner download supports amd64 as x64
+RUN export ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \
+  && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \
+  && mkdir -p /runner \
   && cd /runner \
-  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+  && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \
   && tar xzf ./runner.tar.gz \
   && rm runner.tar.gz \
   && ./bin/installdependencies.sh \
   && rm -rf /var/lib/apt/lists/*
 
 COPY entrypoint.sh /runner
+COPY patched /runner/patched
 
 USER runner:runner
 ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]

runner/Makefile

@@ -1,11 +1,41 @@
 NAME ?= summerwind/actions-runner
+TAG ?= latest
 
-RUNNER_VERSION ?= 2.272.0
+RUNNER_VERSION ?= 2.273.5
 DOCKER_VERSION ?= 19.03.12
 
+# default list of platforms for which multiarch image is built
+ifeq (${PLATFORMS}, )
+	export PLATFORMS="linux/amd64,linux/arm64"
+endif
+
+# if IMG_RESULT is unspecified, by default the image will be pushed to registry
+ifeq (${IMG_RESULT}, load)
+	export PUSH_ARG="--load"
+    # if load is specified, image will be built only for the build machine architecture.
+    export PLATFORMS="local"
+else ifeq (${IMG_RESULT}, cache)
+	# if cache is specified, image will only be available in the build cache, it won't be pushed or loaded
+	# therefore no PUSH_ARG will be specified
+else
+	export PUSH_ARG="--push"
+endif
+
 docker-build:
-	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:latest -t ${NAME}:v${RUNNER_VERSION} .
+	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:${TAG} -t ${NAME}:v${RUNNER_VERSION} .
 
 docker-push:
-	docker push ${NAME}:latest
+	docker push ${NAME}:${TAG}
 	docker push ${NAME}:v${RUNNER_VERSION}
+
+docker-buildx:
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+	@if ! docker buildx ls | grep -q container-builder; then\
+		docker buildx create --platform ${PLATFORMS} --name container-builder --use;\
+	fi
+	docker buildx build --platform ${PLATFORMS} \
+		--build-arg RUNNER_VERSION=${RUNNER_VERSION} \
+		--build-arg DOCKER_VERSION=${DOCKER_VERSION} \
+		-t "${NAME}:latest" \
+		-f Dockerfile \
+		. ${PUSH_ARG}

runner/entrypoint.sh

@@ -28,5 +28,11 @@ fi
 cd /runner
 ./config.sh --unattended --replace --name "${RUNNER_NAME}" --url "https://github.com/${ATTACH}" --token "${RUNNER_TOKEN}" ${LABEL_ARG}
 
+for f in runsvc.sh RunnerService.js; do
+  diff {bin,patched}/${f} || :
+  sudo mv bin/${f}{,.bak}
+  sudo mv {patched,bin}/${f}
+done
+
 unset RUNNER_NAME RUNNER_REPO RUNNER_TOKEN
-exec ./run.sh --once
+exec ./bin/runsvc.sh --once

runner/patched/RunnerService.js

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+// Copyright (c) GitHub. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+var childProcess = require("child_process");
+var path = require("path")
+
+var supported = ['linux', 'darwin']
+
+if (supported.indexOf(process.platform) == -1) {
+    console.log('Unsupported platform: ' + process.platform);
+    console.log('Supported platforms are: ' + supported.toString());
+    process.exit(1);
+}
+
+var stopping = false;
+var listener = null;
+
+var runService = function() {
+    var listenerExePath = path.join(__dirname, '../bin/Runner.Listener');
+    var interactive = process.argv[2] === "interactive";
+
+    if(!stopping) {
+        try {
+            if (interactive) {
+                console.log('Starting Runner listener interactively');
+                listener = childProcess.spawn(listenerExePath, ['run'].concat(process.argv.slice(3)), { env: process.env });
+            } else {
+                console.log('Starting Runner listener with startup type: service');
+                listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'].concat(process.argv.slice(2)), { env: process.env });
+            }
+
+            console.log('Started listener process');
+
+            listener.stdout.on('data', (data) => {
+                process.stdout.write(data.toString('utf8'));
+            });
+
+            listener.stderr.on('data', (data) => {
+                process.stdout.write(data.toString('utf8'));
+            });
+
+            listener.on('close', (code) => {
+                console.log(`Runner listener exited with error code ${code}`);
+
+                if (code === 0) {
+                    console.log('Runner listener exit with 0 return code, stop the service, no retry needed.');
+                    stopping = true;
+                } else if (code === 1) {
+                    console.log('Runner listener exit with terminated error, stop the service, no retry needed.');
+                    stopping = true;
+                } else if (code === 2) {
+                    console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.');
+                } else if (code === 3) {
+                    console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.');
+                } else {
+                    console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.');
+                }
+
+                if(!stopping) {
+                    setTimeout(runService, 5000);
+                }
+            });
+
+        } catch(ex) {
+            console.log(ex);
+        }
+    }
+}
+
+runService();
+console.log('Started running service');
+
+var gracefulShutdown = function(code) {
+    console.log('Shutting down runner listener');
+    stopping = true;
+    if (listener) {
+        console.log('Sending SIGINT to runner listener to stop');
+        listener.kill('SIGINT');
+
+        // TODO wait for 30 seconds and send a SIGKILL
+    }
+}
+
+process.on('SIGINT', () => {
+    gracefulShutdown(0);
+});
+
+process.on('SIGTERM', () => {
+    gracefulShutdown(0);
+});

runner/patched/runsvc.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# convert SIGTERM signal to SIGINT
+# for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
+trap 'kill -INT $PID' TERM INT
+
+if [ -f ".path" ]; then
+    # configure
+    export PATH=`cat .path`
+    echo ".path=${PATH}"
+fi
+
+# insert anything to setup env when running as a service
+
+# run the host process which keep the listener alive
+./externals/node12/bin/node ./bin/RunnerService.js $* &
+PID=$!
+wait $PID
+trap - TERM INT
+wait $PID