Merge pull request #76 from summerwind/fix-crash-on-startup

Fix crash on startup after the HRDA addition

README.md

@@ -37,7 +37,7 @@ You can create a GitHub App for either your account or any organization. If you
 
 If you want to create a GitHub App for your organization, replace the `:org` part of the following URL with your organization name before opening it. Then enter any unique name in the "GitHub App name" field, and hit the "Create GitHub App" button at the bottom of the page to create a GitHub App.
 
-- [Create GitHub Apps on your organization](https://github.com/organizations/:org/settings/apps/new?url=http://github.com/summerwind/actions-runner-controller&webhook_active=false&public=false&administration=write)
+- [Create GitHub Apps on your organization](https://github.com/organizations/:org/settings/apps/new?url=http://github.com/summerwind/actions-runner-controller&webhook_active=false&public=false&administration=write&organization_self_hosted_runners=write)
 
 You will see an *App ID* on the page of the GitHub App you created as follows, the value of this App ID will be used later.
 
@@ -189,6 +189,68 @@ example-runnerdeploy2475h595fr   mumoshu/actions-runner-controller-ci   Running
 example-runnerdeploy2475ht2qbr   mumoshu/actions-runner-controller-ci   Running
 ```
 
+#### Autoscaling
+
+`RunnerDeployment` can scale number of runners between `minReplicas` and `maxReplicas` fields, depending on pending workflow runs.
+
+In the below example, `actions-runner` checks for pending workflow runs for each sync period, and scale to e.g. 3 if there're 3 pending jobs at sync time.
+
+```
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runner-deployment
+spec:
+  template:
+    spec:
+      repository: summerwind/actions-runner-controller
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    name: example-runner-deployment
+  minReplicas: 1
+  maxReplicas: 3
+  metrics:
+  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
+    repositoryNames:
+    - summerwind/actions-runner-controller
+```
+
+Please also note that the sync period is set to 10 minutes by default and it's configurable via `--sync-period` flag.
+
+Additionally, the autoscaling feature has an anti-flapping option that prevents periodic loop of scaling up and down.
+By default, it doesn't scale down until the grace period of 10 minutes passes after a scale up. The grace period can be configured by setting `scaleDownDelaySecondsAfterScaleUp`:
+
+```
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerDeployment
+metadata:
+  name: example-runner-deployment
+spec:
+  template:
+    spec:
+      repository: summerwind/actions-runner-controller
+---
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: HorizontalRunnerAutoscaler
+metadata:
+  name: example-runner-deployment-autoscaler
+spec:
+  scaleTargetRef:
+    name: example-runner-deployment
+  minReplicas: 1
+  maxReplicas: 3
+  scaleDownDelaySecondsAfterScaleOut: 60
+  metrics:
+  - type: TotalNumberOfQueuedAndInProgressWorkflowRuns
+    repositoryNames:
+    - summerwind/actions-runner-controller
+```
+
 ## Additional tweaks
 
 You can pass details through the spec selector. Here's an eg. of what you may like to do:
@@ -212,8 +274,8 @@ spec:
         operator: Exists
 
       repository: mumoshu/actions-runner-controller-ci
-      ImagePullPolicy: Always
       image: custom-image/actions-runner:latest
+      imagePullPolicy: Always
       resources:
         limits:
           cpu: "4.0"

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// HorizontalRunnerAutoscalerSpec defines the desired state of HorizontalRunnerAutoscaler
+type HorizontalRunnerAutoscalerSpec struct {
+	// ScaleTargetRef sis the reference to scaled resource like RunnerDeployment
+	ScaleTargetRef ScaleTargetRef `json:"scaleTargetRef,omitempty"`
+
+	// MinReplicas is the minimum number of replicas the deployment is allowed to scale
+	// +optional
+	MinReplicas *int `json:"minReplicas,omitempty"`
+
+	// MinReplicas is the maximum number of replicas the deployment is allowed to scale
+	// +optional
+	MaxReplicas *int `json:"maxReplicas,omitempty"`
+
+	// ScaleDownDelaySecondsAfterScaleUp is the approximate delay for a scale down followed by a scale up
+	// Used to prevent flapping (down->up->down->... loop)
+	// +optional
+	ScaleDownDelaySecondsAfterScaleUp *int `json:"scaleDownDelaySecondsAfterScaleOut,omitempty"`
+
+	// Metrics is the collection of various metric targets to calculate desired number of runners
+	// +optional
+	Metrics []MetricSpec `json:"metrics,omitempty"`
+}
+
+type ScaleTargetRef struct {
+	Name string `json:"name,omitempty"`
+}
+
+type MetricSpec struct {
+	// Type is the type of metric to be used for autoscaling.
+	// The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+	Type string `json:"type,omitempty"`
+
+	// RepositoryNames is the list of repository names to be used for calculating the metric.
+	// For example, a repository name is the REPO part of `github.com/USER/REPO`.
+	// +optional
+	RepositoryNames []string `json:"repositoryNames,omitempty"`
+}
+
+type HorizontalRunnerAutoscalerStatus struct {
+	// ObservedGeneration is the most recent generation observed for the target. It corresponds to e.g.
+	// RunnerDeployment's generation, which is updated on mutation by the API Server.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// DesiredReplicas is the total number of desired, non-terminated and latest pods to be set for the primary RunnerSet
+	// This doesn't include outdated pods while upgrading the deployment and replacing the runnerset.
+	// +optional
+	DesiredReplicas *int `json:"desiredReplicas,omitempty"`
+
+	// +optional
+	LastSuccessfulScaleOutTime *metav1.Time `json:"lastSuccessfulScaleOutTime,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:JSONPath=".spec.minReplicas",name=Min,type=number
+// +kubebuilder:printcolumn:JSONPath=".spec.maxReplicas",name=Max,type=number
+// +kubebuilder:printcolumn:JSONPath=".status.desiredReplicas",name=Desired,type=number
+
+// HorizontalRunnerAutoscaler is the Schema for the horizontalrunnerautoscaler API
+type HorizontalRunnerAutoscaler struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   HorizontalRunnerAutoscalerSpec   `json:"spec,omitempty"`
+	Status HorizontalRunnerAutoscalerStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// HorizontalRunnerAutoscalerList contains a list of HorizontalRunnerAutoscaler
+type HorizontalRunnerAutoscalerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []HorizontalRunnerAutoscaler `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&HorizontalRunnerAutoscaler{}, &HorizontalRunnerAutoscalerList{})
+}

api/v1alpha1/runner_types.go

@@ -48,6 +48,8 @@ type RunnerSpec struct {
 	// +optional
 	Image string `json:"image"`
 	// +optional
+	ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
+	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
 
 	// +optional

api/v1alpha1/runnerdeployment_types.go

@@ -20,9 +20,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns = "TotalNumberOfQueuedAndInProgressWorkflowRuns"
+)
+
 // RunnerReplicaSetSpec defines the desired state of RunnerDeployment
 type RunnerDeploymentSpec struct {
-	Replicas *int `json:"replicas"`
+	// +optional
+	Replicas *int `json:"replicas,omitempty"`
 
 	Template RunnerTemplate `json:"template"`
 }
@@ -30,6 +35,11 @@ type RunnerDeploymentSpec struct {
 type RunnerDeploymentStatus struct {
 	AvailableReplicas int `json:"availableReplicas"`
 	ReadyReplicas     int `json:"readyReplicas"`
+
+	// Replicas is the total number of desired, non-terminated and latest pods to be set for the primary RunnerSet
+	// This doesn't include outdated pods while upgrading the deployment and replacing the runnerset.
+	// +optional
+	Replicas *int `json:"desiredReplicas,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1alpha1/zz_generated.deepcopy.go

@@ -25,6 +25,147 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscaler) DeepCopyInto(out *HorizontalRunnerAutoscaler) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscaler.
+func (in *HorizontalRunnerAutoscaler) DeepCopy() *HorizontalRunnerAutoscaler {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscaler)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *HorizontalRunnerAutoscaler) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscalerList) DeepCopyInto(out *HorizontalRunnerAutoscalerList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]HorizontalRunnerAutoscaler, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerList.
+func (in *HorizontalRunnerAutoscalerList) DeepCopy() *HorizontalRunnerAutoscalerList {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscalerList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *HorizontalRunnerAutoscalerList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAutoscalerSpec) {
+	*out = *in
+	out.ScaleTargetRef = in.ScaleTargetRef
+	if in.MinReplicas != nil {
+		in, out := &in.MinReplicas, &out.MinReplicas
+		*out = new(int)
+		**out = **in
+	}
+	if in.MaxReplicas != nil {
+		in, out := &in.MaxReplicas, &out.MaxReplicas
+		*out = new(int)
+		**out = **in
+	}
+	if in.ScaleDownDelaySecondsAfterScaleUp != nil {
+		in, out := &in.ScaleDownDelaySecondsAfterScaleUp, &out.ScaleDownDelaySecondsAfterScaleUp
+		*out = new(int)
+		**out = **in
+	}
+	if in.Metrics != nil {
+		in, out := &in.Metrics, &out.Metrics
+		*out = make([]MetricSpec, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec.
+func (in *HorizontalRunnerAutoscalerSpec) DeepCopy() *HorizontalRunnerAutoscalerSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscalerSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HorizontalRunnerAutoscalerStatus) DeepCopyInto(out *HorizontalRunnerAutoscalerStatus) {
+	*out = *in
+	if in.DesiredReplicas != nil {
+		in, out := &in.DesiredReplicas, &out.DesiredReplicas
+		*out = new(int)
+		**out = **in
+	}
+	if in.LastSuccessfulScaleOutTime != nil {
+		in, out := &in.LastSuccessfulScaleOutTime, &out.LastSuccessfulScaleOutTime
+		*out = (*in).DeepCopy()
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerStatus.
+func (in *HorizontalRunnerAutoscalerStatus) DeepCopy() *HorizontalRunnerAutoscalerStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(HorizontalRunnerAutoscalerStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricSpec) DeepCopyInto(out *MetricSpec) {
+	*out = *in
+	if in.RepositoryNames != nil {
+		in, out := &in.RepositoryNames, &out.RepositoryNames
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricSpec.
+func (in *MetricSpec) DeepCopy() *MetricSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Runner) DeepCopyInto(out *Runner) {
 	*out = *in
@@ -58,7 +199,7 @@ func (in *RunnerDeployment) DeepCopyInto(out *RunnerDeployment) {
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	in.Spec.DeepCopyInto(&out.Spec)
-	out.Status = in.Status
+	in.Status.DeepCopyInto(&out.Status)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerDeployment.
@@ -135,6 +276,11 @@ func (in *RunnerDeploymentSpec) DeepCopy() *RunnerDeploymentSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RunnerDeploymentStatus) DeepCopyInto(out *RunnerDeploymentStatus) {
 	*out = *in
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerDeploymentStatus.
@@ -443,3 +589,18 @@ func (in *RunnerTemplate) DeepCopy() *RunnerTemplate {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ScaleTargetRef) DeepCopyInto(out *ScaleTargetRef) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleTargetRef.
+func (in *ScaleTargetRef) DeepCopy() *ScaleTargetRef {
+	if in == nil {
+		return nil
+	}
+	out := new(ScaleTargetRef)
+	in.DeepCopyInto(out)
+	return out
+}

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -0,0 +1,118 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.4
+  creationTimestamp: null
+  name: horizontalrunnerautoscalers.actions.summerwind.dev
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .spec.minReplicas
+    name: Min
+    type: number
+  - JSONPath: .spec.maxReplicas
+    name: Max
+    type: number
+  - JSONPath: .status.desiredReplicas
+    name: Desired
+    type: number
+  group: actions.summerwind.dev
+  names:
+    kind: HorizontalRunnerAutoscaler
+    listKind: HorizontalRunnerAutoscalerList
+    plural: horizontalrunnerautoscalers
+    singular: horizontalrunnerautoscaler
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: HorizontalRunnerAutoscaler is the Schema for the horizontalrunnerautoscaler
+        API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: HorizontalRunnerAutoscalerSpec defines the desired state of
+            HorizontalRunnerAutoscaler
+          properties:
+            maxReplicas:
+              description: MinReplicas is the maximum number of replicas the deployment
+                is allowed to scale
+              type: integer
+            metrics:
+              description: Metrics is the collection of various metric targets to
+                calculate desired number of runners
+              items:
+                properties:
+                  repositoryNames:
+                    description: RepositoryNames is the list of repository names to
+                      be used for calculating the metric. For example, a repository
+                      name is the REPO part of `github.com/USER/REPO`.
+                    items:
+                      type: string
+                    type: array
+                  type:
+                    description: Type is the type of metric to be used for autoscaling.
+                      The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns
+                    type: string
+                type: object
+              type: array
+            minReplicas:
+              description: MinReplicas is the minimum number of replicas the deployment
+                is allowed to scale
+              type: integer
+            scaleDownDelaySecondsAfterScaleOut:
+              description: ScaleDownDelaySecondsAfterScaleUp is the approximate delay
+                for a scale down followed by a scale up Used to prevent flapping (down->up->down->...
+                loop)
+              type: integer
+            scaleTargetRef:
+              description: ScaleTargetRef sis the reference to scaled resource like
+                RunnerDeployment
+              properties:
+                name:
+                  type: string
+              type: object
+          type: object
+        status:
+          properties:
+            desiredReplicas:
+              description: DesiredReplicas is the total number of desired, non-terminated
+                and latest pods to be set for the primary RunnerSet This doesn't include
+                outdated pods while upgrading the deployment and replacing the runnerset.
+              type: integer
+            lastSuccessfulScaleOutTime:
+              format: date-time
+              type: string
+            observedGeneration:
+              description: ObservedGeneration is the most recent generation observed
+                for the target. It corresponds to e.g. RunnerDeployment's generation,
+                which is updated on mutation by the API Server.
+              format: int64
+              type: integer
+          type: object
+      type: object
+  version: v1alpha1
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crd/bases/actions.summerwind.dev_runnerdeployments.yaml

@@ -3006,6 +3006,10 @@ spec:
                       type: array
                     image:
                       type: string
+                    imagePullPolicy:
+                      description: PullPolicy describes a policy for if/when to pull
+                        a container image
+                      type: string
                     imagePullSecrets:
                       items:
                         description: LocalObjectReference contains enough information
@@ -6717,13 +6721,17 @@ spec:
                   type: object
               type: object
           required:
-          - replicas
           - template
           type: object
         status:
           properties:
             availableReplicas:
               type: integer
+            desiredReplicas:
+              description: Replicas is the total number of desired, non-terminated
+                and latest pods to be set for the primary RunnerSet This doesn't include
+                outdated pods while upgrading the deployment and replacing the runnerset.
+              type: integer
             readyReplicas:
               type: integer
           required:

config/crd/bases/actions.summerwind.dev_runnerreplicasets.yaml

@@ -3006,6 +3006,10 @@ spec:
                       type: array
                     image:
                       type: string
+                    imagePullPolicy:
+                      description: PullPolicy describes a policy for if/when to pull
+                        a container image
+                      type: string
                     imagePullSecrets:
                       items:
                         description: LocalObjectReference contains enough information

config/crd/bases/actions.summerwind.dev_runners.yaml

@@ -2815,6 +2815,10 @@ spec:
               type: array
             image:
               type: string
+            imagePullPolicy:
+              description: PullPolicy describes a policy for if/when to pull a container
+                image
+              type: string
             imagePullSecrets:
               items:
                 description: LocalObjectReference contains enough information to let

config/rbac/role.yaml

@@ -6,6 +6,26 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - actions.summerwind.dev
+  resources:
+  - horizontalrunnerautoscalers/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - actions.summerwind.dev
   resources:

controllers/autoscaling.go

@@ -0,0 +1,104 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"strings"
+)
+
+func (r *HorizontalRunnerAutoscalerReconciler) determineDesiredReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+	if hra.Spec.MinReplicas == nil {
+		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
+	} else if hra.Spec.MaxReplicas == nil {
+		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing maxReplicas", hra.Namespace, hra.Name)
+	}
+
+	var repos [][]string
+
+	repoID := rd.Spec.Template.Spec.Repository
+	if repoID == "" {
+		orgName := rd.Spec.Template.Spec.Organization
+		if orgName == "" {
+			return nil, fmt.Errorf("asserting runner deployment spec to detect bug: spec.template.organization should not be empty on this code path")
+		}
+
+		metrics := hra.Spec.Metrics
+
+		if len(metrics) == 0 {
+			return nil, fmt.Errorf("validating autoscaling metrics: one or more metrics is required")
+		} else if tpe := metrics[0].Type; tpe != v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns {
+			return nil, fmt.Errorf("validting autoscaling metrics: unsupported metric type %q: only supported value is %s", tpe, v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns)
+		} else if len(metrics[0].RepositoryNames) == 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment")
+		}
+
+		for _, repoName := range metrics[0].RepositoryNames {
+			repos = append(repos, []string{orgName, repoName})
+		}
+	} else {
+		repo := strings.Split(repoID, "/")
+
+		repos = append(repos, repo)
+	}
+
+	var total, inProgress, queued, completed, unknown int
+
+	for _, repo := range repos {
+		user, repoName := repo[0], repo[1]
+		list, _, err := r.GitHubClient.Actions.ListRepositoryWorkflowRuns(context.TODO(), user, repoName, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, r := range list.WorkflowRuns {
+			total++
+
+			// In May 2020, there are only 3 statuses.
+			// Follow the below links for more details:
+			// - https://developer.github.com/v3/actions/workflow-runs/#list-repository-workflow-runs
+			// - https://developer.github.com/v3/checks/runs/#create-a-check-run
+			switch r.GetStatus() {
+			case "completed":
+				completed++
+			case "in_progress":
+				inProgress++
+			case "queued":
+				queued++
+			default:
+				unknown++
+			}
+		}
+	}
+
+	minReplicas := *hra.Spec.MinReplicas
+	maxReplicas := *hra.Spec.MaxReplicas
+	necessaryReplicas := queued + inProgress
+
+	var desiredReplicas int
+
+	if necessaryReplicas < minReplicas {
+		desiredReplicas = minReplicas
+	} else if necessaryReplicas > maxReplicas {
+		desiredReplicas = maxReplicas
+	} else {
+		desiredReplicas = necessaryReplicas
+	}
+
+	rd.Status.Replicas = &desiredReplicas
+	replicas := desiredReplicas
+
+	r.Log.V(1).Info(
+		"Calculated desired replicas",
+		"computed_replicas_desired", desiredReplicas,
+		"spec_replicas_min", minReplicas,
+		"spec_replicas_max", maxReplicas,
+		"workflow_runs_completed", completed,
+		"workflow_runs_in_progress", inProgress,
+		"workflow_runs_queued", queued,
+		"workflow_runs_unknown", unknown,
+	)
+
+	return &replicas, nil
+}

controllers/autoscaling_test.go

@@ -0,0 +1,399 @@
+package controllers
+
+import (
+	"fmt"
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+	"github.com/summerwind/actions-runner-controller/github"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"net/http/httptest"
+	"net/url"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"testing"
+)
+
+func newGithubClient(server *httptest.Server) *github.Client {
+	client, err := github.NewClientWithAccessToken("token")
+	if err != nil {
+		panic(err)
+	}
+
+	baseURL, err := url.Parse(server.URL + "/")
+	if err != nil {
+		panic(err)
+	}
+	client.Client.BaseURL = baseURL
+
+	return client
+}
+
+func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
+	intPtr := func(v int) *int {
+		return &v
+	}
+
+	metav1Now := metav1.Now()
+	testcases := []struct {
+		repo         string
+		org          string
+		fixed        *int
+		max          *int
+		min          *int
+		sReplicas    *int
+		sTime        *metav1.Time
+		workflowRuns string
+		want         int
+		err          string
+	}{
+		// 3 demanded, max at 3
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			sReplicas:    intPtr(3),
+			sTime:        &metav1Now,
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 3 demanded, max at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(2),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 2 demanded, min at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			repo:         "test/valid",
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 1
+		{
+			repo:         "test/valid",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// 1 demanded, min at 1
+		{
+			repo:         "test/valid",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// fixed at 3
+		{
+			repo:         "test/valid",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			fixed:        intPtr(3),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+	}
+
+	for i := range testcases {
+		tc := testcases[i]
+
+		log := zap.New(func(o *zap.Options) {
+			o.Development = true
+		})
+
+		scheme := runtime.NewScheme()
+		_ = clientgoscheme.AddToScheme(scheme)
+		_ = v1alpha1.AddToScheme(scheme)
+
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			defer server.Close()
+			client := newGithubClient(server)
+
+			h := &HorizontalRunnerAutoscalerReconciler{
+				Log:          log,
+				GitHubClient: client,
+				Scheme:       scheme,
+			}
+
+			rd := v1alpha1.RunnerDeployment{
+				TypeMeta: metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testrd",
+				},
+				Spec: v1alpha1.RunnerDeploymentSpec{
+					Template: v1alpha1.RunnerTemplate{
+						Spec: v1alpha1.RunnerSpec{
+							Repository: tc.repo,
+						},
+					},
+					Replicas: tc.fixed,
+				},
+				Status: v1alpha1.RunnerDeploymentStatus{
+					Replicas: tc.sReplicas,
+				},
+			}
+
+			hra := v1alpha1.HorizontalRunnerAutoscaler{
+				Spec: v1alpha1.HorizontalRunnerAutoscalerSpec{
+					MaxReplicas: tc.max,
+					MinReplicas: tc.min,
+				},
+				Status: v1alpha1.HorizontalRunnerAutoscalerStatus{
+					DesiredReplicas:            tc.sReplicas,
+					LastSuccessfulScaleOutTime: tc.sTime,
+				},
+			}
+
+			got, err := h.computeReplicas(rd, hra)
+			if err != nil {
+				if tc.err == "" {
+					t.Fatalf("unexpected error: expected none, got %v", err)
+				} else if err.Error() != tc.err {
+					t.Fatalf("unexpected error: expected %v, got %v", tc.err, err)
+				}
+				return
+			}
+
+			if got == nil {
+				t.Fatalf("unexpected value of rs.Spec.Replicas: nil")
+			}
+
+			if *got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, *got)
+			}
+		})
+	}
+}
+
+func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
+	intPtr := func(v int) *int {
+		return &v
+	}
+
+	metav1Now := metav1.Now()
+	testcases := []struct {
+		repos        []string
+		org          string
+		fixed        *int
+		max          *int
+		min          *int
+		sReplicas    *int
+		sTime        *metav1.Time
+		workflowRuns string
+		want         int
+		err          string
+	}{
+		// 3 demanded, max at 3
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			sReplicas:    intPtr(3),
+			sTime:        &metav1Now,
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// 3 demanded, max at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(2),
+			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 2 demanded, min at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 2
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(2),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         2,
+		},
+		// 1 demanded, min at 1
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// 1 demanded, min at 1
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         1,
+		},
+		// fixed at 3
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			fixed:        intPtr(1),
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// org runner, fixed at 3
+		{
+			org:          "test",
+			repos:        []string{"valid"},
+			fixed:        intPtr(1),
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			want:         3,
+		},
+		// org runner, 1 demanded, min at 1, no repos
+		{
+			org:          "test",
+			min:          intPtr(1),
+			max:          intPtr(3),
+			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			err:          "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
+		},
+	}
+
+	for i := range testcases {
+		tc := testcases[i]
+
+		log := zap.New(func(o *zap.Options) {
+			o.Development = true
+		})
+
+		scheme := runtime.NewScheme()
+		_ = clientgoscheme.AddToScheme(scheme)
+		_ = v1alpha1.AddToScheme(scheme)
+
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns))
+			defer server.Close()
+			client := newGithubClient(server)
+
+			h := &HorizontalRunnerAutoscalerReconciler{
+				Log:          log,
+				Scheme:       scheme,
+				GitHubClient: client,
+			}
+
+			rd := v1alpha1.RunnerDeployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testrd",
+				},
+				Spec: v1alpha1.RunnerDeploymentSpec{
+					Template: v1alpha1.RunnerTemplate{
+						Spec: v1alpha1.RunnerSpec{
+							Organization: tc.org,
+						},
+					},
+					Replicas: tc.fixed,
+				},
+				Status: v1alpha1.RunnerDeploymentStatus{
+					Replicas: tc.sReplicas,
+				},
+			}
+
+			hra := v1alpha1.HorizontalRunnerAutoscaler{
+				Spec: v1alpha1.HorizontalRunnerAutoscalerSpec{
+					ScaleTargetRef: v1alpha1.ScaleTargetRef{
+						Name: "testrd",
+					},
+					MaxReplicas: tc.max,
+					MinReplicas: tc.min,
+					Metrics: []v1alpha1.MetricSpec{
+						{
+							Type:            v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns,
+							RepositoryNames: tc.repos,
+						},
+					},
+				},
+				Status: v1alpha1.HorizontalRunnerAutoscalerStatus{
+					DesiredReplicas:            tc.sReplicas,
+					LastSuccessfulScaleOutTime: tc.sTime,
+				},
+			}
+
+			got, err := h.computeReplicas(rd, hra)
+			if err != nil {
+				if tc.err == "" {
+					t.Fatalf("unexpected error: expected none, got %v", err)
+				} else if err.Error() != tc.err {
+					t.Fatalf("unexpected error: expected %v, got %v", tc.err, err)
+				}
+				return
+			}
+
+			if got == nil {
+				t.Fatalf("unexpected value of rs.Spec.Replicas: nil, wanted %v", tc.want)
+			}
+
+			if *got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, *got)
+			}
+		})
+	}
+}

controllers/horizontalrunnerautoscaler_controller.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	"github.com/summerwind/actions-runner-controller/github"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+const (
+	DefaultScaleDownDelay = 10 * time.Minute
+)
+
+// HorizontalRunnerAutoscalerReconciler reconciles a HorizontalRunnerAutoscaler object
+type HorizontalRunnerAutoscalerReconciler struct {
+	client.Client
+	GitHubClient *github.Client
+	Log          logr.Logger
+	Recorder     record.EventRecorder
+	Scheme       *runtime.Scheme
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	ctx := context.Background()
+	log := r.Log.WithValues("horizontalrunnerautoscaler", req.NamespacedName)
+
+	var hra v1alpha1.HorizontalRunnerAutoscaler
+	if err := r.Get(ctx, req.NamespacedName, &hra); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	var rd v1alpha1.RunnerDeployment
+	if err := r.Get(ctx, types.NamespacedName{
+		Namespace: req.Namespace,
+		Name:      hra.Spec.ScaleTargetRef.Name,
+	}, &rd); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !rd.ObjectMeta.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	replicas, err := r.computeReplicas(rd, hra)
+	if err != nil {
+		r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
+
+		log.Error(err, "Could not compute replicas")
+
+		return ctrl.Result{}, err
+	}
+
+	const defaultReplicas = 1
+
+	currentDesiredReplicas := getIntOrDefault(rd.Spec.Replicas, defaultReplicas)
+	newDesiredReplicas := getIntOrDefault(replicas, defaultReplicas)
+
+	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
+	if currentDesiredReplicas != newDesiredReplicas {
+		copy := rd.DeepCopy()
+		copy.Spec.Replicas = &newDesiredReplicas
+
+		if err := r.Client.Update(ctx, copy); err != nil {
+			log.Error(err, "Failed to update runnerderployment resource")
+
+			return ctrl.Result{}, err
+		}
+
+		return ctrl.Result{}, err
+	}
+
+	if hra.Status.DesiredReplicas == nil || *hra.Status.DesiredReplicas != *replicas {
+		updated := hra.DeepCopy()
+
+		if (hra.Status.DesiredReplicas == nil && *replicas > 1) ||
+			(hra.Status.DesiredReplicas != nil && *replicas > *hra.Status.DesiredReplicas) {
+
+			updated.Status.LastSuccessfulScaleOutTime = &metav1.Time{Time: time.Now()}
+		}
+
+		updated.Status.DesiredReplicas = replicas
+
+		if err := r.Status().Update(ctx, updated); err != nil {
+			log.Error(err, "Failed to update horizontalrunnerautoscaler status")
+
+			return ctrl.Result{}, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.Recorder = mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller")
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Complete(r)
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicas(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+	var computedReplicas *int
+
+	replicas, err := r.determineDesiredReplicas(rd, hra)
+	if err != nil {
+		return nil, err
+	}
+
+	var scaleDownDelay time.Duration
+
+	if hra.Spec.ScaleDownDelaySecondsAfterScaleUp != nil {
+		scaleDownDelay = time.Duration(*hra.Spec.ScaleDownDelaySecondsAfterScaleUp) * time.Second
+	} else {
+		scaleDownDelay = DefaultScaleDownDelay
+	}
+
+	now := time.Now()
+
+	if hra.Status.DesiredReplicas == nil ||
+		*hra.Status.DesiredReplicas < *replicas ||
+		hra.Status.LastSuccessfulScaleOutTime == nil ||
+		hra.Status.LastSuccessfulScaleOutTime.Add(scaleDownDelay).Before(now) {
+
+		computedReplicas = replicas
+	} else {
+		computedReplicas = hra.Status.DesiredReplicas
+	}
+
+	return computedReplicas, nil
+}

controllers/integration_test.go

@@ -0,0 +1,254 @@
+package controllers
+
+import (
+	"context"
+	"github.com/summerwind/actions-runner-controller/github/fake"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+// SetupIntegrationTest will set up a testing environment.
+// This includes:
+// * creating a Namespace to be used during the test
+// * starting all the reconcilers
+// * stopping all the reconcilers after the test ends
+// Call this function at the start of each of your tests.
+func SetupIntegrationTest(ctx context.Context) *corev1.Namespace {
+	var stopCh chan struct{}
+	ns := &corev1.Namespace{}
+
+	workflowRuns := `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`
+	server := fake.NewServer(fake.WithListRepositoryWorkflowRunsResponse(200, workflowRuns))
+
+	BeforeEach(func() {
+		stopCh = make(chan struct{})
+		*ns = corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)},
+		}
+
+		err := k8sClient.Create(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
+		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
+
+		replicasetController := &RunnerReplicaSetReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+		}
+		err = replicasetController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		deploymentsController := &RunnerDeploymentReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerdeployment-controller"),
+		}
+		err = deploymentsController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		client := newGithubClient(server)
+
+		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			GitHubClient: client,
+			Recorder:     mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+		}
+		err = autoscalerController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		go func() {
+			defer GinkgoRecover()
+
+			err := mgr.Start(stopCh)
+			Expect(err).NotTo(HaveOccurred(), "failed to start manager")
+		}()
+	})
+
+	AfterEach(func() {
+		close(stopCh)
+
+		server.Close()
+
+		err := k8sClient.Delete(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
+	})
+
+	return ns
+}
+
+var _ = Context("Inside of a new namespace", func() {
+	ctx := context.TODO()
+	ns := SetupIntegrationTest(ctx)
+
+	Describe("when no existing resources exist", func() {
+
+		It("should create and scale runners", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rs := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								Repository: "test/valid",
+								Image:      "bar",
+								Env: []corev1.EnvVar{
+									{Name: "FOO", Value: "FOOVALUE"},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerDeployment resource")
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+			}
+
+			{
+				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+				Eventually(func() error {
+					var rd actionsv1alpha1.RunnerDeployment
+
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
+
+					Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerDeployment resource")
+
+					rd.Spec.Replicas = intPtr(2)
+
+					return k8sClient.Update(ctx, &rd)
+				},
+					time.Second*1, time.Millisecond*500).Should(BeNil())
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
+			}
+
+			{
+				rs := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(3),
+						ScaleDownDelaySecondsAfterScaleUp: nil,
+						Metrics:                           nil,
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerDeployment resource")
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(3))
+			}
+		})
+	})
+})

controllers/runner_controller.go

@@ -261,6 +261,11 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 		runnerImage = r.RunnerImage
 	}
 
+	runnerImagePullPolicy := runner.Spec.ImagePullPolicy
+	if runnerImagePullPolicy == "" {
+		runnerImagePullPolicy = corev1.PullAlways
+	}
+
 	env := []corev1.EnvVar{
 		{
 			Name:  "RUNNER_NAME",
@@ -298,7 +303,7 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 				{
 					Name:            containerName,
 					Image:           runnerImage,
-					ImagePullPolicy: "Always",
+					ImagePullPolicy: runnerImagePullPolicy,
 					Env:             env,
 					EnvFrom:         runner.Spec.EnvFrom,
 					VolumeMounts: []corev1.VolumeMount{

controllers/runnerdeployment_controller.go

@@ -20,10 +20,11 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
-	"k8s.io/apimachinery/pkg/types"
 	"sort"
 	"time"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/davecgh/go-spew/spew"
 	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -96,13 +97,15 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 
 	desiredRS, err := r.newRunnerReplicaSet(rd)
 	if err != nil {
+		r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
+
 		log.Error(err, "Could not create runnerreplicaset")
 
 		return ctrl.Result{}, err
 	}
 
 	if newestSet == nil {
-		if err := r.Client.Create(ctx, &desiredRS); err != nil {
+		if err := r.Client.Create(ctx, desiredRS); err != nil {
 			log.Error(err, "Failed to create runnerreplicaset resource")
 
 			return ctrl.Result{}, err
@@ -118,7 +121,7 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		return ctrl.Result{}, nil
 	}
 
-	desiredTemplateHash, ok := getTemplateHash(&desiredRS)
+	desiredTemplateHash, ok := getTemplateHash(desiredRS)
 	if !ok {
 		log.Info("Failed to get template hash of desired runnerreplicaset resource. It must be in an invalid state. Please manually delete the runnerreplicaset so that it is recreated")
 
@@ -126,7 +129,7 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	}
 
 	if newestTemplateHash != desiredTemplateHash {
-		if err := r.Client.Create(ctx, &desiredRS); err != nil {
+		if err := r.Client.Create(ctx, desiredRS); err != nil {
 			log.Error(err, "Failed to create runnerreplicaset resource")
 
 			return ctrl.Result{}, err
@@ -184,6 +187,17 @@ func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		}
 	}
 
+	if rd.Spec.Replicas == nil && desiredRS.Spec.Replicas != nil {
+		updated := rd.DeepCopy()
+		updated.Status.Replicas = desiredRS.Spec.Replicas
+
+		if err := r.Status().Update(ctx, updated); err != nil {
+			log.Error(err, "Failed to update runnerdeployment status")
+
+			return ctrl.Result{}, err
+		}
+	}
+
 	return ctrl.Result{}, nil
 }
 
@@ -241,7 +255,7 @@ func CloneAndAddLabel(labels map[string]string, labelKey, labelValue string) map
 	return newLabels
 }
 
-func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeployment) (v1alpha1.RunnerReplicaSet, error) {
+func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeployment) (*v1alpha1.RunnerReplicaSet, error) {
 	newRSTemplate := *rd.Spec.Template.DeepCopy()
 	templateHash := ComputeHash(&newRSTemplate)
 	// Add template hash label to selector.
@@ -263,10 +277,10 @@ func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeplo
 	}
 
 	if err := ctrl.SetControllerReference(&rd, &rs, r.Scheme); err != nil {
-		return rs, err
+		return &rs, err
 	}
 
-	return rs, nil
+	return &rs, nil
 }
 
 func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {

github/fake/fake.go

@@ -32,7 +32,13 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 }
 
 // NewServer creates a fake server for running unit tests
-func NewServer() *httptest.Server {
+func NewServer(opts ...Option) *httptest.Server {
+	var responses FixedResponses
+
+	for _, o := range opts {
+		o(&responses)
+	}
+
 	routes := map[string]handler{
 		// For CreateRegistrationToken
 		"/repos/test/valid/actions/runners/registration-token": handler{
@@ -111,6 +117,9 @@ func NewServer() *httptest.Server {
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
+
+		// For auto-scaling based on the number of queued(pending) workflow runs
+		"/repos/test/valid/actions/runs": responses.listRepositoryWorkflowRuns.handler(),
 	}
 
 	mux := http.NewServeMux()

github/fake/options.go

@@ -0,0 +1,28 @@
+package fake
+
+type FixedResponses struct {
+	listRepositoryWorkflowRuns FixedResponse
+}
+
+type FixedResponse struct {
+	Status int
+	Body   string
+}
+
+func (r FixedResponse) handler() handler {
+	return handler{
+		Status: r.Status,
+		Body:   r.Body,
+	}
+}
+
+type Option func(responses *FixedResponses)
+
+func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
+	return func(r *FixedResponses) {
+		r.listRepositoryWorkflowRuns = FixedResponse{
+			Status: status,
+			Body:   body,
+		}
+	}
+}

github/github.go

@@ -27,8 +27,10 @@ func NewClient(appID, installationID int64, privateKeyPath string) (*Client, err
 		return nil, fmt.Errorf("authentication failed: %v", err)
 	}
 
+	gh := github.NewClient(&http.Client{Transport: tr})
+
 	return &Client{
-		Client:    github.NewClient(&http.Client{Transport: tr}),
+		Client:    gh,
 		regTokens: map[string]*github.RegistrationToken{},
 		mu:        sync.Mutex{},
 	}, nil

go.mod

@@ -15,5 +15,6 @@ require (
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
 	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
 	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
+	k8s.io/klog v0.4.0
 	sigs.k8s.io/controller-runtime v0.4.0
 )

main.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"time"
 
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
 	"github.com/summerwind/actions-runner-controller/controllers"
@@ -57,6 +58,7 @@ func main() {
 
 		metricsAddr          string
 		enableLeaderElection bool
+		syncPeriod           time.Duration
 
 		runnerImage string
 		dockerImage string
@@ -76,6 +78,7 @@ func main() {
 	flag.Int64Var(&ghAppID, "github-app-id", 0, "The application ID of GitHub App.")
 	flag.Int64Var(&ghAppInstallationID, "github-app-installation-id", 0, "The installation ID of GitHub App.")
 	flag.StringVar(&ghAppPrivateKey, "github-app-private-key", "", "The path of a private key file to authenticate as a GitHub App")
+	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
 	flag.Parse()
 
 	if ghToken == "" {
@@ -133,6 +136,7 @@ func main() {
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
 		Port:               9443,
+		SyncPeriod:         &syncPeriod,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
@@ -175,6 +179,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	horizontalRunnerAutoscaler := &controllers.HorizontalRunnerAutoscalerReconciler{
+		Client:       mgr.GetClient(),
+		Log:          ctrl.Log.WithName("controllers").WithName("HorizontalRunnerAutoscaler"),
+		Scheme:       mgr.GetScheme(),
+		GitHubClient: ghClient,
+	}
+
+	if err = horizontalRunnerAutoscaler.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "HorizontalRunnerAutoscaler")
+		os.Exit(1)
+	}
+
 	if err = (&actionsv1alpha1.Runner{}).SetupWebhookWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create webhook", "webhook", "Runner")
 		os.Exit(1)

runner/Makefile

@@ -1,7 +1,7 @@
 NAME ?= summerwind/actions-runner
 
-RUNNER_VERSION ?= 2.169.1
-DOCKER_VERSION ?= 19.03.8
+RUNNER_VERSION ?= 2.272.0
+DOCKER_VERSION ?= 19.03.12
 
 docker-build:
 	docker build --build-arg RUNNER_VERSION=${RUNNER_VERSION} --build-arg DOCKER_VERSION=${DOCKER_VERSION} -t ${NAME}:latest -t ${NAME}:v${RUNNER_VERSION} .