Merge pull request #328 from int128/fix-port-name-length

Changes:

1. Fix length of github-webhook-server port name
2. Add a cluster role binding for github-webhook-server
3. Remove --enable-leader-election from github-webhook-server

.github/workflows/build-and-release-runners.yml

@@ -6,7 +6,7 @@ on:
       - '**'
     paths:
       - 'runner/**'
-      - .github/workflows/build-runner.yml
+      - .github/workflows/build-and-release-runners.yml
   push:
     branches:
       - master
@@ -15,10 +15,8 @@ on:
       - runner/Dockerfile
       - runner/dindrunner.Dockerfile
       - runner/entrypoint.sh
-      - .github/workflows/build-runner.yml
+      - .github/workflows/build-and-release-runners.yml
-  release:
+
-    types: [published]
-name: Runner
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -31,7 +29,7 @@ jobs:
           - name: actions-runner-dind
             dockerfile: dindrunner.Dockerfile
     env:
-      RUNNER_VERSION: 2.276.1
+      RUNNER_VERSION: 2.277.1
       DOCKER_VERSION: 19.03.12
       DOCKERHUB_USERNAME: ${{ github.repository_owner }}
     steps:
@@ -52,36 +50,17 @@ jobs:
 
       - name: Login to DockerHub
         uses: docker/login-action@v1
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name == 'push' || github.event_name == 'release' }}
         with:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
-      # Considered unstable builds
+      - name: Build and Push
-      # Mutable (no sha) and immutable (include sha) tags are created, see Issue 285 and PR 286 for why
-      - name: Build and push canary builds
         uses: docker/build-push-action@v2
         with:
           context: ./runner
           file: ./runner/${{ matrix.dockerfile }}
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' && github.event_name != 'release' }}
-          build-args: |
-            RUNNER_VERSION=${{ env.RUNNER_VERSION }}
-            DOCKER_VERSION=${{ env.DOCKER_VERSION }}
-          tags: |
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-canary
-            ${{ env.DOCKERHUB_USERNAME }}/${{ matrix.name }}:v${{ env.RUNNER_VERSION }}-canary-${{ steps.vars.outputs.sha_short }}
-
-      # Considered stable builds
-      # Mutable (no sha) and immutable (include sha) tags are created, see Issue 285 and PR 286 for why
-      - name: Build and push release builds
-        uses: docker/build-push-action@v2
-        with:
-          context: ./runner
-          file: ./runner/${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name == 'release' }}
           build-args: |
             RUNNER_VERSION=${{ env.RUNNER_VERSION }}
             DOCKER_VERSION=${{ env.DOCKER_VERSION }}

.github/workflows/release.yml

@@ -57,6 +57,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}
             ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:${{ env.VERSION }}-${{ steps.vars.outputs.sha_short }}
 

.github/workflows/test.yaml

@@ -6,6 +6,7 @@ on:
       - master
     paths-ignore:
     - 'runner/**'
+    - .github/workflows/build-and-release-runners.yml
 
 jobs:
   test:

.github/workflows/wip.yml

@@ -30,11 +30,13 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
+      # Considered unstable builds
+      # See Issue #285, PR #286, and PR #323 for more information
       - name: Build and Push
         uses: docker/build-push-action@v2
         with:
           file: Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:latest 
+          tags: |
-
+            ${{ env.DOCKERHUB_USERNAME }}/actions-runner-controller:canary

README.md

@@ -1,5 +1,7 @@
 # actions-runner-controller
 
+[![awesome-runners](https://img.shields.io/badge/listed%20on-awesome--runners-blue.svg)](https://github.com/jonico/awesome-runners)
+
 This controller operates self-hosted runners for GitHub Actions on your Kubernetes cluster.
 
 ToC:

api/v1alpha1/horizontalrunnerautoscaler_types.go

@@ -126,6 +126,16 @@ type MetricSpec struct {
 	// to determine how many pods should be removed.
 	// +optional
 	ScaleDownFactor string `json:"scaleDownFactor,omitempty"`
+
+	// ScaleUpAdjustment is the number of runners added on scale-up.
+	// You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+	// +optional
+	ScaleUpAdjustment int `json:"scaleUpAdjustment,omitempty"`
+
+	// ScaleDownAdjustment is the number of runners removed on scale-down.
+	// You can only specify either ScaleDownFactor or ScaleDownAdjustment.
+	// +optional
+	ScaleDownAdjustment int `json:"scaleDownAdjustment,omitempty"`
 }
 
 type HorizontalRunnerAutoscalerStatus struct {

charts/actions-runner-controller/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.0
+version: 0.5.2
 
 home: https://github.com/summerwind/actions-runner-controller
 

charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -78,6 +78,11 @@ spec:
                     items:
                       type: string
                     type: array
+                  scaleDownAdjustment:
+                    description: ScaleDownAdjustment is the number of runners removed
+                      on scale-down. You can only specify either ScaleDownFactor or
+                      ScaleDownAdjustment.
+                    type: integer
                   scaleDownFactor:
                     description: ScaleDownFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many
@@ -87,6 +92,10 @@ spec:
                     description: ScaleDownThreshold is the percentage of busy runners
                       less than which will trigger the hpa to scale the runners down.
                     type: string
+                  scaleUpAdjustment:
+                    description: ScaleUpAdjustment is the number of runners added
+                      on scale-up. You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+                    type: integer
                   scaleUpFactor:
                     description: ScaleUpFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many

charts/actions-runner-controller/templates/_helpers.tpl

@@ -85,11 +85,11 @@ Create the name of the service account to use
 {{- end }}
 
 {{- define "actions-runner-controller.webhookServiceName" -}}
-{{- include "actions-runner-controller.fullname" . }}-webhook
+{{- include "actions-runner-controller.fullname" . | trunc 55 }}-webhook
 {{- end }}
 
 {{- define "actions-runner-controller.authProxyServiceName" -}}
-{{- include "actions-runner-controller.fullname" . }}-metrics-service
+{{- include "actions-runner-controller.fullname" . | trunc 47 }}-metrics-service
 {{- end }}
 
 {{- define "actions-runner-controller.selfsignedIssuerName" -}}

charts/actions-runner-controller/templates/githubwebhook.deployment.yaml

@@ -32,7 +32,6 @@ spec:
       containers:
       - args:
         - "--metrics-addr=127.0.0.1:8080"
-        - "--enable-leader-election"
         - "--sync-period={{ .Values.githubWebhookServer.syncPeriod }}"
         command:
         - "/github-webhook-server"
@@ -52,7 +51,7 @@ spec:
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         ports:
         - containerPort: 8000
-          name: github-webhook-server
+          name: http
           protocol: TCP
         resources:
           {{- toYaml .Values.githubWebhookServer.resources | nindent 12 }}

charts/actions-runner-controller/templates/githubwebhook.role_binding.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.githubWebhookServer.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "actions-runner-controller-github-webhook-server.roleName" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "actions-runner-controller-github-webhook-server.roleName" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "actions-runner-controller-github-webhook-server.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

charts/actions-runner-controller/values.yaml

@@ -28,8 +28,8 @@ image:
 
 kube_rbac_proxy:
   image:
-    repository: gcr.io/kubebuilder/kube-rbac-proxy
+    repository: quay.io/brancz/kube-rbac-proxy
-    tag: v0.4.1
+    tag: v0.8.0
 
 imagePullSecrets: []
 nameOverride: ""
@@ -147,7 +147,7 @@ githubWebhookServer:
     type: NodePort
     ports:
     - port: 80
-      targetPort: 8000
+      targetPort: http
       protocol: TCP
       name: http
       #nodePort: someFixedPortForUseWithTerraformCdkCfnEtc

config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml

@@ -78,6 +78,11 @@ spec:
                     items:
                       type: string
                     type: array
+                  scaleDownAdjustment:
+                    description: ScaleDownAdjustment is the number of runners removed
+                      on scale-down. You can only specify either ScaleDownFactor or
+                      ScaleDownAdjustment.
+                    type: integer
                   scaleDownFactor:
                     description: ScaleDownFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many
@@ -87,6 +92,10 @@ spec:
                     description: ScaleDownThreshold is the percentage of busy runners
                       less than which will trigger the hpa to scale the runners down.
                     type: string
+                  scaleUpAdjustment:
+                    description: ScaleUpAdjustment is the number of runners added
+                      on scale-up. You can only specify either ScaleUpFactor or ScaleUpAdjustment.
+                    type: integer
                   scaleUpFactor:
                     description: ScaleUpFactor is the multiplicative factor applied
                       to the current number of runners used to determine how many

config/default/manager_auth_proxy_patch.yaml

@@ -10,7 +10,7 @@ spec:
     spec:
       containers:
       - name: kube-rbac-proxy
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
+        image: quay.io/brancz/kube-rbac-proxy:v0.8.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8080/"

controllers/autoscaling.go

@@ -189,6 +189,9 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByQueuedAndInPro
 		"workflow_runs_in_progress", inProgress,
 		"workflow_runs_queued", queued,
 		"workflow_runs_unknown", unknown,
+		"namespace", hra.Namespace,
+		"runner_deployment", rd.Name,
+		"horizontal_runner_autoscaler", hra.Name,
 	)
 
 	return &replicas, nil
@@ -196,7 +199,6 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByQueuedAndInPro
 
 func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunnersBusy(rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
 	ctx := context.Background()
-	orgName := rd.Spec.Template.Spec.Organization
 	minReplicas := *hra.Spec.MinReplicas
 	maxReplicas := *hra.Spec.MaxReplicas
 	metrics := hra.Spec.Metrics[0]
@@ -220,14 +222,34 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunn
 
 		scaleDownThreshold = sdt
 	}
+
+	scaleUpAdjustment := metrics.ScaleUpAdjustment
+	if scaleUpAdjustment != 0 {
+		if metrics.ScaleUpAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpAdjustment cannot be lower than 0")
+		}
+
 		if metrics.ScaleUpFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleUpAdjustment and scaleUpFactor cannot be specified together")
+		}
+	} else if metrics.ScaleUpFactor != "" {
 		suf, err := strconv.ParseFloat(metrics.ScaleUpFactor, 64)
 		if err != nil {
 			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpFactor cannot be parsed into a float64")
 		}
 		scaleUpFactor = suf
 	}
+
+	scaleDownAdjustment := metrics.ScaleDownAdjustment
+	if scaleDownAdjustment != 0 {
+		if metrics.ScaleDownAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownAdjustment cannot be lower than 0")
+		}
+
 		if metrics.ScaleDownFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleDownAdjustment and scaleDownFactor cannot be specified together")
+		}
+	} else if metrics.ScaleDownFactor != "" {
 		sdf, err := strconv.ParseFloat(metrics.ScaleDownFactor, 64)
 		if err != nil {
 			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownFactor cannot be parsed into a float64")
@@ -245,8 +267,18 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunn
 		runnerMap[items.Name] = struct{}{}
 	}
 
+	var (
+		enterprise   = rd.Spec.Template.Spec.Enterprise
+		organization = rd.Spec.Template.Spec.Organization
+		repository   = rd.Spec.Template.Spec.Repository
+	)
+
 	// ListRunners will return all runners managed by GitHub - not restricted to ns
-	runners, err := r.GitHubClient.ListRunners(ctx, "", orgName, "")
+	runners, err := r.GitHubClient.ListRunners(
+		ctx,
+		enterprise,
+		organization,
+		repository)
 	if err != nil {
 		return nil, err
 	}
@@ -261,9 +293,17 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunn
 	var desiredReplicas int
 	fractionBusy := float64(numRunnersBusy) / float64(numRunners)
 	if fractionBusy >= scaleUpThreshold {
+		if scaleUpAdjustment > 0 {
+			desiredReplicas = numRunners + scaleUpAdjustment
+		} else {
 			desiredReplicas = int(math.Ceil(float64(numRunners) * scaleUpFactor))
+		}
 	} else if fractionBusy < scaleDownThreshold {
+		if scaleDownAdjustment > 0 {
+			desiredReplicas = numRunners - scaleDownAdjustment
+		} else {
 			desiredReplicas = int(float64(numRunners) * scaleDownFactor)
+		}
 	} else {
 		desiredReplicas = *rd.Spec.Replicas
 	}
@@ -282,6 +322,12 @@ func (r *HorizontalRunnerAutoscalerReconciler) calculateReplicasByPercentageRunn
 		"current_replicas", rd.Spec.Replicas,
 		"num_runners", numRunners,
 		"num_runners_busy", numRunnersBusy,
+		"namespace", hra.Namespace,
+		"runner_deployment", rd.Name,
+		"horizontal_runner_autoscaler", hra.Name,
+		"enterprise", enterprise,
+		"organization", organization,
+		"repository", repository,
 	)
 
 	rd.Status.Replicas = &desiredReplicas

controllers/autoscaling_test.go

@@ -47,7 +47,11 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 		min       *int
 		sReplicas *int
 		sTime     *metav1.Time
+
 		workflowRuns             string
+		workflowRuns_queued      string
+		workflowRuns_in_progress string
+
 		workflowJobs map[int]string
 		want         int
 		err          string
@@ -59,6 +63,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
 			want:                     3,
 		},
 		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
@@ -68,7 +74,9 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			max:                      intPtr(3),
 			sReplicas:                intPtr(3),
 			sTime:                    &metav1Now,
-			workflowRuns: `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     3,
 		},
 		// 3 demanded, max at 2
@@ -77,6 +85,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(2),
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
 			want:                     2,
 		},
 		// 2 demanded, min at 2
@@ -85,6 +95,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     2,
 		},
 		// 1 demanded, min at 2
@@ -93,6 +105,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
 			want:                     2,
 		},
 		// 1 demanded, min at 2
@@ -101,6 +115,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     2,
 		},
 		// 1 demanded, min at 1
@@ -109,6 +125,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(1),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
 			want:                     1,
 		},
 		// 1 demanded, min at 1
@@ -117,6 +135,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(1),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     1,
 		},
 		// fixed at 3
@@ -126,6 +146,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			max:                      intPtr(3),
 			fixed:                    intPtr(3),
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}]}"`,
 			want:                     3,
 		},
 
@@ -136,6 +158,8 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(10),
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
 			workflowJobs: map[int]string{
 				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
 				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
@@ -158,7 +182,7 @@ func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
 			server := fake.NewServer(
-				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns),
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
 				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
 				fake.WithListRunnersResponse(200, fake.RunnersListBody),
 			)
@@ -235,7 +259,11 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 		min       *int
 		sReplicas *int
 		sTime     *metav1.Time
+
 		workflowRuns             string
+		workflowRuns_queued      string
+		workflowRuns_in_progress string
+
 		workflowJobs map[int]string
 		want         int
 		err          string
@@ -247,6 +275,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
 			want:                     3,
 		},
 		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
@@ -258,6 +288,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			sReplicas:                intPtr(3),
 			sTime:                    &metav1Now,
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     3,
 		},
 		// 3 demanded, max at 2
@@ -267,6 +299,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(2),
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
 			want:                     2,
 		},
 		// 2 demanded, min at 2
@@ -276,6 +310,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     2,
 		},
 		// 1 demanded, min at 2
@@ -285,6 +321,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
 			want:                     2,
 		},
 		// 1 demanded, min at 2
@@ -294,6 +332,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     2,
 		},
 		// 1 demanded, min at 1
@@ -303,6 +343,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(1),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
 			want:                     1,
 		},
 		// 1 demanded, min at 1
@@ -312,6 +354,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(1),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			want:                     1,
 		},
 		// fixed at 3
@@ -321,7 +365,9 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			fixed:                    intPtr(1),
 			min:                      intPtr(1),
 			max:                      intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
 			want:                     3,
 		},
 		// org runner, fixed at 3
@@ -331,7 +377,9 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			fixed:                    intPtr(1),
 			min:                      intPtr(1),
 			max:                      intPtr(3),
-			workflowRuns: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
 			want:                     3,
 		},
 		// org runner, 1 demanded, min at 1, no repos
@@ -340,6 +388,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(1),
 			max:                      intPtr(3),
 			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
 			err:                      "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
 		},
 
@@ -351,6 +401,8 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 			min:                      intPtr(2),
 			max:                      intPtr(10),
 			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
 			workflowJobs: map[int]string{
 				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
 				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
@@ -373,7 +425,7 @@ func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
 
 		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
 			server := fake.NewServer(
-				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns),
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
 				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
 				fake.WithListRunnersResponse(200, fake.RunnersListBody),
 			)

controllers/horizontal_runner_autoscaler_webhook.go

@@ -349,7 +349,8 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) tryScaleUp(ctx contex
 }
 
 func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr ctrl.Manager) error {
-	autoscaler.Recorder = mgr.GetEventRecorderFor("webhookbasedautoscaler")
+	name := "webhookbasedautoscaler"
+	autoscaler.Recorder = mgr.GetEventRecorderFor(name)
 
 	if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.HorizontalRunnerAutoscaler{}, scaleTargetKey, func(rawObj runtime.Object) []string {
 		hra := rawObj.(*v1alpha1.HorizontalRunnerAutoscaler)
@@ -371,5 +372,6 @@ func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Named(name).
 		Complete(autoscaler)
 }

controllers/horizontalrunnerautoscaler_controller.go

@@ -183,10 +183,12 @@ func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(req ctrl.Request) (ctrl
 }
 
 func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller")
+	name := "horizontalrunnerautoscaler-controller"
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Named(name).
 		Complete(r)
 }
 

controllers/integration_test.go

@@ -2,11 +2,14 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"github.com/google/go-github/v33/github"
 	github3 "github.com/google/go-github/v33/github"
 	github2 "github.com/summerwind/actions-runner-controller/github"
+	"k8s.io/apimachinery/pkg/runtime"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"time"
 
 	"github.com/summerwind/actions-runner-controller/github/fake"
@@ -32,7 +35,11 @@ type testEnvironment struct {
 
 var (
 	workflowRunsFor3Replicas             = `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`
+	workflowRunsFor3Replicas_queued      = `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"queued"}]}"`
+	workflowRunsFor3Replicas_in_progress = `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`
 	workflowRunsFor1Replicas             = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"`
+	workflowRunsFor1Replicas_queued      = `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`
+	workflowRunsFor1Replicas_in_progress = `{"total_count": 0, "workflow_runs":[]}"`
 )
 
 var webhookServer *httptest.Server
@@ -56,6 +63,10 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 	responses.ListRepositoryWorkflowRuns = &fake.Handler{
 		Status: 200,
 		Body:   workflowRunsFor3Replicas,
+		Statuses: map[string]string{
+			"queued":      workflowRunsFor3Replicas_queued,
+			"in_progress": workflowRunsFor3Replicas_in_progress,
+		},
 	}
 	fakeRunnerList = fake.NewRunnersList()
 	responses.ListRunners = fakeRunnerList.HandleList()
@@ -154,7 +165,7 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 			name := "example-runnerdeploy"
 
 			{
-				rs := &actionsv1alpha1.RunnerDeployment{
+				rd := &actionsv1alpha1.RunnerDeployment{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      name,
 						Namespace: ns.Name,
@@ -174,80 +185,17 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 					},
 				}
 
-				err := k8sClient.Create(ctx, rs)
+				ExpectCreate(ctx, rd, "test RunnerDeployment")
-
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
-				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerDeployment resource")
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
-
-				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						return len(runnerSets.Items)
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						if len(runnerSets.Items) == 0 {
-							logf.Log.Info("No runnerreplicasets exist yet")
-							return -1
-						}
-
-						return *runnerSets.Items[0].Spec.Replicas
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
 			}
 
 			{
-				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+				ExpectRunnerDeploymentEventuallyUpdates(ctx, ns.Name, name, func(rd *actionsv1alpha1.RunnerDeployment) {
-				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
-				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
-				Eventually(func() error {
-					var rd actionsv1alpha1.RunnerDeployment
-
-					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
-
-					Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerDeployment resource")
-
 					rd.Spec.Replicas = intPtr(2)
-
+				})
-					return k8sClient.Update(ctx, &rd)
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
-				},
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Namespace, 2)
-					time.Second*1, time.Millisecond*500).Should(BeNil())
-
-				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						return len(runnerSets.Items)
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						return *runnerSets.Items[0].Spec.Replicas
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
 			}
 
 			// Scale-up to 3 replicas
@@ -280,38 +228,10 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 					},
 				}
 
-				err := k8sClient.Create(ctx, hra)
+				ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler")
 
-				Expect(err).NotTo(HaveOccurred(), "failed to create test HorizontalRunnerAutoscaler resource")
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
-
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3)
-				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						return len(runnerSets.Items)
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
-
-				Eventually(
-					func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						if len(runnerSets.Items) == 0 {
-							logf.Log.Info("No runnerreplicasets exist yet")
-							return -1
-						}
-
-						return *runnerSets.Items[0].Spec.Replicas
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(3))
 			}
 
 			{
@@ -342,6 +262,8 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 				time.Sleep(time.Second)
 
 				responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas
+				responses.ListRepositoryWorkflowRuns.Statuses["queued"] = workflowRunsFor1Replicas_queued
+				responses.ListRepositoryWorkflowRuns.Statuses["in_progress"] = workflowRunsFor1Replicas_in_progress
 
 				var hra actionsv1alpha1.HorizontalRunnerAutoscaler
 
@@ -357,64 +279,97 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 				Expect(err).NotTo(HaveOccurred(), "failed to get test HorizontalRunnerAutoscaler resource")
 
-				Eventually(
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1, "runners after HRA force update for scale-down")
-					func() int {
-						var runnerSets actionsv1alpha1.RunnerReplicaSetList
-
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
-						if err != nil {
-							logf.Log.Error(err, "list runner sets")
-						}
-
-						if len(runnerSets.Items) == 0 {
-							logf.Log.Info("No runnerreplicasets exist yet")
-							return -1
-						}
-
-						return *runnerSets.Items[0].Spec.Replicas
-					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runners after HRA force update for scale-down")
 			}
 
+			// Scale-up to 2 replicas on first pull_request create webhook event
 			{
+				SendPullRequestEvent("test/valid", "main", "created")
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
+			}
+
+			// Scale-up to 3 replicas on second pull_request create webhook event
+			{
+				SendPullRequestEvent("test/valid", "main", "created")
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after second webhook event")
+			}
+		})
+	})
+})
+
+func SendPullRequestEvent(repo string, branch string, action string) {
+	org := strings.Split(repo, "/")[0]
+
 	resp, err := sendWebhook(webhookServer, "pull_request", &github.PullRequestEvent{
 		PullRequest: &github.PullRequest{
 			Base: &github.PullRequestBranch{
-							Ref: github.String("main"),
+				Ref: github.String(branch),
 			},
 		},
 		Repo: &github.Repository{
-						Name: github.String("test/valid"),
+			Name: github.String(repo),
 			Organization: &github.Organization{
-							Name: github.String("test"),
+				Name: github.String(org),
 			},
 		},
-					Action: github.String("created"),
+		Action: github.String(action),
 	})
 
-				Expect(err).NotTo(HaveOccurred(), "failed to send pull_request event")
+	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to send pull_request event")
 
-				Expect(resp.StatusCode).To(Equal(200))
+	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
 }
 
-			// Scale-up to 2 replicas
+func ExpectCreate(ctx context.Context, rd runtime.Object, s string) {
-			{
+	err := k8sClient.Create(ctx, rd)
+
+	ExpectWithOffset(1, err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create %s resource", s))
+}
+
+func ExpectRunnerDeploymentEventuallyUpdates(ctx context.Context, ns string, name string, f func(rd *actionsv1alpha1.RunnerDeployment)) {
+	// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+	// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+	//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+	EventuallyWithOffset(
+		1,
+		func() error {
+			var rd actionsv1alpha1.RunnerDeployment
+
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &rd)
+
+			Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerDeployment resource")
+
+			f(&rd)
+
+			return k8sClient.Update(ctx, &rd)
+		},
+		time.Second*1, time.Millisecond*500).Should(BeNil())
+}
+
+func ExpectRunnerSetsCountEventuallyEquals(ctx context.Context, ns string, count int, optionalDescription ...interface{}) {
 	runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
 
-				Eventually(
+	EventuallyWithOffset(
+		1,
 		func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+			err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns))
 			if err != nil {
 				logf.Log.Error(err, "list runner sets")
 			}
 
 			return len(runnerSets.Items)
 		},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runner sets after webhook")
+		time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...)
+}
 
-				Eventually(
+func ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx context.Context, ns string, count int, optionalDescription ...interface{}) {
+	runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+	EventuallyWithOffset(
+		1,
 		func() int {
-						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+			err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns))
 			if err != nil {
 				logf.Log.Error(err, "list runner sets")
 			}
@@ -426,8 +381,5 @@ var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 
 			return *runnerSets.Items[0].Spec.Replicas
 		},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2), "runners after webhook")
+		time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...)
 }
-		})
-	})
-})

controllers/runner_controller.go

@@ -250,7 +250,7 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		if err != nil {
 			var e *github.RunnerNotFound
 			if errors.As(err, &e) {
-				log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub.")
+				log.V(1).Info("Failed to check if runner is busy. Either this runner has never been successfully registered to GitHub or it still needs more time.", "runnerName", runner.Name)
 
 				notRegistered = true
 			} else {
@@ -655,11 +655,14 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 }
 
 func (r *RunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("runner-controller")
+	name := "runner-controller"
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.Runner{}).
 		Owns(&corev1.Pod{}).
+		Named(name).
 		Complete(r)
 }
 

controllers/runnerdeployment_controller.go

@@ -51,6 +51,7 @@ type RunnerDeploymentReconciler struct {
 	Log                logr.Logger
 	Recorder           record.EventRecorder
 	Scheme             *runtime.Scheme
+	CommonRunnerLabels []string
 }
 
 // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
@@ -262,6 +263,10 @@ func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeplo
 	// Add template hash label to selector.
 	labels := CloneAndAddLabel(rd.Spec.Template.Labels, LabelKeyRunnerTemplateHash, templateHash)
 
+	for _, l := range r.CommonRunnerLabels {
+		newRSTemplate.Spec.Labels = append(newRSTemplate.Spec.Labels, l)
+	}
+
 	newRSTemplate.Labels = labels
 
 	rs := v1alpha1.RunnerReplicaSet{
@@ -285,7 +290,8 @@ func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeplo
 }
 
 func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("runnerdeployment-controller")
+	name := "runnerdeployment-controller"
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	if err := mgr.GetFieldIndexer().IndexField(&v1alpha1.RunnerReplicaSet{}, runnerSetOwnerKey, func(rawObj runtime.Object) []string {
 		runnerSet := rawObj.(*v1alpha1.RunnerReplicaSet)
@@ -306,5 +312,6 @@ func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.RunnerDeployment{}).
 		Owns(&v1alpha1.RunnerReplicaSet{}).
+		Named(name).
 		Complete(r)
 }

controllers/runnerdeployment_controller_test.go

@@ -2,6 +2,9 @@ package controllers
 
 import (
 	"context"
+	"github.com/google/go-cmp/cmp"
+	"k8s.io/apimachinery/pkg/runtime"
+	"testing"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -18,6 +21,40 @@ import (
 	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
 )
 
+func TestNewRunnerReplicaSet(t *testing.T) {
+	scheme := runtime.NewScheme()
+	if err := actionsv1alpha1.AddToScheme(scheme); err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	r := &RunnerDeploymentReconciler{
+		CommonRunnerLabels: []string{"dev"},
+		Scheme:             scheme,
+	}
+	rd := actionsv1alpha1.RunnerDeployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "example",
+		},
+		Spec: actionsv1alpha1.RunnerDeploymentSpec{
+			Template: actionsv1alpha1.RunnerTemplate{
+				Spec: actionsv1alpha1.RunnerSpec{
+					Labels: []string{"project1"},
+				},
+			},
+		},
+	}
+
+	rs, err := r.newRunnerReplicaSet(rd)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	want := []string{"project1", "dev"}
+	if d := cmp.Diff(want, rs.Spec.Template.Spec.Labels); d != "" {
+		t.Errorf("%s", d)
+	}
+}
+
 // SetupDeploymentTest will set up a testing environment.
 // This includes:
 // * creating a Namespace to be used during the test

controllers/runnerreplicaset_controller.go

@@ -111,7 +111,7 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 
 				var e *github.RunnerNotFound
 				if errors.As(err, &e) {
-					log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub, and therefore we prioritize it for deletion", "runnerName", runner.Name)
+					log.V(1).Info("Failed to check if runner is busy. Either this runner has never been successfully registered to GitHub or has not managed yet to, and therefore we prioritize it for deletion", "runnerName", runner.Name)
 					notRegistered = true
 				} else {
 					var e *gogithub.RateLimitError
@@ -221,10 +221,12 @@ func (r *RunnerReplicaSetReconciler) newRunner(rs v1alpha1.RunnerReplicaSet) (v1
 }
 
 func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	r.Recorder = mgr.GetEventRecorderFor("runnerreplicaset-controller")
+	name := "runnerreplicaset-controller"
+	r.Recorder = mgr.GetEventRecorderFor(name)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.RunnerReplicaSet{}).
 		Owns(&v1alpha1.Runner{}).
+		Named(name).
 		Complete(r)
 }

github/fake/fake.go

@@ -37,10 +37,21 @@ func (h *ListRunnersHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
 type Handler struct {
 	Status int
 	Body   string
+
+	Statuses map[string]string
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	w.WriteHeader(h.Status)
+
+	status := req.URL.Query().Get("status")
+	if h.Statuses != nil {
+		if body, ok := h.Statuses[status]; ok {
+			fmt.Fprintf(w, body)
+			return
+		}
+	}
+
 	fmt.Fprintf(w, h.Body)
 }
 
@@ -102,6 +113,18 @@ func NewServer(opts ...Option) *httptest.Server {
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
+		"/enterprises/test/actions/runners/registration-token": &Handler{
+			Status: http.StatusCreated,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/enterprises/invalid/actions/runners/registration-token": &Handler{
+			Status: http.StatusOK,
+			Body:   fmt.Sprintf("{\"token\": \"%s\", \"expires_at\": \"%s\"}", RegistrationToken, time.Now().Add(time.Hour*1).Format(time.RFC3339)),
+		},
+		"/enterprises/error/actions/runners/registration-token": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
 
 		// For ListRunners
 		"/repos/test/valid/actions/runners": config.FixedResponses.ListRunners,
@@ -125,6 +148,18 @@ func NewServer(opts ...Option) *httptest.Server {
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
+		"/enterprises/test/actions/runners": &Handler{
+			Status: http.StatusOK,
+			Body:   RunnersListBody,
+		},
+		"/enterprises/invalid/actions/runners": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/enterprises/error/actions/runners": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
 
 		// For RemoveRunner
 		"/repos/test/valid/actions/runners/1": &Handler{
@@ -151,6 +186,18 @@ func NewServer(opts ...Option) *httptest.Server {
 			Status: http.StatusBadRequest,
 			Body:   "",
 		},
+		"/enterprises/test/actions/runners/1": &Handler{
+			Status: http.StatusNoContent,
+			Body:   "",
+		},
+		"/enterprises/invalid/actions/runners/1": &Handler{
+			Status: http.StatusOK,
+			Body:   "",
+		},
+		"/enterprises/error/actions/runners/1": &Handler{
+			Status: http.StatusBadRequest,
+			Body:   "",
+		},
 
 		// For auto-scaling based on the number of queued(pending) workflow runs
 		"/repos/test/valid/actions/runs": config.FixedResponses.ListRepositoryWorkflowRuns,

github/fake/options.go

@@ -10,11 +10,15 @@ type FixedResponses struct {
 
 type Option func(*ServerConfig)
 
-func WithListRepositoryWorkflowRunsResponse(status int, body string) Option {
+func WithListRepositoryWorkflowRunsResponse(status int, body, queued, in_progress string) Option {
 	return func(c *ServerConfig) {
 		c.FixedResponses.ListRepositoryWorkflowRuns = &Handler{
 			Status: status,
 			Body:   body,
+			Statuses: map[string]string{
+				"queued":      queued,
+				"in_progress": in_progress,
+			},
 		}
 	}
 }

github/github.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/bradleyfalzon/ghinstallation"
 	"github.com/google/go-github/v33/github"
+	"github.com/summerwind/actions-runner-controller/github/metrics"
 	"golang.org/x/oauth2"
 )
 
@@ -34,15 +35,9 @@ type Client struct {
 
 // NewClient creates a Github Client
 func (c *Config) NewClient() (*Client, error) {
-	var (
+	var transport http.RoundTripper
-		httpClient *http.Client
-		client     *github.Client
-	)
-	githubBaseURL := "https://github.com/"
 	if len(c.Token) > 0 {
-		httpClient = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(
+		transport = oauth2.NewClient(context.Background(), oauth2.StaticTokenSource(&oauth2.Token{AccessToken: c.Token})).Transport
-			&oauth2.Token{AccessToken: c.Token},
-		))
 	} else {
 		tr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.AppID, c.AppInstallationID, c.AppPrivateKey)
 		if err != nil {
@@ -55,9 +50,13 @@ func (c *Config) NewClient() (*Client, error) {
 			}
 			tr.BaseURL = githubAPIURL
 		}
-		httpClient = &http.Client{Transport: tr}
+		transport = tr
 	}
+	transport = metrics.Transport{Transport: transport}
+	httpClient := &http.Client{Transport: transport}
 
+	var client *github.Client
+	var githubBaseURL string
 	if len(c.EnterpriseURL) > 0 {
 		var err error
 		client, err = github.NewEnterpriseClient(c.EnterpriseURL, c.EnterpriseURL, httpClient)
@@ -67,6 +66,7 @@ func (c *Config) NewClient() (*Client, error) {
 		githubBaseURL = fmt.Sprintf("%s://%s%s", client.BaseURL.Scheme, client.BaseURL.Host, strings.TrimSuffix(client.BaseURL.Path, "api/v3/"))
 	} else {
 		client = github.NewClient(httpClient)
+		githubBaseURL = "https://github.com/"
 	}
 
 	return &Client{
@@ -82,7 +82,7 @@ func (c *Client) GetRegistrationToken(ctx context.Context, enterprise, org, repo
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	key := getRegistrationKey(org, repo)
+	key := getRegistrationKey(org, repo, enterprise)
 	rt, ok := c.regTokens[key]
 
 	if ok && rt.GetExpiresAt().After(time.Now()) {
@@ -210,12 +210,34 @@ func (c *Client) listRunners(ctx context.Context, enterprise, org, repo string,
 func (c *Client) ListRepositoryWorkflowRuns(ctx context.Context, user string, repoName string) ([]*github.WorkflowRun, error) {
 	c.Client.Actions.ListRepositoryWorkflowRuns(ctx, user, repoName, nil)
 
+	queued, err := c.listRepositoryWorkflowRuns(ctx, user, repoName, "queued")
+	if err != nil {
+		return nil, fmt.Errorf("listing queued workflow runs: %w", err)
+	}
+
+	inProgress, err := c.listRepositoryWorkflowRuns(ctx, user, repoName, "in_progress")
+	if err != nil {
+		return nil, fmt.Errorf("listing in_progress workflow runs: %w", err)
+	}
+
+	var workflowRuns []*github.WorkflowRun
+
+	workflowRuns = append(workflowRuns, queued...)
+	workflowRuns = append(workflowRuns, inProgress...)
+
+	return workflowRuns, nil
+}
+
+func (c *Client) listRepositoryWorkflowRuns(ctx context.Context, user string, repoName, status string) ([]*github.WorkflowRun, error) {
+	c.Client.Actions.ListRepositoryWorkflowRuns(ctx, user, repoName, nil)
+
 	var workflowRuns []*github.WorkflowRun
 
 	opts := github.ListWorkflowRunsOptions{
 		ListOptions: github.ListOptions{
 			PerPage: 100,
 		},
+		Status: status,
 	}
 
 	for {
@@ -250,11 +272,8 @@ func getEnterpriseOrganisationAndRepo(enterprise, org, repo string) (string, str
 	return "", "", "", fmt.Errorf("enterprise, organization and repository are all empty")
 }
 
-func getRegistrationKey(org, repo string) string {
+func getRegistrationKey(org, repo, enterprise string) string {
-	if len(org) > 0 {
+	return fmt.Sprintf("org=%s,repo=%s,enterprise=%s", org, repo, enterprise)
-		return org
-	}
-	return repo
 }
 
 func splitOwnerAndRepo(repo string) (string, string, error) {

github/metrics/transport.go

@@ -0,0 +1,63 @@
+// Package metrics provides monitoring of the GitHub related metrics.
+//
+// This depends on the metrics exporter of kubebuilder.
+// See https://book.kubebuilder.io/reference/metrics.html for details.
+package metrics
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+func init() {
+	metrics.Registry.MustRegister(metricRateLimit, metricRateLimitRemaining)
+}
+
+var (
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+	metricRateLimit = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit",
+			Help: "The maximum number of requests you're permitted to make per hour",
+		},
+	)
+	metricRateLimitRemaining = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "github_rate_limit_remaining",
+			Help: "The number of requests remaining in the current rate limit window",
+		},
+	)
+)
+
+const (
+	// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
+	headerRateLimit          = "X-RateLimit-Limit"
+	headerRateLimitRemaining = "X-RateLimit-Remaining"
+)
+
+// Transport wraps a transport with metrics monitoring
+type Transport struct {
+	Transport http.RoundTripper
+}
+
+func (t Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := t.Transport.RoundTrip(req)
+	if resp != nil {
+		parseResponse(resp)
+	}
+	return resp, err
+}
+
+func parseResponse(resp *http.Response) {
+	rateLimit, err := strconv.Atoi(resp.Header.Get(headerRateLimit))
+	if err == nil {
+		metricRateLimit.Set(float64(rateLimit))
+	}
+	rateLimitRemaining, err := strconv.Atoi(resp.Header.Get(headerRateLimitRemaining))
+	if err == nil {
+		metricRateLimitRemaining.Set(float64(rateLimitRemaining))
+	}
+}

go.mod

@@ -6,11 +6,13 @@ require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-logr/logr v0.1.0
+	github.com/google/go-cmp v0.3.1
 	github.com/google/go-github/v33 v33.0.1-0.20210204004227-319dcffb518a
 	github.com/gorilla/mux v1.8.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.5.0
+	github.com/prometheus/client_golang v0.9.2
 	github.com/stretchr/testify v1.4.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f

main.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/kelseyhightower/envconfig"
@@ -62,6 +63,8 @@ func main() {
 
 		runnerImage string
 		dockerImage string
+
+		commonRunnerLabels commaSeparatedStringSlice
 	)
 
 	var c github.Config
@@ -80,6 +83,7 @@ func main() {
 	flag.Int64Var(&c.AppInstallationID, "github-app-installation-id", c.AppInstallationID, "The installation ID of GitHub App.")
 	flag.StringVar(&c.AppPrivateKey, "github-app-private-key", c.AppPrivateKey, "The path of a private key file to authenticate as a GitHub App")
 	flag.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, "Determines the minimum frequency at which K8s resources managed by this controller are reconciled. When you use autoscaling, set to a lower value like 10 minute, because this corresponds to the minimum time to react on demand change")
+	flag.Var(&commonRunnerLabels, "common-runner-labels", "Runner labels in the K1=V1,K2=V2,... format that are inherited all the runners created by the controller. See https://github.com/summerwind/actions-runner-controller/issues/321 for more information")
 	flag.Parse()
 
 	logger := zap.New(func(o *zap.Options) {
@@ -136,6 +140,7 @@ func main() {
 		Client:             mgr.GetClient(),
 		Log:                ctrl.Log.WithName("controllers").WithName("RunnerDeployment"),
 		Scheme:             mgr.GetScheme(),
+		CommonRunnerLabels: commonRunnerLabels,
 	}
 
 	if err = runnerDeploymentReconciler.SetupWithManager(mgr); err != nil {
@@ -176,3 +181,20 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+type commaSeparatedStringSlice []string
+
+func (s *commaSeparatedStringSlice) String() string {
+	return fmt.Sprintf("%v", *s)
+}
+
+func (s *commaSeparatedStringSlice) Set(value string) error {
+	for _, v := range strings.Split(value, ",") {
+		if v == "" {
+			continue
+		}
+
+		*s = append(*s, v)
+	}
+	return nil
+}