Changes to folder structure to allow multigroups and changed go mod name (#2105)

* Changed folder structure to allow multi group registration

* included actions.github.com directory for resources and controllers

* updated go module to actions/actions-runner-controller

* publish arc packages under actions-runner-controller

* Update charts/actions-runner-controller/docs/UPGRADING.md

Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>

controllers/actions.summerwind.net/autoscaling.go

@@ -0,0 +1,432 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	prometheus_metrics "github.com/actions/actions-runner-controller/controllers/actions.summerwind.net/metrics"
+	arcgithub "github.com/actions/actions-runner-controller/github"
+	"github.com/google/go-github/v47/github"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	defaultScaleUpThreshold   = 0.8
+	defaultScaleDownThreshold = 0.3
+	defaultScaleUpFactor      = 1.3
+	defaultScaleDownFactor    = 0.7
+)
+
+func (r *HorizontalRunnerAutoscalerReconciler) suggestDesiredReplicas(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, error) {
+	if hra.Spec.MinReplicas == nil {
+		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing minReplicas", hra.Namespace, hra.Name)
+	} else if hra.Spec.MaxReplicas == nil {
+		return nil, fmt.Errorf("horizontalrunnerautoscaler %s/%s is missing maxReplicas", hra.Namespace, hra.Name)
+	}
+
+	metrics := hra.Spec.Metrics
+	numMetrics := len(metrics)
+	if numMetrics == 0 {
+		// We don't default to anything since ARC 0.23.0
+		// See https://github.com/actions/actions-runner-controller/issues/728
+		return nil, nil
+	} else if numMetrics > 2 {
+		return nil, fmt.Errorf("too many autoscaling metrics configured: It must be 0 to 2, but got %d", numMetrics)
+	}
+
+	primaryMetric := metrics[0]
+	primaryMetricType := primaryMetric.Type
+
+	var (
+		suggested *int
+		err       error
+	)
+
+	switch primaryMetricType {
+	case v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns:
+		suggested, err = r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &primaryMetric)
+	case v1alpha1.AutoscalingMetricTypePercentageRunnersBusy:
+		suggested, err = r.suggestReplicasByPercentageRunnersBusy(ghc, st, hra, primaryMetric)
+	default:
+		return nil, fmt.Errorf("validating autoscaling metrics: unsupported metric type %q", primaryMetric)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	if suggested != nil && *suggested > 0 {
+		return suggested, nil
+	}
+
+	if len(metrics) == 1 {
+		// This is never supposed to happen but anyway-
+		// Fall-back to `minReplicas + capacityReservedThroughWebhook`.
+		return nil, nil
+	}
+
+	// At this point, we are sure that there are exactly 2 Metrics entries.
+
+	fallbackMetric := metrics[1]
+	fallbackMetricType := fallbackMetric.Type
+
+	if primaryMetricType != v1alpha1.AutoscalingMetricTypePercentageRunnersBusy ||
+		fallbackMetricType != v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns {
+
+		return nil, fmt.Errorf(
+			"invalid HRA Spec: Metrics[0] of %s cannot be combined with Metrics[1] of %s: The only allowed combination is 0=PercentageRunnersBusy and 1=TotalNumberOfQueuedAndInProgressWorkflowRuns",
+			primaryMetricType, fallbackMetricType,
+		)
+	}
+
+	return r.suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc, st, hra, &fallbackMetric)
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByQueuedAndInProgressWorkflowRuns(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics *v1alpha1.MetricSpec) (*int, error) {
+	var repos [][]string
+	repoID := st.repo
+	if repoID == "" {
+		orgName := st.org
+		if orgName == "" {
+			return nil, fmt.Errorf("asserting runner deployment spec to detect bug: spec.template.organization should not be empty on this code path")
+		}
+
+		// In case it's an organizational runners deployment without any scaling metrics defined,
+		// we assume that the desired replicas should always be `minReplicas + capacityReservedThroughWebhook`.
+		// See https://github.com/actions/actions-runner-controller/issues/377#issuecomment-793372693
+		if metrics == nil {
+			return nil, nil
+		}
+
+		if len(metrics.RepositoryNames) == 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment")
+		}
+
+		for _, repoName := range metrics.RepositoryNames {
+			repos = append(repos, []string{orgName, repoName})
+		}
+	} else {
+		repo := strings.Split(repoID, "/")
+
+		repos = append(repos, repo)
+	}
+
+	var total, inProgress, queued, completed, unknown int
+	type callback func()
+	listWorkflowJobs := func(user string, repoName string, runID int64, fallback_cb callback) {
+		if runID == 0 {
+			fallback_cb()
+			return
+		}
+		opt := github.ListWorkflowJobsOptions{ListOptions: github.ListOptions{PerPage: 50}}
+		var allJobs []*github.WorkflowJob
+		for {
+			jobs, resp, err := ghc.Actions.ListWorkflowJobs(context.TODO(), user, repoName, runID, &opt)
+			if err != nil {
+				r.Log.Error(err, "Error listing workflow jobs")
+				return //err
+			}
+			allJobs = append(allJobs, jobs.Jobs...)
+			if resp.NextPage == 0 {
+				break
+			}
+			opt.Page = resp.NextPage
+		}
+		if len(allJobs) == 0 {
+			fallback_cb()
+		} else {
+		JOB:
+			for _, job := range allJobs {
+				runnerLabels := make(map[string]struct{}, len(st.labels))
+				for _, l := range st.labels {
+					runnerLabels[l] = struct{}{}
+				}
+
+				if len(job.Labels) == 0 {
+					// This shouldn't usually happen
+					r.Log.Info("Detected job with no labels, which is not supported by ARC. Skipping anyway.", "labels", job.Labels, "run_id", job.GetRunID(), "job_id", job.GetID())
+					continue JOB
+				}
+
+				for _, l := range job.Labels {
+					if l == "self-hosted" {
+						continue
+					}
+
+					if _, ok := runnerLabels[l]; !ok {
+						continue JOB
+					}
+				}
+
+				switch job.GetStatus() {
+				case "completed":
+					// We add a case for `completed` so it is not counted in `unknown`.
+					// And we do not increment the counter for completed because
+					// that counter only refers to workflows. The reason for
+					// this is because we do not get a list of jobs for
+					// completed workflows in order to keep the number of API
+					// calls to a minimum.
+				case "in_progress":
+					inProgress++
+				case "queued":
+					queued++
+				default:
+					unknown++
+				}
+			}
+		}
+	}
+
+	for _, repo := range repos {
+		user, repoName := repo[0], repo[1]
+		workflowRuns, err := ghc.ListRepositoryWorkflowRuns(context.TODO(), user, repoName)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, run := range workflowRuns {
+			total++
+
+			// In May 2020, there are only 3 statuses.
+			// Follow the below links for more details:
+			// - https://developer.github.com/v3/actions/workflow-runs/#list-repository-workflow-runs
+			// - https://developer.github.com/v3/checks/runs/#create-a-check-run
+			switch run.GetStatus() {
+			case "completed":
+				completed++
+			case "in_progress":
+				listWorkflowJobs(user, repoName, run.GetID(), func() { inProgress++ })
+			case "queued":
+				listWorkflowJobs(user, repoName, run.GetID(), func() { queued++ })
+			default:
+				unknown++
+			}
+		}
+	}
+
+	necessaryReplicas := queued + inProgress
+
+	prometheus_metrics.SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		necessaryReplicas,
+		completed,
+		inProgress,
+		queued,
+		unknown,
+	)
+
+	r.Log.V(1).Info(
+		fmt.Sprintf("Suggested desired replicas of %d by TotalNumberOfQueuedAndInProgressWorkflowRuns", necessaryReplicas),
+		"workflow_runs_completed", completed,
+		"workflow_runs_in_progress", inProgress,
+		"workflow_runs_queued", queued,
+		"workflow_runs_unknown", unknown,
+		"namespace", hra.Namespace,
+		"kind", st.kind,
+		"name", st.st,
+		"horizontal_runner_autoscaler", hra.Name,
+	)
+
+	return &necessaryReplicas, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) suggestReplicasByPercentageRunnersBusy(ghc *arcgithub.Client, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, metrics v1alpha1.MetricSpec) (*int, error) {
+	ctx := context.Background()
+	scaleUpThreshold := defaultScaleUpThreshold
+	scaleDownThreshold := defaultScaleDownThreshold
+	scaleUpFactor := defaultScaleUpFactor
+	scaleDownFactor := defaultScaleDownFactor
+
+	if metrics.ScaleUpThreshold != "" {
+		sut, err := strconv.ParseFloat(metrics.ScaleUpThreshold, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpThreshold cannot be parsed into a float64")
+		}
+		scaleUpThreshold = sut
+	}
+	if metrics.ScaleDownThreshold != "" {
+		sdt, err := strconv.ParseFloat(metrics.ScaleDownThreshold, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownThreshold cannot be parsed into a float64")
+		}
+
+		scaleDownThreshold = sdt
+	}
+
+	scaleUpAdjustment := metrics.ScaleUpAdjustment
+	if scaleUpAdjustment != 0 {
+		if metrics.ScaleUpAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpAdjustment cannot be lower than 0")
+		}
+
+		if metrics.ScaleUpFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleUpAdjustment and scaleUpFactor cannot be specified together")
+		}
+	} else if metrics.ScaleUpFactor != "" {
+		suf, err := strconv.ParseFloat(metrics.ScaleUpFactor, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleUpFactor cannot be parsed into a float64")
+		}
+		scaleUpFactor = suf
+	}
+
+	scaleDownAdjustment := metrics.ScaleDownAdjustment
+	if scaleDownAdjustment != 0 {
+		if metrics.ScaleDownAdjustment < 0 {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownAdjustment cannot be lower than 0")
+		}
+
+		if metrics.ScaleDownFactor != "" {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[]: scaleDownAdjustment and scaleDownFactor cannot be specified together")
+		}
+	} else if metrics.ScaleDownFactor != "" {
+		sdf, err := strconv.ParseFloat(metrics.ScaleDownFactor, 64)
+		if err != nil {
+			return nil, errors.New("validating autoscaling metrics: spec.autoscaling.metrics[].scaleDownFactor cannot be parsed into a float64")
+		}
+		scaleDownFactor = sdf
+	}
+
+	runnerMap, err := st.getRunnerMap()
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		enterprise   = st.enterprise
+		organization = st.org
+		repository   = st.repo
+	)
+
+	// ListRunners will return all runners managed by GitHub - not restricted to ns
+	runners, err := ghc.ListRunners(
+		ctx,
+		enterprise,
+		organization,
+		repository)
+	if err != nil {
+		return nil, err
+	}
+
+	var desiredReplicasBefore int
+
+	if v := st.replicas; v == nil {
+		desiredReplicasBefore = 1
+	} else {
+		desiredReplicasBefore = *v
+	}
+
+	var (
+		numRunners           int
+		numRunnersRegistered int
+		numRunnersBusy       int
+		numTerminatingBusy   int
+	)
+
+	numRunners = len(runnerMap)
+
+	busyTerminatingRunnerPods := map[string]struct{}{}
+
+	kindLabel := LabelKeyRunnerDeploymentName
+	if hra.Spec.ScaleTargetRef.Kind == "RunnerSet" {
+		kindLabel = LabelKeyRunnerSetName
+	}
+
+	var runnerPodList corev1.PodList
+	if err := r.Client.List(ctx, &runnerPodList, client.InNamespace(hra.Namespace), client.MatchingLabels(map[string]string{
+		kindLabel: hra.Spec.ScaleTargetRef.Name,
+	})); err != nil {
+		return nil, err
+	}
+
+	for _, p := range runnerPodList.Items {
+		if p.Annotations[AnnotationKeyUnregistrationFailureMessage] != "" {
+			busyTerminatingRunnerPods[p.Name] = struct{}{}
+		}
+	}
+
+	for _, runner := range runners {
+		if _, ok := runnerMap[*runner.Name]; ok {
+			numRunnersRegistered++
+
+			if runner.GetBusy() {
+				numRunnersBusy++
+			} else if _, ok := busyTerminatingRunnerPods[*runner.Name]; ok {
+				numTerminatingBusy++
+			}
+
+			delete(busyTerminatingRunnerPods, *runner.Name)
+		}
+	}
+
+	// Remaining busyTerminatingRunnerPods are runners that were not on the ListRunners API response yet
+	for range busyTerminatingRunnerPods {
+		numTerminatingBusy++
+	}
+
+	var desiredReplicas int
+	fractionBusy := float64(numRunnersBusy+numTerminatingBusy) / float64(desiredReplicasBefore)
+	if fractionBusy >= scaleUpThreshold {
+		if scaleUpAdjustment > 0 {
+			desiredReplicas = desiredReplicasBefore + scaleUpAdjustment
+		} else {
+			desiredReplicas = int(math.Ceil(float64(desiredReplicasBefore) * scaleUpFactor))
+		}
+	} else if fractionBusy < scaleDownThreshold {
+		if scaleDownAdjustment > 0 {
+			desiredReplicas = desiredReplicasBefore - scaleDownAdjustment
+		} else {
+			desiredReplicas = int(float64(desiredReplicasBefore) * scaleDownFactor)
+		}
+	} else {
+		desiredReplicas = *st.replicas
+	}
+
+	// NOTES for operators:
+	//
+	// - num_runners can be as twice as large as replicas_desired_before while
+	//   the runnerdeployment controller is replacing RunnerReplicaSet for runner update.
+	prometheus_metrics.SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+		hra.ObjectMeta,
+		st.enterprise,
+		st.org,
+		st.repo,
+		st.kind,
+		st.st,
+		desiredReplicas,
+		numRunners,
+		numRunnersRegistered,
+		numRunnersBusy,
+		numTerminatingBusy,
+	)
+
+	r.Log.V(1).Info(
+		fmt.Sprintf("Suggested desired replicas of %d by PercentageRunnersBusy", desiredReplicas),
+		"replicas_desired_before", desiredReplicasBefore,
+		"replicas_desired", desiredReplicas,
+		"num_runners", numRunners,
+		"num_runners_registered", numRunnersRegistered,
+		"num_runners_busy", numRunnersBusy,
+		"num_terminating_busy", numTerminatingBusy,
+		"namespace", hra.Namespace,
+		"kind", st.kind,
+		"name", st.st,
+		"horizontal_runner_autoscaler", hra.Name,
+		"enterprise", enterprise,
+		"organization", organization,
+		"repository", repository,
+	)
+
+	return &desiredReplicas, nil
+}

controllers/actions.summerwind.net/autoscaling_test.go

@@ -0,0 +1,797 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/github"
+	"github.com/actions/actions-runner-controller/github/fake"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+func newGithubClient(server *httptest.Server) *github.Client {
+	c := github.Config{
+		Token: "token",
+	}
+	client, err := c.NewClient()
+	if err != nil {
+		panic(err)
+	}
+
+	baseURL, err := url.Parse(server.URL + "/")
+	if err != nil {
+		panic(err)
+	}
+	client.Client.BaseURL = baseURL
+
+	return client
+}
+
+func TestDetermineDesiredReplicas_RepositoryRunner(t *testing.T) {
+	intPtr := func(v int) *int {
+		return &v
+	}
+
+	metav1Now := metav1.Now()
+	testcases := []struct {
+		description string
+
+		repo   string
+		org    string
+		labels []string
+
+		fixed     *int
+		max       *int
+		min       *int
+		sReplicas *int
+		sTime     *metav1.Time
+
+		workflowRuns             string
+		workflowRuns_queued      string
+		workflowRuns_in_progress string
+
+		workflowJobs map[int]string
+		want         int
+		err          string
+	}{
+		// Legacy functionality
+		// 3 demanded, max at 3
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+		// Explicitly speified the default `self-hosted` label which is ignored by the simulator,
+		// as we assume that GitHub Actions automatically associates the `self-hosted` label to every self-hosted runner.
+		// 3 demanded, max at 3
+		{
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			sReplicas:                intPtr(3),
+			sTime:                    &metav1Now,
+			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+		// 3 demanded, max at 2
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(2),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     2,
+		},
+		// 2 demanded, min at 2
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
+		},
+		// 1 demanded, min at 2
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     2,
+		},
+		// 1 demanded, min at 2
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
+		},
+		// 1 demanded, min at 1
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     1,
+		},
+		// 1 demanded, min at 1
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     1,
+		},
+		// fixed at 3
+		{
+			repo:                     "test/valid",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			fixed:                    intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+
+		{
+			description:              "Job-level autoscaling with no explicit runner label (runners have implicit self-hosted, requested self-hosted, 5 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"completed", "labels":["self-hosted"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with no explicit runner label (runners have implicit self-hosted, requested self-hosted+custom, 0 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with no label (runners have implicit self-hosted, jobs had no labels, 0 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with default runner label (runners have self-hosted only, requested self-hosted+custom, 0 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with custom runner label (runners have custom2, requested self-hosted+custom, 0 jobs from 5 workflows",
+			repo:                     "test/valid",
+			labels:                   []string{"custom2"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling with default runner label (runners have self-hosted, requested managed-runner-label, 0 jobs from 3 runs)",
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["managed-runner-label"]}, {"status":"queued", "labels":["managed-runner-label"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["managed-runner-label"]}, {"status":"completed", "labels":["managed-runner-label"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["managed-runner-label"]}, {"status":"queued", "labels":["managed-runner-label"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Job-level autoscaling with default + custom runner label (runners have self-hosted+custom, requested self-hosted+custom, 5 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			labels:                   []string{"self-hosted", "custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Job-level autoscaling with custom runner label (runners have custom, requested self-hosted+custom, 5 jobs from 3 workflows)",
+			repo:                     "test/valid",
+			labels:                   []string{"custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 5,
+		},
+	}
+
+	for i := range testcases {
+		tc := testcases[i]
+
+		log := zap.New(func(o *zap.Options) {
+			o.Development = true
+		})
+
+		scheme := runtime.NewScheme()
+		_ = clientgoscheme.AddToScheme(scheme)
+		_ = v1alpha1.AddToScheme(scheme)
+
+		testName := fmt.Sprintf("case %d", i)
+		if tc.description != "" {
+			testName = tc.description
+		}
+
+		t.Run(testName, func(t *testing.T) {
+			server := fake.NewServer(
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
+				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
+				fake.WithListRunnersResponse(200, fake.RunnersListBody),
+			)
+			defer server.Close()
+			client := newGithubClient(server)
+
+			h := &HorizontalRunnerAutoscalerReconciler{
+				Log:                   log,
+				Scheme:                scheme,
+				DefaultScaleDownDelay: DefaultScaleDownDelay,
+			}
+
+			rd := v1alpha1.RunnerDeployment{
+				TypeMeta: metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testrd",
+				},
+				Spec: v1alpha1.RunnerDeploymentSpec{
+					Template: v1alpha1.RunnerTemplate{
+						Spec: v1alpha1.RunnerSpec{
+							RunnerConfig: v1alpha1.RunnerConfig{
+								Repository: tc.repo,
+								Labels:     tc.labels,
+							},
+						},
+					},
+					Replicas: tc.fixed,
+				},
+				Status: v1alpha1.RunnerDeploymentStatus{
+					DesiredReplicas: tc.sReplicas,
+				},
+			}
+
+			hra := v1alpha1.HorizontalRunnerAutoscaler{
+				Spec: v1alpha1.HorizontalRunnerAutoscalerSpec{
+					MaxReplicas: tc.max,
+					MinReplicas: tc.min,
+					Metrics: []v1alpha1.MetricSpec{
+						{
+							Type: "TotalNumberOfQueuedAndInProgressWorkflowRuns",
+						},
+					},
+				},
+				Status: v1alpha1.HorizontalRunnerAutoscalerStatus{
+					DesiredReplicas:            tc.sReplicas,
+					LastSuccessfulScaleOutTime: tc.sTime,
+				},
+			}
+
+			minReplicas, _, _, err := h.getMinReplicas(log, metav1Now.Time, hra)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			st := h.scaleTargetFromRD(context.Background(), rd)
+
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
+			if err != nil {
+				if tc.err == "" {
+					t.Fatalf("unexpected error: expected none, got %v", err)
+				} else if err.Error() != tc.err {
+					t.Fatalf("unexpected error: expected %v, got %v", tc.err, err)
+				}
+				return
+			}
+
+			if got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, got)
+			}
+		})
+	}
+}
+
+func TestDetermineDesiredReplicas_OrganizationalRunner(t *testing.T) {
+	intPtr := func(v int) *int {
+		return &v
+	}
+
+	metav1Now := metav1.Now()
+	testcases := []struct {
+		description string
+
+		repos  []string
+		org    string
+		labels []string
+
+		fixed     *int
+		max       *int
+		min       *int
+		sReplicas *int
+		sTime     *metav1.Time
+
+		workflowRuns             string
+		workflowRuns_queued      string
+		workflowRuns_in_progress string
+
+		workflowJobs map[int]string
+		want         int
+		err          string
+	}{
+		// 3 demanded, max at 3
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+		// 2 demanded, max at 3, currently 3, delay scaling down due to grace period
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			sReplicas:                intPtr(3),
+			sTime:                    &metav1Now,
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+		// 3 demanded, max at 2
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(2),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}]}"`,
+			want:                     2,
+		},
+		// 2 demanded, min at 2
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 3, "workflow_runs":[{"status":"queued"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
+		},
+		// 1 demanded, min at 2
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     2,
+		},
+		// 1 demanded, min at 2
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     2,
+		},
+		// 1 demanded, min at 1
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 0, "workflow_runs":[]}"`,
+			want:                     1,
+		},
+		// 1 demanded, min at 1
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			want:                     1,
+		},
+		// fixed at 3
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			fixed:                    intPtr(1),
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+		// org runner, fixed at 3
+		{
+			org:                      "test",
+			repos:                    []string{"valid"},
+			fixed:                    intPtr(1),
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"status":"in_progress"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 3, "workflow_runs":[{"status":"in_progress"},{"status":"in_progress"},{"status":"in_progress"}]}"`,
+			want:                     3,
+		},
+		// org runner, 1 demanded, min at 1, no repos
+		{
+			org:                      "test",
+			min:                      intPtr(1),
+			max:                      intPtr(3),
+			workflowRuns:             `{"total_count": 2, "workflow_runs":[{"status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 0, "workflow_runs":[]}"`,
+			workflowRuns_in_progress: `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`,
+			err:                      "validating autoscaling metrics: spec.autoscaling.metrics[].repositoryNames is required and must have one more more entries for organizational runner deployment",
+		},
+
+		{
+			description:              "Job-level autoscaling (runners have implicit self-hosted, requested self-hosted, 5 jobs from 3 runs)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"completed", "labels":["self-hosted"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Job-level autoscaling (runners have explicit self-hosted, requested self-hosted, 5 jobs from 3 runs)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"completed", "labels":["self-hosted"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted"]}, {"status":"queued", "labels":["self-hosted"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (jobs lack labels, 0 requested from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued"}, {"status":"queued"}]}`,
+				2: `{"jobs": [{"status": "in_progress"}, {"status":"completed"}]}`,
+				3: `{"jobs": [{"status": "in_progress"}, {"status":"queued"}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (runners have valid and implicit self-hosted, requested self-hosted+custom, 0 jobs from 3 runs)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (runners have self-hosted, requested self-hosted+custom, 0 jobs from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"self-hosted"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+
+		{
+			description:              "Job-level autoscaling (runners have custom, requested self-hosted+custom, 5 requested from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Job-level autoscaling (runners have custom, requested custom, 5 requested from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"custom"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["custom"]}, {"status":"queued", "labels":["custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["custom"]}, {"status":"completed", "labels":["custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["custom"]}, {"status":"queued", "labels":["custom"]}]}`,
+			},
+			want: 5,
+		},
+
+		{
+			description:              "Skipped job-level autoscaling (specified custom2, 0 requested from 3 workflows)",
+			org:                      "test",
+			repos:                    []string{"valid"},
+			labels:                   []string{"custom2"},
+			min:                      intPtr(2),
+			max:                      intPtr(10),
+			workflowRuns:             `{"total_count": 4, "workflow_runs":[{"id": 1, "status":"queued"}, {"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowRuns_queued:      `{"total_count": 1, "workflow_runs":[{"id": 1, "status":"queued"}]}"`,
+			workflowRuns_in_progress: `{"total_count": 2, "workflow_runs":[{"id": 2, "status":"in_progress"}, {"id": 3, "status":"in_progress"}, {"status":"completed"}]}"`,
+			workflowJobs: map[int]string{
+				1: `{"jobs": [{"status":"queued", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+				2: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"completed", "labels":["self-hosted", "custom"]}]}`,
+				3: `{"jobs": [{"status": "in_progress", "labels":["self-hosted", "custom"]}, {"status":"queued", "labels":["self-hosted", "custom"]}]}`,
+			},
+			want: 2,
+		},
+	}
+
+	for i := range testcases {
+		tc := testcases[i]
+
+		log := zap.New(func(o *zap.Options) {
+			o.Development = true
+		})
+
+		scheme := runtime.NewScheme()
+		_ = clientgoscheme.AddToScheme(scheme)
+		_ = v1alpha1.AddToScheme(scheme)
+
+		testName := fmt.Sprintf("case %d", i)
+		if tc.description != "" {
+			testName = tc.description
+		}
+
+		t.Run(testName, func(t *testing.T) {
+			t.Helper()
+
+			server := fake.NewServer(
+				fake.WithListRepositoryWorkflowRunsResponse(200, tc.workflowRuns, tc.workflowRuns_queued, tc.workflowRuns_in_progress),
+				fake.WithListWorkflowJobsResponse(200, tc.workflowJobs),
+				fake.WithListRunnersResponse(200, fake.RunnersListBody),
+			)
+			defer server.Close()
+			client := newGithubClient(server)
+
+			h := &HorizontalRunnerAutoscalerReconciler{
+				Log:                   log,
+				Scheme:                scheme,
+				DefaultScaleDownDelay: DefaultScaleDownDelay,
+			}
+
+			rd := v1alpha1.RunnerDeployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testrd",
+				},
+				Spec: v1alpha1.RunnerDeploymentSpec{
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							"foo": "bar",
+						},
+					},
+					Template: v1alpha1.RunnerTemplate{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Spec: v1alpha1.RunnerSpec{
+							RunnerConfig: v1alpha1.RunnerConfig{
+								Organization: tc.org,
+								Labels:       tc.labels,
+							},
+						},
+					},
+					Replicas: tc.fixed,
+				},
+				Status: v1alpha1.RunnerDeploymentStatus{
+					DesiredReplicas: tc.sReplicas,
+				},
+			}
+
+			hra := v1alpha1.HorizontalRunnerAutoscaler{
+				Spec: v1alpha1.HorizontalRunnerAutoscalerSpec{
+					ScaleTargetRef: v1alpha1.ScaleTargetRef{
+						Name: "testrd",
+					},
+					MaxReplicas: tc.max,
+					MinReplicas: tc.min,
+					Metrics: []v1alpha1.MetricSpec{
+						{
+							Type:            v1alpha1.AutoscalingMetricTypeTotalNumberOfQueuedAndInProgressWorkflowRuns,
+							RepositoryNames: tc.repos,
+						},
+					},
+				},
+				Status: v1alpha1.HorizontalRunnerAutoscalerStatus{
+					DesiredReplicas:            tc.sReplicas,
+					LastSuccessfulScaleOutTime: tc.sTime,
+				},
+			}
+
+			minReplicas, _, _, err := h.getMinReplicas(log, metav1Now.Time, hra)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			st := h.scaleTargetFromRD(context.Background(), rd)
+
+			got, err := h.computeReplicasWithCache(client, log, metav1Now.Time, st, hra, minReplicas)
+			if err != nil {
+				if tc.err == "" {
+					t.Fatalf("unexpected error: expected none, got %v", err)
+				} else if err.Error() != tc.err {
+					t.Fatalf("unexpected error: expected %v, got %v", tc.err, err)
+				}
+				return
+			}
+
+			if got != tc.want {
+				t.Errorf("%d: incorrect desired replicas: want %d, got %d", i, tc.want, got)
+			}
+		})
+	}
+}

controllers/actions.summerwind.net/constants.go

@@ -0,0 +1,72 @@
+package controllers
+
+import "time"
+
+const (
+	LabelKeyRunnerSetName = "runnerset-name"
+	LabelKeyRunner        = "actions-runner"
+)
+
+const (
+	// This names requires at least one slash to work.
+	// See https://github.com/google/knative-gcp/issues/378
+	runnerPodFinalizerName             = "actions.summerwind.dev/runner-pod"
+	runnerLinkedResourcesFinalizerName = "actions.summerwind.dev/linked-resources"
+
+	annotationKeyPrefix = "actions-runner/"
+
+	AnnotationKeyLastRegistrationCheckTime = "actions-runner-controller/last-registration-check-time"
+
+	// AnnotationKeyUnregistrationFailureMessage is the annotation that is added onto the pod once it failed to be unregistered from GitHub due to e.g. 422 error
+	AnnotationKeyUnregistrationFailureMessage = annotationKeyPrefix + "unregistration-failure-message"
+
+	// AnnotationKeyUnregistrationCompleteTimestamp is the annotation that is added onto the pod once the previously started unregistration process has been completed.
+	AnnotationKeyUnregistrationCompleteTimestamp = annotationKeyPrefix + "unregistration-complete-timestamp"
+
+	// AnnotationKeyRunnerCompletionWaitStartTimestamp is the annotation that is added onto the pod when
+	// ARC decided to wait until the pod to complete by itself, without the need for ARC to unregister the corresponding runner.
+	AnnotationKeyRunnerCompletionWaitStartTimestamp = annotationKeyPrefix + "runner-completion-wait-start-timestamp"
+
+	// unregistarionStartTimestamp is the annotation that contains the time that the requested unregistration process has been started
+	AnnotationKeyUnregistrationStartTimestamp = annotationKeyPrefix + "unregistration-start-timestamp"
+
+	// AnnotationKeyUnregistrationRequestTimestamp is the annotation that contains the time that the unregistration has been requested.
+	// This doesn't immediately start the unregistration. Instead, ARC will first check if the runner has already been registered.
+	// If not, ARC will hold on until the registration to complete first, and only after that it starts the unregistration process.
+	// This is crucial to avoid a race between ARC marking the runner pod for deletion while the actions-runner registers itself to GitHub, leaving the assigned job
+	// hang like forever.
+	AnnotationKeyUnregistrationRequestTimestamp = annotationKeyPrefix + "unregistration-request-timestamp"
+
+	AnnotationKeyRunnerID = annotationKeyPrefix + "id"
+
+	// This can be any value but a larger value can make an unregistration timeout longer than configured in practice.
+	DefaultUnregistrationRetryDelay = time.Minute
+
+	// RetryDelayOnCreateRegistrationError is the delay between retry attempts for runner registration token creation.
+	// Usually, a retry in this case happens when e.g. your PAT has no access to certain scope of runners, like you're using repository admin's token
+	// for creating a broader scoped runner token, like organizationa or enterprise runner token.
+	// Such permission issue will never fixed automatically, so we don't need to retry so often, hence this value.
+	RetryDelayOnCreateRegistrationError = 3 * time.Minute
+
+	// registrationTimeout is the duration until a pod times out after it becomes Ready and Running.
+	// A pod that is timed out can be terminated if needed.
+	registrationTimeout = 10 * time.Minute
+
+	// DefaultRunnerPodRecreationDelayAfterWebhookScale is the delay until syncing the runners with the desired replicas
+	// after a webhook-based scale up.
+	// This is used to prevent ARC from recreating completed runner pods that are deleted soon without being used at all.
+	// In other words, this is used as a timer to wait for the completed runner to emit the next `workflow_job` webhook event to decrease the desired replicas.
+	// So if we set 30 seconds for this, you are basically saying that you would assume GitHub and your installation of ARC to
+	// emit and propagate a workflow_job completion event down to the RunnerSet or RunnerReplicaSet, vha ARC's github webhook server and HRA, in approximately 30 seconds.
+	// In case it actually took more than DefaultRunnerPodRecreationDelayAfterWebhookScale for the workflow_job completion event to arrive,
+	// ARC will recreate the completed runner(s), assuming something went wrong in either GitHub, your K8s cluster, or ARC, so ARC needs to resync anyway.
+	//
+	// See https://github.com/actions/actions-runner-controller/pull/1180
+	DefaultRunnerPodRecreationDelayAfterWebhookScale = 10 * time.Minute
+
+	EnvVarRunnerName  = "RUNNER_NAME"
+	EnvVarRunnerToken = "RUNNER_TOKEN"
+
+	// defaultHookPath is path to the hook script used when the "containerMode: kubernetes" is specified
+	defaultRunnerHookPath = "/runner/k8s/index.js"
+)

controllers/actions.summerwind.net/horizontal_runner_autoscaler_batch_scale.go

@@ -0,0 +1,206 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/go-logr/logr"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type batchScaler struct {
+	Ctx      context.Context
+	Client   client.Client
+	Log      logr.Logger
+	interval time.Duration
+
+	queue       chan *ScaleTarget
+	workerStart sync.Once
+}
+
+func newBatchScaler(ctx context.Context, client client.Client, log logr.Logger) *batchScaler {
+	return &batchScaler{
+		Ctx:      ctx,
+		Client:   client,
+		Log:      log,
+		interval: 3 * time.Second,
+	}
+}
+
+type batchScaleOperation struct {
+	namespacedName types.NamespacedName
+	scaleOps       []scaleOperation
+}
+
+type scaleOperation struct {
+	trigger v1alpha1.ScaleUpTrigger
+	log     logr.Logger
+}
+
+// Add the scale target to the unbounded queue, blocking until the target is successfully added to the queue.
+// All the targets in the queue are dequeued every 3 seconds, grouped by the HRA, and applied.
+// In a happy path, batchScaler update each HRA only once, even though the HRA had two or more associated webhook events in the 3 seconds interval,
+// which results in less K8s API calls and less HRA update conflicts in case your ARC installation receives a lot of webhook events
+func (s *batchScaler) Add(st *ScaleTarget) {
+	if st == nil {
+		return
+	}
+
+	s.workerStart.Do(func() {
+		var expBackoff = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second}
+
+		s.queue = make(chan *ScaleTarget)
+
+		log := s.Log
+
+		go func() {
+			log.Info("Starting batch worker")
+			defer log.Info("Stopped batch worker")
+
+			for {
+				select {
+				case <-s.Ctx.Done():
+					return
+				default:
+				}
+
+				log.V(2).Info("Batch worker is dequeueing operations")
+
+				batches := map[types.NamespacedName]batchScaleOperation{}
+				after := time.After(s.interval)
+				var ops uint
+
+			batch:
+				for {
+					select {
+					case <-after:
+						break batch
+					case st := <-s.queue:
+						nsName := types.NamespacedName{
+							Namespace: st.HorizontalRunnerAutoscaler.Namespace,
+							Name:      st.HorizontalRunnerAutoscaler.Name,
+						}
+						b, ok := batches[nsName]
+						if !ok {
+							b = batchScaleOperation{
+								namespacedName: nsName,
+							}
+						}
+						b.scaleOps = append(b.scaleOps, scaleOperation{
+							log:     *st.log,
+							trigger: st.ScaleUpTrigger,
+						})
+						batches[nsName] = b
+						ops++
+					}
+				}
+
+				log.V(2).Info("Batch worker dequeued operations", "ops", ops, "batches", len(batches))
+
+			retry:
+				for i := 0; ; i++ {
+					failed := map[types.NamespacedName]batchScaleOperation{}
+
+					for nsName, b := range batches {
+						b := b
+						if err := s.batchScale(context.Background(), b); err != nil {
+							log.V(2).Info("Failed to scale due to error", "error", err)
+							failed[nsName] = b
+						} else {
+							log.V(2).Info("Successfully ran batch scale", "hra", b.namespacedName)
+						}
+					}
+
+					if len(failed) == 0 {
+						break retry
+					}
+
+					batches = failed
+
+					delay := 16 * time.Second
+					if i < len(expBackoff) {
+						delay = expBackoff[i]
+					}
+					time.Sleep(delay)
+				}
+			}
+		}()
+	})
+
+	s.queue <- st
+}
+
+func (s *batchScaler) batchScale(ctx context.Context, batch batchScaleOperation) error {
+	var hra v1alpha1.HorizontalRunnerAutoscaler
+
+	if err := s.Client.Get(ctx, batch.namespacedName, &hra); err != nil {
+		return err
+	}
+
+	copy := hra.DeepCopy()
+
+	copy.Spec.CapacityReservations = getValidCapacityReservations(copy)
+
+	var added, completed int
+
+	for _, scale := range batch.scaleOps {
+		amount := 1
+
+		if scale.trigger.Amount != 0 {
+			amount = scale.trigger.Amount
+		}
+
+		scale.log.V(2).Info("Adding capacity reservation", "amount", amount)
+
+		if amount > 0 {
+			now := time.Now()
+			copy.Spec.CapacityReservations = append(copy.Spec.CapacityReservations, v1alpha1.CapacityReservation{
+				EffectiveTime:  metav1.Time{Time: now},
+				ExpirationTime: metav1.Time{Time: now.Add(scale.trigger.Duration.Duration)},
+				Replicas:       amount,
+			})
+
+			added += amount
+		} else if amount < 0 {
+			var reservations []v1alpha1.CapacityReservation
+
+			var found bool
+
+			for _, r := range copy.Spec.CapacityReservations {
+				if !found && r.Replicas+amount == 0 {
+					found = true
+				} else {
+					reservations = append(reservations, r)
+				}
+			}
+
+			copy.Spec.CapacityReservations = reservations
+
+			completed += amount
+		}
+	}
+
+	before := len(hra.Spec.CapacityReservations)
+	expired := before - len(copy.Spec.CapacityReservations)
+	after := len(copy.Spec.CapacityReservations)
+
+	s.Log.V(1).Info(
+		fmt.Sprintf("Patching hra %s for capacityReservations update", hra.Name),
+		"before", before,
+		"expired", expired,
+		"added", added,
+		"completed", completed,
+		"after", after,
+	)
+
+	if err := s.Client.Patch(ctx, copy, client.MergeFrom(&hra)); err != nil {
+		return fmt.Errorf("patching horizontalrunnerautoscaler to add capacity reservation: %w", err)
+	}
+
+	return nil
+}

controllers/actions.summerwind.net/horizontal_runner_autoscaler_webhook.go

@@ -0,0 +1,853 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"github.com/go-logr/logr"
+	gogithub "github.com/google/go-github/v47/github"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/github"
+	"github.com/actions/actions-runner-controller/simulator"
+)
+
+const (
+	scaleTargetKey = "scaleTarget"
+
+	keyPrefixEnterprise = "enterprises/"
+	keyRunnerGroup      = "/group/"
+
+	DefaultQueueLimit = 100
+)
+
+// HorizontalRunnerAutoscalerGitHubWebhook autoscales a HorizontalRunnerAutoscaler and the RunnerDeployment on each
+// GitHub Webhook received
+type HorizontalRunnerAutoscalerGitHubWebhook struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+
+	// SecretKeyBytes is the byte representation of the Webhook secret token
+	// the administrator is generated and specified in GitHub Web UI.
+	SecretKeyBytes []byte
+
+	// GitHub Client to discover runner groups assigned to a repository
+	GitHubClient *github.Client
+
+	// Namespace is the namespace to watch for HorizontalRunnerAutoscaler's to be
+	// scaled on Webhook.
+	// Set to empty for letting it watch for all namespaces.
+	Namespace string
+	Name      string
+
+	// QueueLimit is the maximum length of the bounded queue of scale targets and their associated operations
+	// A scale target is enqueued on each retrieval of each eligible webhook event, so that it is processed asynchronously.
+	QueueLimit int
+
+	worker     *worker
+	workerInit sync.Once
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Reconcile(_ context.Context, request reconcile.Request) (reconcile.Result, error) {
+	return ctrl.Result{}, nil
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) Handle(w http.ResponseWriter, r *http.Request) {
+	var (
+		ok bool
+
+		err error
+	)
+
+	defer func() {
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+
+			if err != nil {
+				msg := err.Error()
+				if written, err := w.Write([]byte(msg)); err != nil {
+					autoscaler.Log.V(1).Error(err, "failed writing http error response", "msg", msg, "written", written)
+				}
+			}
+		}
+	}()
+
+	defer func() {
+		if r.Body != nil {
+			r.Body.Close()
+		}
+	}()
+
+	// respond ok to GET / e.g. for health check
+	if r.Method == http.MethodGet {
+		ok = true
+		fmt.Fprintln(w, "webhook server is running")
+		return
+	}
+
+	var payload []byte
+
+	if len(autoscaler.SecretKeyBytes) > 0 {
+		payload, err = gogithub.ValidatePayload(r, autoscaler.SecretKeyBytes)
+		if err != nil {
+			autoscaler.Log.Error(err, "error validating request body")
+
+			return
+		}
+	} else {
+		payload, err = io.ReadAll(r.Body)
+		if err != nil {
+			autoscaler.Log.Error(err, "error reading request body")
+
+			return
+		}
+	}
+
+	webhookType := gogithub.WebHookType(r)
+	event, err := gogithub.ParseWebHook(webhookType, payload)
+	if err != nil {
+		var s string
+		if payload != nil {
+			s = string(payload)
+		}
+
+		autoscaler.Log.Error(err, "could not parse webhook", "webhookType", webhookType, "payload", s)
+
+		return
+	}
+
+	var target *ScaleTarget
+
+	log := autoscaler.Log.WithValues(
+		"event", webhookType,
+		"hookID", r.Header.Get("X-GitHub-Hook-ID"),
+		"delivery", r.Header.Get("X-GitHub-Delivery"),
+	)
+
+	var enterpriseEvent struct {
+		Enterprise struct {
+			Slug string `json:"slug,omitempty"`
+		} `json:"enterprise,omitempty"`
+	}
+	if err := json.Unmarshal(payload, &enterpriseEvent); err != nil {
+		var s string
+		if payload != nil {
+			s = string(payload)
+		}
+		autoscaler.Log.Error(err, "could not parse webhook payload for extracting enterprise slug", "webhookType", webhookType, "payload", s)
+	}
+	enterpriseSlug := enterpriseEvent.Enterprise.Slug
+
+	switch e := event.(type) {
+	case *gogithub.WorkflowJobEvent:
+		if workflowJob := e.GetWorkflowJob(); workflowJob != nil {
+			log = log.WithValues(
+				"workflowJob.status", workflowJob.GetStatus(),
+				"workflowJob.labels", workflowJob.Labels,
+				"repository.name", e.Repo.GetName(),
+				"repository.owner.login", e.Repo.Owner.GetLogin(),
+				"repository.owner.type", e.Repo.Owner.GetType(),
+				"enterprise.slug", enterpriseSlug,
+				"action", e.GetAction(),
+				"workflowJob.runID", e.WorkflowJob.GetRunID(),
+				"workflowJob.ID", e.WorkflowJob.GetID(),
+			)
+		}
+
+		labels := e.WorkflowJob.Labels
+
+		switch action := e.GetAction(); action {
+		case "queued", "completed":
+			target, err = autoscaler.getJobScaleUpTargetForRepoOrOrg(
+				context.TODO(),
+				log,
+				e.Repo.GetName(),
+				e.Repo.Owner.GetLogin(),
+				e.Repo.Owner.GetType(),
+				enterpriseSlug,
+				labels,
+			)
+			if target == nil {
+				break
+			}
+
+			if e.GetAction() == "queued" {
+				target.Amount = 1
+				break
+			} else if e.GetAction() == "completed" && e.GetWorkflowJob().GetConclusion() != "skipped" {
+				// A nagative amount is processed in the tryScale func as a scale-down request,
+				// that erasese the oldest CapacityReservation with the same amount.
+				// If the first CapacityReservation was with Replicas=1, this negative scale target erases that,
+				// so that the resulting desired replicas decreases by 1.
+				target.Amount = -1
+				break
+			}
+			// If the conclusion is "skipped", we will ignore it and fallthrough to the default case.
+			fallthrough
+		default:
+			ok = true
+
+			w.WriteHeader(http.StatusOK)
+
+			log.V(2).Info("Received and ignored a workflow_job event as it triggers neither scale-up nor scale-down", "action", action)
+
+			return
+		}
+	case *gogithub.PingEvent:
+		ok = true
+
+		w.WriteHeader(http.StatusOK)
+
+		msg := "pong"
+
+		if written, err := w.Write([]byte(msg)); err != nil {
+			log.Error(err, "failed writing http response", "msg", msg, "written", written)
+		}
+
+		log.Info("received ping event")
+
+		return
+	default:
+		log.Info("unknown event type", "eventType", webhookType)
+
+		return
+	}
+
+	if err != nil {
+		log.Error(err, "handling check_run event")
+
+		return
+	}
+
+	if target == nil {
+		log.V(1).Info(
+			"Scale target not found. If this is unexpected, ensure that there is exactly one repository-wide or organizational runner deployment that matches this webhook event. If --watch-namespace is set ensure this is configured correctly.",
+		)
+
+		msg := "no horizontalrunnerautoscaler to scale for this github event"
+
+		ok = true
+
+		w.WriteHeader(http.StatusOK)
+
+		if written, err := w.Write([]byte(msg)); err != nil {
+			log.Error(err, "failed writing http response", "msg", msg, "written", written)
+		}
+
+		return
+	}
+
+	autoscaler.workerInit.Do(func() {
+		batchScaler := newBatchScaler(context.Background(), autoscaler.Client, autoscaler.Log)
+
+		queueLimit := autoscaler.QueueLimit
+		if queueLimit == 0 {
+			queueLimit = DefaultQueueLimit
+		}
+		autoscaler.worker = newWorker(context.Background(), queueLimit, batchScaler.Add)
+	})
+
+	target.log = &log
+	if ok := autoscaler.worker.Add(target); !ok {
+		log.Error(err, "Could not scale up due to queue full")
+		return
+	}
+
+	ok = true
+
+	w.WriteHeader(http.StatusOK)
+
+	msg := fmt.Sprintf("scaled %s by %d", target.Name, target.Amount)
+
+	log.Info(msg)
+
+	if written, err := w.Write([]byte(msg)); err != nil {
+		log.Error(err, "failed writing http response", "msg", msg, "written", written)
+	}
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) findHRAsByKey(ctx context.Context, value string) ([]v1alpha1.HorizontalRunnerAutoscaler, error) {
+	ns := autoscaler.Namespace
+
+	var defaultListOpts []client.ListOption
+
+	if ns != "" {
+		defaultListOpts = append(defaultListOpts, client.InNamespace(ns))
+	}
+
+	var hras []v1alpha1.HorizontalRunnerAutoscaler
+
+	if value != "" {
+		opts := append([]client.ListOption{}, defaultListOpts...)
+		opts = append(opts, client.MatchingFields{scaleTargetKey: value})
+
+		if autoscaler.Namespace != "" {
+			opts = append(opts, client.InNamespace(autoscaler.Namespace))
+		}
+
+		var hraList v1alpha1.HorizontalRunnerAutoscalerList
+
+		if err := autoscaler.List(ctx, &hraList, opts...); err != nil {
+			return nil, err
+		}
+
+		hras = append(hras, hraList.Items...)
+	}
+
+	return hras, nil
+}
+
+func matchTriggerConditionAgainstEvent(types []string, eventAction *string) bool {
+	if len(types) == 0 {
+		return true
+	}
+
+	if eventAction == nil {
+		return false
+	}
+
+	for _, tpe := range types {
+		if tpe == *eventAction {
+			return true
+		}
+	}
+
+	return false
+}
+
+type ScaleTarget struct {
+	v1alpha1.HorizontalRunnerAutoscaler
+	v1alpha1.ScaleUpTrigger
+
+	log *logr.Logger
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) searchScaleTargets(hras []v1alpha1.HorizontalRunnerAutoscaler, f func(v1alpha1.ScaleUpTrigger) bool) []ScaleTarget {
+	var matched []ScaleTarget
+
+	for _, hra := range hras {
+		if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		for _, scaleUpTrigger := range hra.Spec.ScaleUpTriggers {
+			if !f(scaleUpTrigger) {
+				continue
+			}
+
+			matched = append(matched, ScaleTarget{
+				HorizontalRunnerAutoscaler: hra,
+				ScaleUpTrigger:             scaleUpTrigger,
+			})
+		}
+	}
+
+	return matched
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleTarget(ctx context.Context, name string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) {
+	hras, err := autoscaler.findHRAsByKey(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	autoscaler.Log.V(1).Info(fmt.Sprintf("Found %d HRAs by key", len(hras)), "key", name)
+
+	targets := autoscaler.searchScaleTargets(hras, f)
+
+	n := len(targets)
+
+	if n == 0 {
+		return nil, nil
+	}
+
+	if n > 1 {
+		var scaleTargetIDs []string
+
+		for _, t := range targets {
+			scaleTargetIDs = append(scaleTargetIDs, t.HorizontalRunnerAutoscaler.Name)
+		}
+
+		autoscaler.Log.Info(
+			"Found too many scale targets: "+
+				"It must be exactly one to avoid ambiguity. "+
+				"Either set Namespace for the webhook-based autoscaler to let it only find HRAs in the namespace, "+
+				"or update Repository, Organization, or Enterprise fields in your RunnerDeployment resources to fix the ambiguity.",
+			"scaleTargets", strings.Join(scaleTargetIDs, ","))
+
+		return nil, nil
+	}
+
+	return &targets[0], nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTarget(ctx context.Context, log logr.Logger, repo, owner, ownerType, enterprise string, f func(v1alpha1.ScaleUpTrigger) bool) (*ScaleTarget, error) {
+	scaleTarget := func(value string) (*ScaleTarget, error) {
+		return autoscaler.getScaleTarget(ctx, value, f)
+	}
+	return autoscaler.getScaleUpTargetWithFunction(ctx, log, repo, owner, ownerType, enterprise, scaleTarget)
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getJobScaleUpTargetForRepoOrOrg(
+	ctx context.Context, log logr.Logger, repo, owner, ownerType, enterprise string, labels []string,
+) (*ScaleTarget, error) {
+
+	scaleTarget := func(value string) (*ScaleTarget, error) {
+		return autoscaler.getJobScaleTarget(ctx, value, labels)
+	}
+	return autoscaler.getScaleUpTargetWithFunction(ctx, log, repo, owner, ownerType, enterprise, scaleTarget)
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getScaleUpTargetWithFunction(
+	ctx context.Context, log logr.Logger, repo, owner, ownerType, enterprise string, scaleTarget func(value string) (*ScaleTarget, error)) (*ScaleTarget, error) {
+
+	repositoryRunnerKey := owner + "/" + repo
+
+	// Search for repository HRAs
+	if target, err := scaleTarget(repositoryRunnerKey); err != nil {
+		log.Error(err, "finding repository-wide runner", "repository", repositoryRunnerKey)
+		return nil, err
+	} else if target != nil {
+		log.Info("job scale up target is repository-wide runners", "repository", repo)
+		return target, nil
+	}
+
+	if ownerType == "User" {
+		log.V(1).Info("user repositories not supported", "owner", owner)
+		return nil, nil
+	}
+
+	// Find the potential runner groups first to avoid spending API queries needless. Once/if GitHub improves an
+	// API to find related/linked runner groups from a specific repository this logic could be removed
+	managedRunnerGroups, err := autoscaler.getManagedRunnerGroupsFromHRAs(ctx, enterprise, owner)
+	if err != nil {
+		log.Error(err, "finding potential organization/enterprise runner groups from HRAs", "organization", owner)
+		return nil, err
+	}
+	if managedRunnerGroups.IsEmpty() {
+		log.V(1).Info("no repository/organizational/enterprise runner found",
+			"repository", repositoryRunnerKey,
+			"organization", owner,
+			"enterprises", enterprise,
+		)
+	} else {
+		log.V(1).Info("Found some runner groups are managed by ARC", "groups", managedRunnerGroups)
+	}
+
+	var visibleGroups *simulator.VisibleRunnerGroups
+	if autoscaler.GitHubClient != nil {
+		simu := &simulator.Simulator{
+			Client: autoscaler.GitHubClient,
+			Log:    log,
+		}
+		// Get available organization runner groups and enterprise runner groups for a repository
+		// These are the sum of runner groups with repository access = All repositories and runner groups
+		// where owner/repo has access to as well. The list will include default runner group also if it has access to
+		visibleGroups, err = simu.GetRunnerGroupsVisibleToRepository(ctx, owner, repositoryRunnerKey, managedRunnerGroups)
+		log.V(1).Info("Searching in runner groups", "groups", visibleGroups)
+		if err != nil {
+			log.Error(err, "Unable to find runner groups from repository", "organization", owner, "repository", repo)
+			return nil, fmt.Errorf("error while finding visible runner groups: %v", err)
+		}
+	} else {
+		// For backwards compatibility if GitHub authentication is not configured, we assume all runner groups have
+		// visibility=all to honor the previous implementation, therefore any available enterprise/organization runner
+		// is a potential target for scaling. This will also avoid doing extra API calls caused by
+		// GitHubClient.GetRunnerGroupsVisibleToRepository in case users are not using custom visibility on their runner
+		// groups or they are using only default runner groups
+		visibleGroups = managedRunnerGroups
+	}
+
+	scaleTargetKey := func(rg simulator.RunnerGroup) string {
+		switch rg.Kind {
+		case simulator.Default:
+			switch rg.Scope {
+			case simulator.Organization:
+				return owner
+			case simulator.Enterprise:
+				return enterpriseKey(enterprise)
+			}
+		case simulator.Custom:
+			switch rg.Scope {
+			case simulator.Organization:
+				return organizationalRunnerGroupKey(owner, rg.Name)
+			case simulator.Enterprise:
+				return enterpriseRunnerGroupKey(enterprise, rg.Name)
+			}
+		}
+		return ""
+	}
+
+	log.V(1).Info("groups", "groups", visibleGroups)
+
+	var t *ScaleTarget
+
+	traverseErr := visibleGroups.Traverse(func(rg simulator.RunnerGroup) (bool, error) {
+		key := scaleTargetKey(rg)
+
+		target, err := scaleTarget(key)
+
+		if err != nil {
+			log.Error(err, "finding runner group", "enterprise", enterprise, "organization", owner, "repository", repo, "key", key)
+			return false, err
+		} else if target == nil {
+			return false, nil
+		}
+
+		t = target
+		log.V(1).Info("job scale up target found", "enterprise", enterprise, "organization", owner, "repository", repo, "key", key)
+
+		return true, nil
+	})
+
+	if traverseErr != nil {
+		return nil, err
+	}
+
+	if t == nil {
+		log.V(1).Info("no repository/organizational/enterprise runner found",
+			"repository", repositoryRunnerKey,
+			"organization", owner,
+			"enterprise", enterprise,
+		)
+	}
+
+	return t, nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getManagedRunnerGroupsFromHRAs(ctx context.Context, enterprise, org string) (*simulator.VisibleRunnerGroups, error) {
+	groups := simulator.NewVisibleRunnerGroups()
+	ns := autoscaler.Namespace
+
+	var defaultListOpts []client.ListOption
+	if ns != "" {
+		defaultListOpts = append(defaultListOpts, client.InNamespace(ns))
+	}
+
+	opts := append([]client.ListOption{}, defaultListOpts...)
+	if autoscaler.Namespace != "" {
+		opts = append(opts, client.InNamespace(autoscaler.Namespace))
+	}
+
+	var hraList v1alpha1.HorizontalRunnerAutoscalerList
+	if err := autoscaler.List(ctx, &hraList, opts...); err != nil {
+		return groups, err
+	}
+
+	for _, hra := range hraList.Items {
+		var o, e, g string
+
+		kind := hra.Spec.ScaleTargetRef.Kind
+		switch kind {
+		case "RunnerSet":
+			var rs v1alpha1.RunnerSet
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rs); err != nil {
+				return groups, err
+			}
+			o, e, g = rs.Spec.Organization, rs.Spec.Enterprise, rs.Spec.Group
+		case "RunnerDeployment", "":
+			var rd v1alpha1.RunnerDeployment
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil {
+				return groups, err
+			}
+			o, e, g = rd.Spec.Template.Spec.Organization, rd.Spec.Template.Spec.Enterprise, rd.Spec.Template.Spec.Group
+		default:
+			return nil, fmt.Errorf("unsupported scale target kind: %v", kind)
+		}
+
+		if g != "" && e == "" && o == "" {
+			autoscaler.Log.V(1).Info(
+				"invalid runner group config in scale target: spec.group must be set along with either spec.enterprise or spec.organization",
+				"scaleTargetKind", kind,
+				"group", g,
+				"enterprise", e,
+				"organization", o,
+			)
+
+			continue
+		}
+
+		if e != enterprise && o != org {
+			autoscaler.Log.V(1).Info(
+				"Skipped scale target irrelevant to event",
+				"eventOrganization", org,
+				"eventEnterprise", enterprise,
+				"scaleTargetKind", kind,
+				"scaleTargetGroup", g,
+				"scaleTargetEnterprise", e,
+				"scaleTargetOrganization", o,
+			)
+
+			continue
+		}
+
+		rg := simulator.NewRunnerGroupFromProperties(e, o, g)
+
+		if err := groups.Add(rg); err != nil {
+			return groups, fmt.Errorf("failed adding visible group from HRA %s/%s: %w", hra.Namespace, hra.Name, err)
+		}
+	}
+	return groups, nil
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) getJobScaleTarget(ctx context.Context, name string, labels []string) (*ScaleTarget, error) {
+	hras, err := autoscaler.findHRAsByKey(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	autoscaler.Log.V(1).Info(fmt.Sprintf("Found %d HRAs by key", len(hras)), "key", name)
+
+HRA:
+	for _, hra := range hras {
+		if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		if len(hra.Spec.ScaleUpTriggers) > 1 {
+			autoscaler.Log.V(1).Info("Skipping this HRA as it has too many ScaleUpTriggers to be used in workflow_job based scaling", "hra", hra.Name)
+			continue
+		}
+
+		if len(hra.Spec.ScaleUpTriggers) == 0 {
+			autoscaler.Log.V(1).Info("Skipping this HRA as it has no ScaleUpTriggers configured", "hra", hra.Name)
+			continue
+		}
+
+		scaleUpTrigger := hra.Spec.ScaleUpTriggers[0]
+
+		if scaleUpTrigger.GitHubEvent == nil {
+			autoscaler.Log.V(1).Info("Skipping this HRA as it has no `githubEvent` scale trigger configured", "hra", hra.Name)
+
+			continue
+		}
+
+		if scaleUpTrigger.GitHubEvent.WorkflowJob == nil {
+			autoscaler.Log.V(1).Info("Skipping this HRA as it has no `githubEvent.workflowJob` scale trigger configured", "hra", hra.Name)
+
+			continue
+		}
+
+		duration := scaleUpTrigger.Duration
+		if duration.Duration <= 0 {
+			// Try to release the reserved capacity after at least 10 minutes by default,
+			// we won't end up in the reserved capacity remained forever in case GitHub somehow stopped sending us "completed" workflow_job events.
+			// GitHub usually send us those but nothing is 100% guaranteed, e.g. in case of something went wrong on GitHub :)
+			// Probably we'd better make this configurable via custom resources in the future?
+			duration.Duration = 10 * time.Minute
+		}
+
+		switch hra.Spec.ScaleTargetRef.Kind {
+		case "RunnerSet":
+			var rs v1alpha1.RunnerSet
+
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rs); err != nil {
+				return nil, err
+			}
+
+			// Ensure that the RunnerSet-managed runners have all the labels requested by the workflow_job.
+			for _, l := range labels {
+				var matched bool
+
+				// ignore "self-hosted" label as all instance here are self-hosted
+				if l == "self-hosted" {
+					continue
+				}
+
+				// TODO labels related to OS and architecture needs to be explicitly declared or the current implementation will not be able to find them.
+
+				for _, l2 := range rs.Spec.Labels {
+					if l == l2 {
+						matched = true
+						break
+					}
+				}
+
+				if !matched {
+					continue HRA
+				}
+			}
+
+			return &ScaleTarget{HorizontalRunnerAutoscaler: hra, ScaleUpTrigger: v1alpha1.ScaleUpTrigger{Duration: duration}}, nil
+		case "RunnerDeployment", "":
+			var rd v1alpha1.RunnerDeployment
+
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil {
+				return nil, err
+			}
+
+			// Ensure that the RunnerDeployment-managed runners have all the labels requested by the workflow_job.
+			for _, l := range labels {
+				var matched bool
+
+				// ignore "self-hosted" label as all instance here are self-hosted
+				if l == "self-hosted" {
+					continue
+				}
+
+				// TODO labels related to OS and architecture needs to be explicitly declared or the current implementation will not be able to find them.
+
+				for _, l2 := range rd.Spec.Template.Spec.Labels {
+					if l == l2 {
+						matched = true
+						break
+					}
+				}
+
+				if !matched {
+					continue HRA
+				}
+			}
+
+			return &ScaleTarget{HorizontalRunnerAutoscaler: hra, ScaleUpTrigger: v1alpha1.ScaleUpTrigger{Duration: duration}}, nil
+		default:
+			return nil, fmt.Errorf("unsupported scaleTargetRef.kind: %v", hra.Spec.ScaleTargetRef.Kind)
+		}
+	}
+
+	return nil, nil
+}
+
+func getValidCapacityReservations(autoscaler *v1alpha1.HorizontalRunnerAutoscaler) []v1alpha1.CapacityReservation {
+	var capacityReservations []v1alpha1.CapacityReservation
+
+	now := time.Now()
+
+	for _, reservation := range autoscaler.Spec.CapacityReservations {
+		if reservation.ExpirationTime.Time.After(now) {
+			capacityReservations = append(capacityReservations, reservation)
+		}
+	}
+
+	return capacityReservations
+}
+
+func (autoscaler *HorizontalRunnerAutoscalerGitHubWebhook) SetupWithManager(mgr ctrl.Manager) error {
+	name := "webhookbasedautoscaler"
+	if autoscaler.Name != "" {
+		name = autoscaler.Name
+	}
+
+	autoscaler.Recorder = mgr.GetEventRecorderFor(name)
+
+	if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &v1alpha1.HorizontalRunnerAutoscaler{}, scaleTargetKey, func(rawObj client.Object) []string {
+		hra := rawObj.(*v1alpha1.HorizontalRunnerAutoscaler)
+
+		if hra.Spec.ScaleTargetRef.Name == "" {
+			autoscaler.Log.V(1).Info(fmt.Sprintf("scale target ref name not set for hra %s", hra.Name))
+			return nil
+		}
+
+		switch hra.Spec.ScaleTargetRef.Kind {
+		case "", "RunnerDeployment":
+			var rd v1alpha1.RunnerDeployment
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rd); err != nil {
+				autoscaler.Log.V(1).Info(fmt.Sprintf("RunnerDeployment not found with scale target ref name %s for hra %s", hra.Spec.ScaleTargetRef.Name, hra.Name))
+				return nil
+			}
+
+			keys := []string{}
+			if rd.Spec.Template.Spec.Repository != "" {
+				keys = append(keys, rd.Spec.Template.Spec.Repository) // Repository runners
+			}
+			if rd.Spec.Template.Spec.Organization != "" {
+				if group := rd.Spec.Template.Spec.Group; group != "" {
+					keys = append(keys, organizationalRunnerGroupKey(rd.Spec.Template.Spec.Organization, rd.Spec.Template.Spec.Group)) // Organization runner groups
+				} else {
+					keys = append(keys, rd.Spec.Template.Spec.Organization) // Organization runners
+				}
+			}
+			if enterprise := rd.Spec.Template.Spec.Enterprise; enterprise != "" {
+				if group := rd.Spec.Template.Spec.Group; group != "" {
+					keys = append(keys, enterpriseRunnerGroupKey(enterprise, rd.Spec.Template.Spec.Group)) // Enterprise runner groups
+				} else {
+					keys = append(keys, enterpriseKey(enterprise)) // Enterprise runners
+				}
+			}
+			autoscaler.Log.V(2).Info(fmt.Sprintf("HRA keys indexed for HRA %s: %v", hra.Name, keys))
+			return keys
+		case "RunnerSet":
+			var rs v1alpha1.RunnerSet
+			if err := autoscaler.Client.Get(context.Background(), types.NamespacedName{Namespace: hra.Namespace, Name: hra.Spec.ScaleTargetRef.Name}, &rs); err != nil {
+				autoscaler.Log.V(1).Info(fmt.Sprintf("RunnerSet not found with scale target ref name %s for hra %s", hra.Spec.ScaleTargetRef.Name, hra.Name))
+				return nil
+			}
+
+			keys := []string{}
+			if rs.Spec.Repository != "" {
+				keys = append(keys, rs.Spec.Repository) // Repository runners
+			}
+			if rs.Spec.Organization != "" {
+				keys = append(keys, rs.Spec.Organization) // Organization runners
+				if group := rs.Spec.Group; group != "" {
+					keys = append(keys, organizationalRunnerGroupKey(rs.Spec.Organization, rs.Spec.Group)) // Organization runner groups
+				}
+			}
+			if enterprise := rs.Spec.Enterprise; enterprise != "" {
+				keys = append(keys, enterpriseKey(enterprise)) // Enterprise runners
+				if group := rs.Spec.Group; group != "" {
+					keys = append(keys, enterpriseRunnerGroupKey(enterprise, rs.Spec.Group)) // Enterprise runner groups
+				}
+			}
+			autoscaler.Log.V(2).Info(fmt.Sprintf("HRA keys indexed for HRA %s: %v", hra.Name, keys))
+			return keys
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Named(name).
+		Complete(autoscaler)
+}
+
+func enterpriseKey(name string) string {
+	return keyPrefixEnterprise + name
+}
+
+func organizationalRunnerGroupKey(owner, group string) string {
+	return owner + keyRunnerGroup + group
+}
+
+func enterpriseRunnerGroupKey(enterprise, group string) string {
+	return keyPrefixEnterprise + enterprise + keyRunnerGroup + group
+}

controllers/actions.summerwind.net/horizontal_runner_autoscaler_webhook_test.go

@@ -0,0 +1,567 @@
+package controllers
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/go-logr/logr"
+	"github.com/google/go-github/v47/github"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+var (
+	sc = runtime.NewScheme()
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(sc)
+	_ = actionsv1alpha1.AddToScheme(sc)
+}
+
+func TestWebhookPing(t *testing.T) {
+	testServer(t,
+		"ping",
+		&github.PingEvent{
+			Zen: github.String("zen"),
+		},
+		200,
+		"pong",
+	)
+}
+
+func TestWebhookWorkflowJob(t *testing.T) {
+	setupTest := func() github.WorkflowJobEvent {
+		f, err := os.Open("testdata/org_webhook_workflow_job_payload.json")
+		if err != nil {
+			t.Fatalf("could not open the fixture: %s", err)
+		}
+		defer f.Close()
+		var e github.WorkflowJobEvent
+		if err := json.NewDecoder(f).Decode(&e); err != nil {
+			t.Fatalf("invalid json: %s", err)
+		}
+
+		return e
+	}
+	t.Run("Successful", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"label1"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"scaled test-name by 1",
+			initObjs,
+		)
+	})
+	t.Run("WrongLabels", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"bad-label"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"no horizontalrunnerautoscaler to scale for this github event",
+			initObjs,
+		)
+	})
+	// This test verifies that the old way of matching labels doesn't work anymore
+	t.Run("OldLabels", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"label1": "label1",
+						},
+					},
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"bad-label"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"no horizontalrunnerautoscaler to scale for this github event",
+			initObjs,
+		)
+	})
+}
+
+func TestWebhookWorkflowJobWithSelfHostedLabel(t *testing.T) {
+	setupTest := func() github.WorkflowJobEvent {
+		f, err := os.Open("testdata/org_webhook_workflow_job_with_self_hosted_label_payload.json")
+		if err != nil {
+			t.Fatalf("could not open the fixture: %s", err)
+		}
+		defer f.Close()
+		var e github.WorkflowJobEvent
+		if err := json.NewDecoder(f).Decode(&e); err != nil {
+			t.Fatalf("invalid json: %s", err)
+		}
+
+		return e
+	}
+	t.Run("Successful", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"label1"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"scaled test-name by 1",
+			initObjs,
+		)
+	})
+	t.Run("WrongLabels", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"bad-label"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"no horizontalrunnerautoscaler to scale for this github event",
+			initObjs,
+		)
+	})
+	// This test verifies that the old way of matching labels doesn't work anymore
+	t.Run("OldLabels", func(t *testing.T) {
+		e := setupTest()
+		hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+				ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+					Name: "test-name",
+				},
+				ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+					{
+						GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+							WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+						},
+					},
+				},
+			},
+		}
+
+		rd := &actionsv1alpha1.RunnerDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "test-name",
+			},
+			Spec: actionsv1alpha1.RunnerDeploymentSpec{
+				Template: actionsv1alpha1.RunnerTemplate{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"label1": "label1",
+						},
+					},
+					Spec: actionsv1alpha1.RunnerSpec{
+						RunnerConfig: actionsv1alpha1.RunnerConfig{
+							Organization: "MYORG",
+							Labels:       []string{"bad-label"},
+						},
+					},
+				},
+			},
+		}
+
+		initObjs := []runtime.Object{hra, rd}
+
+		testServerWithInitObjs(t,
+			"workflow_job",
+			&e,
+			200,
+			"no horizontalrunnerautoscaler to scale for this github event",
+			initObjs,
+		)
+	})
+}
+
+func TestGetRequest(t *testing.T) {
+	hra := HorizontalRunnerAutoscalerGitHubWebhook{}
+	request, _ := http.NewRequest(http.MethodGet, "/", nil)
+	recorder := httptest.ResponseRecorder{}
+
+	hra.Handle(&recorder, request)
+	response := recorder.Result()
+
+	if response.StatusCode != http.StatusOK {
+		t.Errorf("want %d, got %d", http.StatusOK, response.StatusCode)
+	}
+}
+
+func TestGetValidCapacityReservations(t *testing.T) {
+	now := time.Now()
+
+	hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+		Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+			CapacityReservations: []actionsv1alpha1.CapacityReservation{
+				{
+					ExpirationTime: metav1.Time{Time: now.Add(-time.Second)},
+					Replicas:       1,
+				},
+				{
+					ExpirationTime: metav1.Time{Time: now},
+					Replicas:       2,
+				},
+				{
+					ExpirationTime: metav1.Time{Time: now.Add(time.Second)},
+					Replicas:       3,
+				},
+			},
+		},
+	}
+
+	revs := getValidCapacityReservations(hra)
+
+	var count int
+
+	for _, r := range revs {
+		count += r.Replicas
+	}
+
+	want := 3
+
+	if count != want {
+		t.Errorf("want %d, got %d", want, count)
+	}
+}
+
+func installTestLogger(webhook *HorizontalRunnerAutoscalerGitHubWebhook) *bytes.Buffer {
+	logs := &bytes.Buffer{}
+
+	sink := &testLogSink{
+		name:   "testlog",
+		writer: logs,
+	}
+
+	log := logr.New(sink)
+
+	webhook.Log = log
+
+	return logs
+}
+
+func testServerWithInitObjs(t *testing.T, eventType string, event interface{}, wantCode int, wantBody string, initObjs []runtime.Object) {
+	t.Helper()
+
+	hraWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{}
+
+	client := fake.NewClientBuilder().WithScheme(sc).WithRuntimeObjects(initObjs...).Build()
+
+	logs := installTestLogger(hraWebhook)
+
+	defer func() {
+		if t.Failed() {
+			t.Logf("diagnostics: %s", logs.String())
+		}
+	}()
+
+	hraWebhook.Client = client
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", hraWebhook.Handle)
+
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	resp, err := sendWebhook(server, eventType, event)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if resp != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	if resp.StatusCode != wantCode {
+		t.Error("status:", resp.StatusCode)
+	}
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(respBody) != wantBody {
+		t.Fatal("body:", string(respBody))
+	}
+}
+
+func testServer(t *testing.T, eventType string, event interface{}, wantCode int, wantBody string) {
+	var initObjs []runtime.Object
+	testServerWithInitObjs(t, eventType, event, wantCode, wantBody, initObjs)
+}
+
+func sendWebhook(server *httptest.Server, eventType string, event interface{}) (*http.Response, error) {
+	jsonBuf := &bytes.Buffer{}
+	enc := json.NewEncoder(jsonBuf)
+	enc.SetIndent("  ", "")
+	err := enc.Encode(event)
+	if err != nil {
+		return nil, fmt.Errorf("[bug in test] encoding event to json: %+v", err)
+	}
+
+	reqBody := jsonBuf.Bytes()
+
+	u, err := url.Parse(server.URL)
+	if err != nil {
+		return nil, fmt.Errorf("parsing server url: %v", err)
+	}
+
+	req := &http.Request{
+		Method: http.MethodPost,
+		URL:    u,
+		Header: map[string][]string{
+			"X-GitHub-Event": {eventType},
+			"Content-Type":   {"application/json"},
+		},
+		Body: io.NopCloser(bytes.NewBuffer(reqBody)),
+	}
+
+	return http.DefaultClient.Do(req)
+}
+
+// testLogSink is a sample logr.Logger that logs in-memory.
+// It's only for testing log outputs.
+type testLogSink struct {
+	name      string
+	keyValues map[string]interface{}
+
+	writer io.Writer
+}
+
+var _ logr.LogSink = &testLogSink{}
+
+func (l *testLogSink) Init(_ logr.RuntimeInfo) {
+
+}
+
+func (l *testLogSink) Info(_ int, msg string, kvs ...interface{}) {
+	fmt.Fprintf(l.writer, "%s] %s\t", l.name, msg)
+	for k, v := range l.keyValues {
+		fmt.Fprintf(l.writer, "%s=%+v ", k, v)
+	}
+	for i := 0; i < len(kvs); i += 2 {
+		fmt.Fprintf(l.writer, "%s=%+v ", kvs[i], kvs[i+1])
+	}
+	fmt.Fprintf(l.writer, "\n")
+}
+
+func (*testLogSink) Enabled(level int) bool {
+	return true
+}
+
+func (l *testLogSink) Error(err error, msg string, kvs ...interface{}) {
+	kvs = append(kvs, "error", err)
+	l.Info(0, msg, kvs...)
+}
+
+func (l *testLogSink) WithName(name string) logr.LogSink {
+	return &testLogSink{
+		name:      l.name + "." + name,
+		keyValues: l.keyValues,
+		writer:    l.writer,
+	}
+}
+
+func (l *testLogSink) WithValues(kvs ...interface{}) logr.LogSink {
+	newMap := make(map[string]interface{}, len(l.keyValues)+len(kvs)/2)
+	for k, v := range l.keyValues {
+		newMap[k] = v
+	}
+	for i := 0; i < len(kvs); i += 2 {
+		newMap[kvs[i].(string)] = kvs[i+1]
+	}
+	return &testLogSink{
+		name:      l.name,
+		keyValues: newMap,
+		writer:    l.writer,
+	}
+}

controllers/actions.summerwind.net/horizontal_runner_autoscaler_webhook_worker.go

@@ -0,0 +1,55 @@
+package controllers
+
+import (
+	"context"
+)
+
+// worker is a worker that has a non-blocking bounded queue of scale targets, dequeues scale target and executes the scale operation one by one.
+type worker struct {
+	scaleTargetQueue chan *ScaleTarget
+	work             func(*ScaleTarget)
+	done             chan struct{}
+}
+
+func newWorker(ctx context.Context, queueLimit int, work func(*ScaleTarget)) *worker {
+	w := &worker{
+		scaleTargetQueue: make(chan *ScaleTarget, queueLimit),
+		work:             work,
+		done:             make(chan struct{}),
+	}
+
+	go func() {
+		defer close(w.done)
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case t := <-w.scaleTargetQueue:
+				work(t)
+			}
+		}
+	}()
+
+	return w
+}
+
+// Add the scale target to the bounded queue, returning the result as a bool value. It returns true on successful enqueue, and returns false otherwise.
+// When returned false, the queue is already full so the enqueue operation must be retried later.
+// If the enqueue was triggered by an external source and there's no intermediate queue that we can use,
+// you must instruct the source to resend the original request later.
+// In case you're building a webhook server around this worker, this means that you must return a http error to the webhook server,
+// so that (hopefully) the sender can resend the webhook event later, or at least the human operator can notice or be notified about the
+// webhook develiery failure so that a manual retry can be done later.
+func (w *worker) Add(st *ScaleTarget) bool {
+	select {
+	case w.scaleTargetQueue <- st:
+		return true
+	default:
+		return false
+	}
+}
+
+func (w *worker) Done() chan struct{} {
+	return w.done
+}

controllers/actions.summerwind.net/horizontal_runner_autoscaler_webhook_worker_test.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWorker_Add(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	w := newWorker(ctx, 2, func(st *ScaleTarget) {})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+}
+
+func TestWorker_Work(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var count int
+
+	w := newWorker(ctx, 1, func(st *ScaleTarget) {
+		count++
+		cancel()
+	})
+	require.True(t, w.Add(&ScaleTarget{}))
+	require.False(t, w.Add(&ScaleTarget{}))
+
+	<-w.Done()
+
+	require.Equal(t, count, 1)
+}

controllers/actions.summerwind.net/horizontalrunnerautoscaler_controller.go

@@ -0,0 +1,554 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/go-logr/logr"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/controllers/actions.summerwind.net/metrics"
+	arcgithub "github.com/actions/actions-runner-controller/github"
+)
+
+const (
+	DefaultScaleDownDelay = 10 * time.Minute
+)
+
+// HorizontalRunnerAutoscalerReconciler reconciles a HorizontalRunnerAutoscaler object
+type HorizontalRunnerAutoscalerReconciler struct {
+	client.Client
+	GitHubClient          *MultiGitHubClient
+	Log                   logr.Logger
+	Recorder              record.EventRecorder
+	Scheme                *runtime.Scheme
+	DefaultScaleDownDelay time.Duration
+	Name                  string
+}
+
+const defaultReplicas = 1
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=horizontalrunnerautoscalers/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *HorizontalRunnerAutoscalerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("horizontalrunnerautoscaler", req.NamespacedName)
+
+	var hra v1alpha1.HorizontalRunnerAutoscaler
+	if err := r.Get(ctx, req.NamespacedName, &hra); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !hra.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForHRA(&hra)
+
+		return ctrl.Result{}, nil
+	}
+
+	metrics.SetHorizontalRunnerAutoscalerSpec(hra.ObjectMeta, hra.Spec)
+
+	kind := hra.Spec.ScaleTargetRef.Kind
+
+	switch kind {
+	case "", "RunnerDeployment":
+		var rd v1alpha1.RunnerDeployment
+		if err := r.Get(ctx, types.NamespacedName{
+			Namespace: req.Namespace,
+			Name:      hra.Spec.ScaleTargetRef.Name,
+		}, &rd); err != nil {
+			return ctrl.Result{}, client.IgnoreNotFound(err)
+		}
+
+		if !rd.ObjectMeta.DeletionTimestamp.IsZero() {
+			return ctrl.Result{}, nil
+		}
+
+		st := r.scaleTargetFromRD(ctx, rd)
+
+		return r.reconcile(ctx, req, log, hra, st, func(newDesiredReplicas int) error {
+			currentDesiredReplicas := getIntOrDefault(rd.Spec.Replicas, defaultReplicas)
+
+			ephemeral := rd.Spec.Template.Spec.Ephemeral == nil || *rd.Spec.Template.Spec.Ephemeral
+
+			var effectiveTime *time.Time
+
+			for _, r := range hra.Spec.CapacityReservations {
+				t := r.EffectiveTime
+				if effectiveTime == nil || effectiveTime.Before(t.Time) {
+					effectiveTime = &t.Time
+				}
+			}
+
+			// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
+			if currentDesiredReplicas != newDesiredReplicas {
+				copy := rd.DeepCopy()
+				copy.Spec.Replicas = &newDesiredReplicas
+
+				if ephemeral && effectiveTime != nil {
+					copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+				}
+
+				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rd)); err != nil {
+					return fmt.Errorf("patching runnerdeployment to have %d replicas: %w", newDesiredReplicas, err)
+				}
+			} else if ephemeral && effectiveTime != nil {
+				copy := rd.DeepCopy()
+				copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+
+				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rd)); err != nil {
+					return fmt.Errorf("patching runnerdeployment to have %d replicas: %w", newDesiredReplicas, err)
+				}
+			}
+			return nil
+		})
+	case "RunnerSet":
+		var rs v1alpha1.RunnerSet
+		if err := r.Get(ctx, types.NamespacedName{
+			Namespace: req.Namespace,
+			Name:      hra.Spec.ScaleTargetRef.Name,
+		}, &rs); err != nil {
+			return ctrl.Result{}, client.IgnoreNotFound(err)
+		}
+
+		if !rs.ObjectMeta.DeletionTimestamp.IsZero() {
+			return ctrl.Result{}, nil
+		}
+
+		var replicas *int
+
+		if rs.Spec.Replicas != nil {
+			v := int(*rs.Spec.Replicas)
+			replicas = &v
+		}
+
+		st := scaleTarget{
+			st:         rs.Name,
+			kind:       "runnerset",
+			enterprise: rs.Spec.Enterprise,
+			org:        rs.Spec.Organization,
+			repo:       rs.Spec.Repository,
+			replicas:   replicas,
+			labels:     rs.Spec.RunnerConfig.Labels,
+			getRunnerMap: func() (map[string]struct{}, error) {
+				// return the list of runners in namespace. Horizontal Runner Autoscaler should only be responsible for scaling resources in its own ns.
+				var runnerPodList corev1.PodList
+
+				var opts []client.ListOption
+
+				opts = append(opts, client.InNamespace(rs.Namespace))
+
+				selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
+				if err != nil {
+					return nil, err
+				}
+
+				opts = append(opts, client.MatchingLabelsSelector{Selector: selector})
+
+				r.Log.V(2).Info("Finding runnerset's runner pods with selector", "ns", rs.Namespace)
+
+				if err := r.List(
+					ctx,
+					&runnerPodList,
+					opts...,
+				); err != nil {
+					if !kerrors.IsNotFound(err) {
+						return nil, err
+					}
+				}
+				runnerMap := make(map[string]struct{})
+				for _, items := range runnerPodList.Items {
+					runnerMap[items.Name] = struct{}{}
+				}
+
+				return runnerMap, nil
+			},
+		}
+
+		return r.reconcile(ctx, req, log, hra, st, func(newDesiredReplicas int) error {
+			var replicas *int
+			if rs.Spec.Replicas != nil {
+				v := int(*rs.Spec.Replicas)
+				replicas = &v
+			}
+			currentDesiredReplicas := getIntOrDefault(replicas, defaultReplicas)
+
+			ephemeral := rs.Spec.Ephemeral == nil || *rs.Spec.Ephemeral
+
+			var effectiveTime *time.Time
+
+			for _, r := range hra.Spec.CapacityReservations {
+				t := r.EffectiveTime
+				if effectiveTime == nil || effectiveTime.Before(t.Time) {
+					effectiveTime = &t.Time
+				}
+			}
+
+			if currentDesiredReplicas != newDesiredReplicas {
+				copy := rs.DeepCopy()
+				v := int32(newDesiredReplicas)
+				copy.Spec.Replicas = &v
+
+				if ephemeral && effectiveTime != nil {
+					copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+				}
+
+				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rs)); err != nil {
+					return fmt.Errorf("patching runnerset to have %d replicas: %w", newDesiredReplicas, err)
+				}
+			} else if ephemeral && effectiveTime != nil {
+				copy := rs.DeepCopy()
+				copy.Spec.EffectiveTime = &metav1.Time{Time: *effectiveTime}
+
+				if err := r.Client.Patch(ctx, copy, client.MergeFrom(&rs)); err != nil {
+					return fmt.Errorf("patching runnerset to have %d replicas: %w", newDesiredReplicas, err)
+				}
+			}
+
+			return nil
+		})
+	}
+
+	log.Info(fmt.Sprintf("Unsupported scale target %s %s: kind %s is not supported. valid kinds are %s and %s", kind, hra.Spec.ScaleTargetRef.Name, kind, "RunnerDeployment", "RunnerSet"))
+
+	return ctrl.Result{}, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) scaleTargetFromRD(ctx context.Context, rd v1alpha1.RunnerDeployment) scaleTarget {
+	st := scaleTarget{
+		st:         rd.Name,
+		kind:       "runnerdeployment",
+		enterprise: rd.Spec.Template.Spec.Enterprise,
+		org:        rd.Spec.Template.Spec.Organization,
+		repo:       rd.Spec.Template.Spec.Repository,
+		replicas:   rd.Spec.Replicas,
+		labels:     rd.Spec.Template.Spec.RunnerConfig.Labels,
+		getRunnerMap: func() (map[string]struct{}, error) {
+			// return the list of runners in namespace. Horizontal Runner Autoscaler should only be responsible for scaling resources in its own ns.
+			var runnerList v1alpha1.RunnerList
+
+			var opts []client.ListOption
+
+			opts = append(opts, client.InNamespace(rd.Namespace))
+
+			selector, err := metav1.LabelSelectorAsSelector(getSelector(&rd))
+			if err != nil {
+				return nil, err
+			}
+
+			opts = append(opts, client.MatchingLabelsSelector{Selector: selector})
+
+			r.Log.V(2).Info("Finding runners with selector", "ns", rd.Namespace)
+
+			if err := r.List(
+				ctx,
+				&runnerList,
+				opts...,
+			); err != nil {
+				if !kerrors.IsNotFound(err) {
+					return nil, err
+				}
+			}
+			runnerMap := make(map[string]struct{})
+			for _, items := range runnerList.Items {
+				runnerMap[items.Name] = struct{}{}
+			}
+
+			return runnerMap, nil
+		},
+	}
+
+	return st
+}
+
+type scaleTarget struct {
+	st, kind              string
+	enterprise, repo, org string
+	replicas              *int
+	labels                []string
+
+	getRunnerMap func() (map[string]struct{}, error)
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) reconcile(ctx context.Context, req ctrl.Request, log logr.Logger, hra v1alpha1.HorizontalRunnerAutoscaler, st scaleTarget, updatedDesiredReplicas func(int) error) (ctrl.Result, error) {
+	now := time.Now()
+
+	minReplicas, active, upcoming, err := r.getMinReplicas(log, now, hra)
+	if err != nil {
+		log.Error(err, "Could not compute min replicas")
+
+		return ctrl.Result{}, err
+	}
+
+	ghc, err := r.GitHubClient.InitForHRA(context.Background(), &hra)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	newDesiredReplicas, err := r.computeReplicasWithCache(ghc, log, now, st, hra, minReplicas)
+	if err != nil {
+		r.Recorder.Event(&hra, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
+
+		log.Error(err, "Could not compute replicas")
+
+		return ctrl.Result{}, err
+	}
+
+	if err := updatedDesiredReplicas(newDesiredReplicas); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	updated := hra.DeepCopy()
+
+	if hra.Status.DesiredReplicas == nil || *hra.Status.DesiredReplicas != newDesiredReplicas {
+		if (hra.Status.DesiredReplicas == nil && newDesiredReplicas > 1) ||
+			(hra.Status.DesiredReplicas != nil && newDesiredReplicas > *hra.Status.DesiredReplicas) {
+
+			updated.Status.LastSuccessfulScaleOutTime = &metav1.Time{Time: time.Now()}
+		}
+
+		updated.Status.DesiredReplicas = &newDesiredReplicas
+	}
+
+	var overridesSummary string
+
+	if (active != nil && upcoming == nil) || (active != nil && upcoming != nil && active.Period.EndTime.Before(upcoming.Period.StartTime)) {
+		after := defaultReplicas
+		if hra.Spec.MinReplicas != nil && *hra.Spec.MinReplicas >= 0 {
+			after = *hra.Spec.MinReplicas
+		}
+
+		overridesSummary = fmt.Sprintf("min=%d time=%s", after, active.Period.EndTime)
+	}
+
+	if active == nil && upcoming != nil || (active != nil && upcoming != nil && active.Period.EndTime.After(upcoming.Period.StartTime)) {
+		if upcoming.ScheduledOverride.MinReplicas != nil {
+			overridesSummary = fmt.Sprintf("min=%d time=%s", *upcoming.ScheduledOverride.MinReplicas, upcoming.Period.StartTime)
+		}
+	}
+
+	if overridesSummary != "" {
+		updated.Status.ScheduledOverridesSummary = &overridesSummary
+	} else {
+		updated.Status.ScheduledOverridesSummary = nil
+	}
+
+	if !reflect.DeepEqual(hra.Status, updated.Status) {
+		metrics.SetHorizontalRunnerAutoscalerStatus(updated.ObjectMeta, updated.Status)
+
+		if err := r.Status().Patch(ctx, updated, client.MergeFrom(&hra)); err != nil {
+			return ctrl.Result{}, fmt.Errorf("patching horizontalrunnerautoscaler status: %w", err)
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "horizontalrunnerautoscaler-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.HorizontalRunnerAutoscaler{}).
+		Named(name).
+		Complete(r)
+}
+
+type Override struct {
+	ScheduledOverride v1alpha1.ScheduledOverride
+	Period            Period
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) matchScheduledOverrides(log logr.Logger, now time.Time, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, *Override, *Override, error) {
+	var minReplicas *int
+	var active, upcoming *Override
+
+	for _, o := range hra.Spec.ScheduledOverrides {
+		log.V(1).Info(
+			"Checking scheduled override",
+			"now", now,
+			"startTime", o.StartTime,
+			"endTime", o.EndTime,
+			"frequency", o.RecurrenceRule.Frequency,
+			"untilTime", o.RecurrenceRule.UntilTime,
+		)
+
+		a, u, err := MatchSchedule(
+			now, o.StartTime.Time, o.EndTime.Time,
+			RecurrenceRule{
+				Frequency: o.RecurrenceRule.Frequency,
+				UntilTime: o.RecurrenceRule.UntilTime.Time,
+			},
+		)
+		if err != nil {
+			return minReplicas, nil, nil, err
+		}
+
+		// Use the first when there are two or more active scheduled overrides,
+		// as the spec defines that the earlier scheduled override is prioritized higher than later ones.
+		if a != nil && active == nil {
+			active = &Override{Period: *a, ScheduledOverride: o}
+
+			if o.MinReplicas != nil {
+				minReplicas = o.MinReplicas
+
+				log.V(1).Info(
+					"Found active scheduled override",
+					"activeStartTime", a.StartTime,
+					"activeEndTime", a.EndTime,
+					"activeMinReplicas", minReplicas,
+				)
+			}
+		}
+
+		if u != nil && (upcoming == nil || u.StartTime.Before(upcoming.Period.StartTime)) {
+			upcoming = &Override{Period: *u, ScheduledOverride: o}
+
+			log.V(1).Info(
+				"Found upcoming scheduled override",
+				"upcomingStartTime", u.StartTime,
+				"upcomingEndTime", u.EndTime,
+				"upcomingMinReplicas", o.MinReplicas,
+			)
+		}
+	}
+
+	return minReplicas, active, upcoming, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) getMinReplicas(log logr.Logger, now time.Time, hra v1alpha1.HorizontalRunnerAutoscaler) (int, *Override, *Override, error) {
+	minReplicas := defaultReplicas
+	if hra.Spec.MinReplicas != nil && *hra.Spec.MinReplicas >= 0 {
+		minReplicas = *hra.Spec.MinReplicas
+	}
+
+	m, active, upcoming, err := r.matchScheduledOverrides(log, now, hra)
+	if err != nil {
+		return 0, nil, nil, err
+	} else if m != nil {
+		minReplicas = *m
+	}
+
+	return minReplicas, active, upcoming, nil
+}
+
+func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(ghc *arcgithub.Client, log logr.Logger, now time.Time, st scaleTarget, hra v1alpha1.HorizontalRunnerAutoscaler, minReplicas int) (int, error) {
+	var suggestedReplicas int
+
+	v, err := r.suggestDesiredReplicas(ghc, st, hra)
+	if err != nil {
+		return 0, err
+	}
+
+	if v == nil {
+		suggestedReplicas = minReplicas
+	} else {
+		suggestedReplicas = *v
+	}
+
+	var reserved int
+
+	for _, reservation := range hra.Spec.CapacityReservations {
+		if reservation.ExpirationTime.Time.After(now) {
+			reserved += reservation.Replicas
+		}
+	}
+
+	newDesiredReplicas := suggestedReplicas + reserved
+
+	if newDesiredReplicas < minReplicas {
+		newDesiredReplicas = minReplicas
+	} else if hra.Spec.MaxReplicas != nil && newDesiredReplicas > *hra.Spec.MaxReplicas {
+		newDesiredReplicas = *hra.Spec.MaxReplicas
+	}
+
+	//
+	// Delay scaling-down for ScaleDownDelaySecondsAfterScaleUp or DefaultScaleDownDelay
+	//
+
+	var scaleDownDelay time.Duration
+
+	if hra.Spec.ScaleDownDelaySecondsAfterScaleUp != nil {
+		scaleDownDelay = time.Duration(*hra.Spec.ScaleDownDelaySecondsAfterScaleUp) * time.Second
+	} else {
+		scaleDownDelay = r.DefaultScaleDownDelay
+	}
+
+	var scaleDownDelayUntil *time.Time
+
+	if hra.Status.DesiredReplicas == nil ||
+		*hra.Status.DesiredReplicas < newDesiredReplicas ||
+		hra.Status.LastSuccessfulScaleOutTime == nil {
+
+	} else if hra.Status.LastSuccessfulScaleOutTime != nil {
+		t := hra.Status.LastSuccessfulScaleOutTime.Add(scaleDownDelay)
+
+		// ScaleDownDelay is not passed
+		if t.After(now) {
+			scaleDownDelayUntil = &t
+			newDesiredReplicas = *hra.Status.DesiredReplicas
+		}
+	} else {
+		newDesiredReplicas = *hra.Status.DesiredReplicas
+	}
+
+	//
+	// Logs various numbers for monitoring and debugging purpose
+	//
+
+	kvs := []interface{}{
+		"suggested", suggestedReplicas,
+		"reserved", reserved,
+		"min", minReplicas,
+	}
+
+	if maxReplicas := hra.Spec.MaxReplicas; maxReplicas != nil {
+		kvs = append(kvs, "max", *maxReplicas)
+	}
+
+	if scaleDownDelayUntil != nil {
+		kvs = append(kvs, "last_scale_up_time", *hra.Status.LastSuccessfulScaleOutTime)
+		kvs = append(kvs, "scale_down_delay_until", scaleDownDelayUntil)
+	}
+
+	log.V(1).Info(fmt.Sprintf("Calculated desired replicas of %d", newDesiredReplicas),
+		kvs...,
+	)
+
+	return newDesiredReplicas, nil
+}

controllers/actions.summerwind.net/integration_test.go

@@ -0,0 +1,632 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	github2 "github.com/actions/actions-runner-controller/github"
+	"github.com/google/go-github/v47/github"
+
+	"github.com/actions/actions-runner-controller/github/fake"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+)
+
+type testEnvironment struct {
+	Namespace *corev1.Namespace
+	Responses *fake.FixedResponses
+
+	webhookServer    *httptest.Server
+	ghClient         *github2.Client
+	fakeRunnerList   *fake.RunnersList
+	fakeGithubServer *httptest.Server
+}
+
+var (
+	workflowRunsFor3Replicas             = `{"total_count": 5, "workflow_runs":[{"status":"queued"}, {"status":"queued"}, {"status":"in_progress"}, {"status":"in_progress"}, {"status":"completed"}]}"`
+	workflowRunsFor3Replicas_queued      = `{"total_count": 2, "workflow_runs":[{"status":"queued"}, {"status":"queued"}]}"`
+	workflowRunsFor3Replicas_in_progress = `{"total_count": 1, "workflow_runs":[{"status":"in_progress"}]}"`
+	workflowRunsFor1Replicas             = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"`
+	workflowRunsFor1Replicas_queued      = `{"total_count": 1, "workflow_runs":[{"status":"queued"}]}"`
+	workflowRunsFor1Replicas_in_progress = `{"total_count": 0, "workflow_runs":[]}"`
+)
+
+// SetupIntegrationTest will set up a testing environment.
+// This includes:
+// * creating a Namespace to be used during the test
+// * starting all the reconcilers
+// * stopping all the reconcilers after the test ends
+// Call this function at the start of each of your tests.
+func SetupIntegrationTest(ctx2 context.Context) *testEnvironment {
+	var ctx context.Context
+	var cancel func()
+	ns := &corev1.Namespace{}
+
+	env := &testEnvironment{
+		Namespace:     ns,
+		webhookServer: nil,
+		ghClient:      nil,
+	}
+
+	BeforeEach(func() {
+		ctx, cancel = context.WithCancel(ctx2)
+		*ns = corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)},
+		}
+
+		err := k8sClient.Create(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+			Namespace: ns.Name,
+		})
+		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
+
+		responses := &fake.FixedResponses{}
+		responses.ListRunners = fake.DefaultListRunnersHandler()
+		responses.ListRepositoryWorkflowRuns = &fake.Handler{
+			Status: 200,
+			Body:   workflowRunsFor3Replicas,
+			Statuses: map[string]string{
+				"queued":      workflowRunsFor3Replicas_queued,
+				"in_progress": workflowRunsFor3Replicas_in_progress,
+			},
+		}
+		fakeRunnerList := fake.NewRunnersList()
+		responses.ListRunners = fakeRunnerList.HandleList()
+		fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses))
+
+		env.Responses = responses
+		env.fakeRunnerList = fakeRunnerList
+		env.fakeGithubServer = fakeGithubServer
+		env.ghClient = newGithubClient(fakeGithubServer)
+
+		controllerName := func(name string) string {
+			return fmt.Sprintf("%s%s", ns.Name, name)
+		}
+
+		multiClient := NewMultiGitHubClient(mgr.GetClient(), env.ghClient)
+
+		runnerController := &RunnerReconciler{
+			Client:                      mgr.GetClient(),
+			Scheme:                      scheme.Scheme,
+			Log:                         logf.Log,
+			Recorder:                    mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			GitHubClient:                multiClient,
+			RunnerImage:                 "example/runner:test",
+			DockerImage:                 "example/docker:test",
+			Name:                        controllerName("runner"),
+			RegistrationRecheckInterval: time.Millisecond * 100,
+			RegistrationRecheckJitter:   time.Millisecond * 10,
+			UnregistrationRetryDelay:    1 * time.Second,
+		}
+		err = runnerController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup runner controller")
+
+		replicasetController := &RunnerReplicaSetReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     controllerName("runnerreplicaset"),
+		}
+		err = replicasetController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup runnerreplicaset controller")
+
+		deploymentsController := &RunnerDeploymentReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerdeployment-controller"),
+			Name:     controllerName("runnnerdeployment"),
+		}
+		err = deploymentsController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup runnerdeployment controller")
+
+		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
+			Client:       mgr.GetClient(),
+			Scheme:       scheme.Scheme,
+			Log:          logf.Log,
+			GitHubClient: multiClient,
+			Recorder:     mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+			Name:         controllerName("horizontalrunnerautoscaler"),
+		}
+		err = autoscalerController.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler controller")
+
+		autoscalerWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{
+			Client:    mgr.GetClient(),
+			Scheme:    scheme.Scheme,
+			Log:       logf.Log,
+			Recorder:  mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+			Name:      controllerName("horizontalrunnerautoscalergithubwebhook"),
+			Namespace: ns.Name,
+		}
+		err = autoscalerWebhook.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler webhook")
+
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", autoscalerWebhook.Handle)
+
+		env.webhookServer = httptest.NewServer(mux)
+
+		go func() {
+			defer GinkgoRecover()
+
+			err := mgr.Start(ctx)
+			Expect(err).NotTo(HaveOccurred(), "failed to start manager")
+		}()
+	})
+
+	AfterEach(func() {
+		defer cancel()
+
+		env.fakeGithubServer.Close()
+		env.webhookServer.Close()
+
+		err := k8sClient.Delete(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
+	})
+
+	return env
+}
+
+var _ = Context("INTEGRATION: Inside of a new namespace", func() {
+	ctx := context.TODO()
+	env := SetupIntegrationTest(ctx)
+	ns := env.Namespace
+
+	Describe("when no existing resources exist", func() {
+
+		It("should create and scale organization's repository runners on workflow_job event", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+									Group:      "baz",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, rd, "test RunnerDeployment")
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 1 replica via ScaleUpTriggers.GitHubEvent.WorkflowJob based scaling
+			{
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(5),
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+								},
+								Amount:   1,
+								Duration: metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler")
+
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 2 replicas on first workflow_job.queued webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"self-hosted"}, int64(1234), int64(4321))
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
+			}
+
+			// Scale-up to 3 replicas on second workflow_job.queued webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"self-hosted"}, int64(1234), int64(4321))
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after second webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
+			}
+
+			// Do not scale-up on third workflow_job.queued webhook event
+			// repo "example" doesn't match our Spec
+			{
+				env.SendWorkflowJobEvent("test", "example", "queued", []string{"self-hosted"}, int64(1234), int64(4321))
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 3, "runners after third webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(3, "count of fake list runners")
+			}
+		})
+
+		It("should be able to scale visible organization runner group with default labels", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+									Group:      "baz",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, rd, "test RunnerDeployment")
+
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(5),
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+								},
+								Amount:   1,
+								Duration: metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler")
+
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+			}
+
+			{
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 2 replicas on first workflow_job webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"self-hosted"}, int64(1234), int64(4321))
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
+			}
+		})
+
+		It("should be able to scale visible organization runner group with custom labels", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+									Group:      "baz",
+									Labels:     []string{"custom-label"},
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, rd, "test RunnerDeployment")
+
+				hra := &actionsv1alpha1.HorizontalRunnerAutoscaler{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.HorizontalRunnerAutoscalerSpec{
+						ScaleTargetRef: actionsv1alpha1.ScaleTargetRef{
+							Name: name,
+						},
+						MinReplicas:                       intPtr(1),
+						MaxReplicas:                       intPtr(5),
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									WorkflowJob: &actionsv1alpha1.WorkflowJobSpec{},
+								},
+								Amount:   1,
+								Duration: metav1.Duration{Duration: time.Minute},
+							},
+						},
+					},
+				}
+
+				ExpectCreate(ctx, hra, "test HorizontalRunnerAutoscaler")
+
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1)
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 1)
+			}
+
+			{
+				env.ExpectRegisteredNumberCountEventuallyEquals(1, "count of fake list runners")
+			}
+
+			// Scale-up to 2 replicas on first workflow_job webhook event
+			{
+				env.SendWorkflowJobEvent("test", "valid", "queued", []string{"custom-label"}, int64(1234), int64(4321))
+				ExpectRunnerSetsCountEventuallyEquals(ctx, ns.Name, 1, "runner sets after webhook")
+				ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx, ns.Name, 2, "runners after first webhook event")
+				env.ExpectRegisteredNumberCountEventuallyEquals(2, "count of fake list runners")
+			}
+		})
+
+	})
+})
+
+func ExpectHRADesiredReplicasEquals(ctx context.Context, ns, name string, desired int, optionalDescriptions ...interface{}) {
+	var rd actionsv1alpha1.HorizontalRunnerAutoscaler
+
+	err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &rd)
+
+	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to get test HRA resource")
+
+	replicas := rd.Status.DesiredReplicas
+
+	ExpectWithOffset(1, *replicas).To(Equal(desired), optionalDescriptions...)
+}
+
+func (env *testEnvironment) ExpectRegisteredNumberCountEventuallyEquals(want int, optionalDescriptions ...interface{}) {
+	EventuallyWithOffset(
+		1,
+		func() int {
+			env.SyncRunnerRegistrations()
+
+			rs, err := env.ghClient.ListRunners(context.Background(), "", "", "test/valid")
+			Expect(err).NotTo(HaveOccurred(), "verifying list fake runners response")
+
+			return len(rs)
+		},
+		time.Second*10, time.Millisecond*500).Should(Equal(want), optionalDescriptions...)
+}
+
+func (env *testEnvironment) SendWorkflowJobEvent(org, repo, statusAndAction string, labels []string, runID int64, ID int64) {
+	resp, err := sendWebhook(env.webhookServer, "workflow_job", &github.WorkflowJobEvent{
+		WorkflowJob: &github.WorkflowJob{
+			ID:     &ID,
+			RunID:  &runID,
+			Status: &statusAndAction,
+			Labels: labels,
+		},
+		Org: &github.Organization{
+			Login: github.String(org),
+		},
+		Repo: &github.Repository{
+			Name: github.String(repo),
+			Owner: &github.User{
+				Login: github.String(org),
+				Type:  github.String("Organization"),
+			},
+		},
+		Action: github.String(statusAndAction),
+	})
+
+	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "failed to send workflow_job event")
+
+	ExpectWithOffset(1, resp.StatusCode).To(Equal(200))
+}
+
+func (env *testEnvironment) SyncRunnerRegistrations() {
+	var runnerList actionsv1alpha1.RunnerList
+
+	err := k8sClient.List(context.TODO(), &runnerList, client.InNamespace(env.Namespace.Name))
+	if err != nil {
+		logf.Log.Error(err, "list runners")
+	}
+
+	env.fakeRunnerList.Sync(runnerList.Items)
+}
+
+func ExpectCreate(ctx context.Context, rd client.Object, s string) {
+	err := k8sClient.Create(ctx, rd)
+
+	ExpectWithOffset(1, err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create %s resource", s))
+}
+
+func ExpectRunnerDeploymentEventuallyUpdates(ctx context.Context, ns string, name string, f func(rd *actionsv1alpha1.RunnerDeployment)) {
+	// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+	// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+	//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+	EventuallyWithOffset(
+		1,
+		func() error {
+			var rd actionsv1alpha1.RunnerDeployment
+
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &rd)
+
+			Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerDeployment resource")
+
+			f(&rd)
+
+			return k8sClient.Update(ctx, &rd)
+		},
+		time.Second*1, time.Millisecond*500).Should(BeNil())
+}
+
+func ExpectRunnerSetsCountEventuallyEquals(ctx context.Context, ns string, count int, optionalDescription ...interface{}) {
+	runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+	EventuallyWithOffset(
+		1,
+		func() int {
+			err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns))
+			if err != nil {
+				logf.Log.Error(err, "list runner sets")
+			}
+
+			return len(runnerSets.Items)
+		},
+		time.Second*10, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...)
+}
+
+func ExpectRunnerCountEventuallyEquals(ctx context.Context, ns string, count int, optionalDescription ...interface{}) {
+	runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}}
+
+	EventuallyWithOffset(
+		1,
+		func() int {
+			err := k8sClient.List(ctx, &runners, client.InNamespace(ns))
+			if err != nil {
+				logf.Log.Error(err, "list runner sets")
+			}
+
+			var running int
+
+			for _, r := range runners.Items {
+				if r.Status.Phase == string(corev1.PodRunning) {
+					running++
+				} else {
+					var pod corev1.Pod
+					if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: r.Name}, &pod); err != nil {
+						logf.Log.Error(err, "simulating pod controller")
+						continue
+					}
+
+					copy := pod.DeepCopy()
+					copy.Status.Phase = corev1.PodRunning
+
+					if err := k8sClient.Status().Patch(ctx, copy, client.MergeFrom(&pod)); err != nil {
+						logf.Log.Error(err, "simulating pod controller")
+						continue
+					}
+				}
+			}
+
+			return running
+		},
+		time.Second*10, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...)
+}
+
+func ExpectRunnerSetsManagedReplicasCountEventuallyEquals(ctx context.Context, ns string, count int, optionalDescription ...interface{}) {
+	runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+	EventuallyWithOffset(
+		1,
+		func() int {
+			err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns))
+			if err != nil {
+				logf.Log.Error(err, "list runner sets")
+			}
+
+			if len(runnerSets.Items) == 0 {
+				logf.Log.Info("No runnerreplicasets exist yet")
+				return -1
+			}
+
+			if len(runnerSets.Items) != 1 {
+				logf.Log.Info("Too many runnerreplicasets exist", "runnerSets", runnerSets)
+				return -1
+			}
+
+			return *runnerSets.Items[0].Spec.Replicas
+		},
+		time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(count), optionalDescription...)
+}

controllers/actions.summerwind.net/metrics/horizontalrunnerautoscaler.go

@@ -0,0 +1,212 @@
+package metrics
+
+import (
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/prometheus/client_golang/prometheus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	hraName        = "horizontalrunnerautoscaler"
+	hraNamespace   = "namespace"
+	stEnterprise   = "enterprise"
+	stOrganization = "organization"
+	stRepository   = "repository"
+	stKind         = "kind"
+	stName         = "name"
+)
+
+var (
+	horizontalRunnerAutoscalerMetrics = []prometheus.Collector{
+		horizontalRunnerAutoscalerMinReplicas,
+		horizontalRunnerAutoscalerMaxReplicas,
+		horizontalRunnerAutoscalerDesiredReplicas,
+		horizontalRunnerAutoscalerReplicasDesired,
+		horizontalRunnerAutoscalerRunners,
+		horizontalRunnerAutoscalerRunnersRegistered,
+		horizontalRunnerAutoscalerRunnersBusy,
+		horizontalRunnerAutoscalerTerminatingBusy,
+		horizontalRunnerAutoscalerNecessaryReplicas,
+		horizontalRunnerAutoscalerWorkflowRunsCompleted,
+		horizontalRunnerAutoscalerWorkflowRunsInProgress,
+		horizontalRunnerAutoscalerWorkflowRunsQueued,
+		horizontalRunnerAutoscalerWorkflowRunsUnknown,
+	}
+)
+
+var (
+	horizontalRunnerAutoscalerMinReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_spec_min_replicas",
+			Help: "minReplicas of HorizontalRunnerAutoscaler",
+		},
+		[]string{hraName, hraNamespace},
+	)
+	horizontalRunnerAutoscalerMaxReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_spec_max_replicas",
+			Help: "maxReplicas of HorizontalRunnerAutoscaler",
+		},
+		[]string{hraName, hraNamespace},
+	)
+	horizontalRunnerAutoscalerDesiredReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_status_desired_replicas",
+			Help: "desiredReplicas of HorizontalRunnerAutoscaler",
+		},
+		[]string{hraName, hraNamespace},
+	)
+	// PercentageRunnersBusy
+	horizontalRunnerAutoscalerReplicasDesired = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_replicas_desired",
+			Help: "replicas_desired of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunners = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners",
+			Help: "num_runners of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersRegistered = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_registered",
+			Help: "num_runners_registered of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerRunnersBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_runners_busy",
+			Help: "num_runners_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerTerminatingBusy = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_terminating_busy",
+			Help: "num_terminating_busy of PercentageRunnersBusy",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	// QueuedAndInProgressWorkflowRuns
+	horizontalRunnerAutoscalerNecessaryReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_necessary_replicas",
+			Help: "necessary_replicas of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsCompleted = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_completed",
+			Help: "workflow_runs_completed of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsInProgress = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_in_progress",
+			Help: "workflow_runs_in_progress of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsQueued = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_queued",
+			Help: "workflow_runs_queued of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+	horizontalRunnerAutoscalerWorkflowRunsUnknown = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "horizontalrunnerautoscaler_workflow_runs_unknown",
+			Help: "workflow_runs_unknown of QueuedAndInProgressWorkflowRuns",
+		},
+		[]string{hraName, hraNamespace, stEnterprise, stOrganization, stRepository, stKind, stName},
+	)
+)
+
+func SetHorizontalRunnerAutoscalerSpec(o metav1.ObjectMeta, spec v1alpha1.HorizontalRunnerAutoscalerSpec) {
+	labels := prometheus.Labels{
+		hraName:      o.Name,
+		hraNamespace: o.Namespace,
+	}
+	if spec.MaxReplicas != nil {
+		horizontalRunnerAutoscalerMaxReplicas.With(labels).Set(float64(*spec.MaxReplicas))
+	}
+	if spec.MinReplicas != nil {
+		horizontalRunnerAutoscalerMinReplicas.With(labels).Set(float64(*spec.MinReplicas))
+	}
+}
+
+func SetHorizontalRunnerAutoscalerStatus(o metav1.ObjectMeta, status v1alpha1.HorizontalRunnerAutoscalerStatus) {
+	labels := prometheus.Labels{
+		hraName:      o.Name,
+		hraNamespace: o.Namespace,
+	}
+	if status.DesiredReplicas != nil {
+		horizontalRunnerAutoscalerDesiredReplicas.With(labels).Set(float64(*status.DesiredReplicas))
+	}
+}
+
+func SetHorizontalRunnerAutoscalerPercentageRunnersBusy(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	desiredReplicas int,
+	numRunners int,
+	numRunnersRegistered int,
+	numRunnersBusy int,
+	numTerminatingBusy int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerReplicasDesired.With(labels).Set(float64(desiredReplicas))
+	horizontalRunnerAutoscalerRunners.With(labels).Set(float64(numRunners))
+	horizontalRunnerAutoscalerRunnersRegistered.With(labels).Set(float64(numRunnersRegistered))
+	horizontalRunnerAutoscalerRunnersBusy.With(labels).Set(float64(numRunnersBusy))
+	horizontalRunnerAutoscalerTerminatingBusy.With(labels).Set(float64(numTerminatingBusy))
+}
+
+func SetHorizontalRunnerAutoscalerQueuedAndInProgressWorkflowRuns(
+	o metav1.ObjectMeta,
+	enterprise string,
+	organization string,
+	repository string,
+	kind string,
+	name string,
+	necessaryReplicas int,
+	workflowRunsCompleted int,
+	workflowRunsInProgress int,
+	workflowRunsQueued int,
+	workflowRunsUnknown int,
+) {
+	labels := prometheus.Labels{
+		hraName:        o.Name,
+		hraNamespace:   o.Namespace,
+		stEnterprise:   enterprise,
+		stOrganization: organization,
+		stRepository:   repository,
+		stKind:         kind,
+		stName:         name,
+	}
+	horizontalRunnerAutoscalerNecessaryReplicas.With(labels).Set(float64(necessaryReplicas))
+	horizontalRunnerAutoscalerWorkflowRunsCompleted.With(labels).Set(float64(workflowRunsCompleted))
+	horizontalRunnerAutoscalerWorkflowRunsInProgress.With(labels).Set(float64(workflowRunsInProgress))
+	horizontalRunnerAutoscalerWorkflowRunsQueued.With(labels).Set(float64(workflowRunsQueued))
+	horizontalRunnerAutoscalerWorkflowRunsUnknown.With(labels).Set(float64(workflowRunsUnknown))
+}

controllers/actions.summerwind.net/metrics/metrics.go

@@ -0,0 +1,14 @@
+// Package metrics provides the metrics of custom resources such as HRA.
+//
+// This depends on the metrics exporter of kubebuilder.
+// See https://book.kubebuilder.io/reference/metrics.html for details.
+package metrics
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+func init() {
+	metrics.Registry.MustRegister(runnerDeploymentMetrics...)
+	metrics.Registry.MustRegister(horizontalRunnerAutoscalerMetrics...)
+}

controllers/actions.summerwind.net/metrics/runnerdeployment.go

@@ -0,0 +1,37 @@
+package metrics
+
+import (
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	rdName      = "runnerdeployment"
+	rdNamespace = "namespace"
+)
+
+var (
+	runnerDeploymentMetrics = []prometheus.Collector{
+		runnerDeploymentReplicas,
+	}
+)
+
+var (
+	runnerDeploymentReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "runnerdeployment_spec_replicas",
+			Help: "replicas of RunnerDeployment",
+		},
+		[]string{rdName, rdNamespace},
+	)
+)
+
+func SetRunnerDeployment(rd v1alpha1.RunnerDeployment) {
+	labels := prometheus.Labels{
+		rdName:      rd.Name,
+		rdNamespace: rd.Namespace,
+	}
+	if rd.Spec.Replicas != nil {
+		runnerDeploymentReplicas.With(labels).Set(float64(*rd.Spec.Replicas))
+	}
+}

controllers/actions.summerwind.net/metrics/runnerset.go

@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	rsName      = "runnerset"
+	rsNamespace = "namespace"
+)
+
+var (
+	runnerSetReplicas = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "runnerset_spec_replicas",
+			Help: "replicas of RunnerSet",
+		},
+		[]string{rsName, rsNamespace},
+	)
+)
+
+func SetRunnerSet(rd v1alpha1.RunnerSet) {
+	labels := prometheus.Labels{
+		rsName:      rd.Name,
+		rsNamespace: rd.Namespace,
+	}
+	if rd.Spec.Replicas != nil {
+		runnerSetReplicas.With(labels).Set(float64(*rd.Spec.Replicas))
+	}
+}

controllers/actions.summerwind.net/multi_githubclient.go

@@ -0,0 +1,334 @@
+package controllers
+
+import (
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"strconv"
+	"sync"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/github"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// The api creds scret annotation is added by the runner controller or the runnerset controller according to runner.spec.githubAPICredentialsFrom.secretRef.name,
+	// so that the runner pod controller can share the same GitHub API credentials and the instance of the GitHub API client with the upstream controllers.
+	annotationKeyGitHubAPICredsSecret = annotationKeyPrefix + "github-api-creds-secret"
+)
+
+type runnerOwnerRef struct {
+	// kind is either StatefulSet or Runner, and populated via the owner reference in the runner pod controller or via the reconcilation target's kind in
+	// runnerset and runner controllers.
+	kind     string
+	ns, name string
+}
+
+type secretRef struct {
+	ns, name string
+}
+
+// savedClient is the each cache entry that contains the client for the specific set of credentials,
+// like a PAT or a pair of key and cert.
+// the `hash` is a part of the savedClient not the key because we are going to keep only the client for the latest creds
+// in case the operator updated the k8s secret containing the credentials.
+type savedClient struct {
+	hash string
+
+	// refs is the map of all the objects that references this client, used for reference counting to gc
+	// the client if unneeded.
+	refs map[runnerOwnerRef]struct{}
+
+	*github.Client
+}
+
+type resourceReader interface {
+	Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error
+}
+
+type MultiGitHubClient struct {
+	mu sync.Mutex
+
+	client resourceReader
+
+	githubClient *github.Client
+
+	// The saved client is freed once all its dependents disappear, or the contents of the secret changed.
+	// We track dependents via a golang map embedded within the savedClient struct. Each dependent is checked on their respective Kubernetes finalizer,
+	// so that we won't miss any dependent's termination.
+	// The change is the secret is determined using the hash of its contents.
+	clients map[secretRef]savedClient
+}
+
+func NewMultiGitHubClient(client resourceReader, githubClient *github.Client) *MultiGitHubClient {
+	return &MultiGitHubClient{
+		client:       client,
+		githubClient: githubClient,
+		clients:      map[secretRef]savedClient{},
+	}
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerPod(ctx context.Context, pod *corev1.Pod) (*github.Client, error) {
+	// These 3 default values are used only when the user created the pod directly, not via Runner, RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunnerPod(pod)
+	secretName := pod.Annotations[annotationKeyGitHubAPICredsSecret]
+
+	// kind can be any of Pod, Runner, RunnerReplicaSet, RunnerDeployment, or RunnerSet depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, pod.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunner(ctx context.Context, r *v1alpha1.Runner) (*github.Client, error) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	// These 3 default values are used only when the user created the runner resource directly, not via RunnerReplicaSet, RunnerDeploment, or RunnerSet resources.
+	ref := refFromRunner(r)
+	if ref.ns != r.Namespace {
+		return nil, fmt.Errorf("referencing github api creds secret from owner in another namespace is not supported yet")
+	}
+
+	// kind can be any of Runner, RunnerReplicaSet, or RunnerDeployment depending on which custom resource the user directly created.
+	return c.initClientWithSecretName(ctx, r.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForRunnerSet(ctx context.Context, rs *v1alpha1.RunnerSet) (*github.Client, error) {
+	ref := refFromRunnerSet(rs)
+
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, rs.Namespace, secretName, ref)
+}
+
+// Init sets up and return the *github.Client for the object.
+// In case the object (like RunnerDeployment) does not request a custom client, it returns the default client.
+func (c *MultiGitHubClient) InitForHRA(ctx context.Context, hra *v1alpha1.HorizontalRunnerAutoscaler) (*github.Client, error) {
+	ref := refFromHorizontalRunnerAutoscaler(hra)
+
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	return c.initClientWithSecretName(ctx, hra.Namespace, secretName, ref)
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerPod(p *corev1.Pod) {
+	secretName := p.Annotations[annotationKeyGitHubAPICredsSecret]
+	c.derefClient(p.Namespace, secretName, refFromRunnerPod(p))
+}
+
+func (c *MultiGitHubClient) DeinitForRunner(r *v1alpha1.Runner) {
+	var secretName string
+	if r.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = r.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(r.Namespace, secretName, refFromRunner(r))
+}
+
+func (c *MultiGitHubClient) DeinitForRunnerSet(rs *v1alpha1.RunnerSet) {
+	var secretName string
+	if rs.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = rs.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(rs.Namespace, secretName, refFromRunnerSet(rs))
+}
+
+func (c *MultiGitHubClient) DeinitForHRA(hra *v1alpha1.HorizontalRunnerAutoscaler) {
+	var secretName string
+	if hra.Spec.GitHubAPICredentialsFrom != nil {
+		secretName = hra.Spec.GitHubAPICredentialsFrom.SecretRef.Name
+	}
+
+	c.derefClient(hra.Namespace, secretName, refFromHorizontalRunnerAutoscaler(hra))
+}
+
+func (c *MultiGitHubClient) initClientForSecret(secret *corev1.Secret, dependent *runnerOwnerRef) (*savedClient, error) {
+	secRef := secretRef{
+		ns:   secret.Namespace,
+		name: secret.Name,
+	}
+
+	cliRef := c.clients[secRef]
+
+	var ks []string
+
+	for k := range secret.Data {
+		ks = append(ks, k)
+	}
+
+	sort.SliceStable(ks, func(i, j int) bool { return ks[i] < ks[j] })
+
+	hash := sha1.New()
+	for _, k := range ks {
+		hash.Write(secret.Data[k])
+	}
+	hashStr := hex.EncodeToString(hash.Sum(nil))
+
+	if cliRef.hash != hashStr {
+		delete(c.clients, secRef)
+
+		conf, err := secretDataToGitHubClientConfig(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		// Fallback to the controller-wide setting if EnterpriseURL is not set and the original client is an enterprise client.
+		if conf.EnterpriseURL == "" && c.githubClient.IsEnterprise {
+			conf.EnterpriseURL = c.githubClient.GithubBaseURL
+		}
+
+		cli, err := conf.NewClient()
+		if err != nil {
+			return nil, err
+		}
+
+		cliRef = savedClient{
+			hash:   hashStr,
+			refs:   map[runnerOwnerRef]struct{}{},
+			Client: cli,
+		}
+
+		c.clients[secRef] = cliRef
+	}
+
+	if dependent != nil {
+		c.clients[secRef].refs[*dependent] = struct{}{}
+	}
+
+	return &cliRef, nil
+}
+
+func (c *MultiGitHubClient) initClientWithSecretName(ctx context.Context, ns, secretName string, runRef *runnerOwnerRef) (*github.Client, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if secretName == "" {
+		return c.githubClient, nil
+	}
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if _, ok := c.clients[secRef]; !ok {
+		c.clients[secRef] = savedClient{}
+	}
+
+	var sec corev1.Secret
+	if err := c.client.Get(ctx, types.NamespacedName{Namespace: ns, Name: secretName}, &sec); err != nil {
+		return nil, err
+	}
+
+	savedClient, err := c.initClientForSecret(&sec, runRef)
+	if err != nil {
+		return nil, err
+	}
+
+	return savedClient.Client, nil
+}
+
+func (c *MultiGitHubClient) derefClient(ns, secretName string, dependent *runnerOwnerRef) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	secRef := secretRef{
+		ns:   ns,
+		name: secretName,
+	}
+
+	if dependent != nil {
+		delete(c.clients[secRef].refs, *dependent)
+	}
+
+	cliRef := c.clients[secRef]
+
+	if dependent == nil || len(cliRef.refs) == 0 {
+		delete(c.clients, secRef)
+	}
+}
+
+func secretDataToGitHubClientConfig(data map[string][]byte) (*github.Config, error) {
+	var (
+		conf github.Config
+
+		err error
+	)
+
+	conf.URL = string(data["github_url"])
+
+	conf.UploadURL = string(data["github_upload_url"])
+
+	conf.EnterpriseURL = string(data["github_enterprise_url"])
+
+	conf.RunnerGitHubURL = string(data["github_runner_url"])
+
+	conf.Token = string(data["github_token"])
+
+	appID := string(data["github_app_id"])
+
+	conf.AppID, err = strconv.ParseInt(appID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	instID := string(data["github_app_installation_id"])
+
+	conf.AppInstallationID, err = strconv.ParseInt(instID, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	conf.AppPrivateKey = string(data["github_app_private_key"])
+
+	return &conf, nil
+}
+
+func refFromRunner(r *v1alpha1.Runner) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: r.Kind,
+		ns:   r.Namespace,
+		name: r.Name,
+	}
+}
+
+func refFromRunnerPod(po *corev1.Pod) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: po.Kind,
+		ns:   po.Namespace,
+		name: po.Name,
+	}
+}
+func refFromRunnerSet(rs *v1alpha1.RunnerSet) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: rs.Kind,
+		ns:   rs.Namespace,
+		name: rs.Name,
+	}
+}
+
+func refFromHorizontalRunnerAutoscaler(hra *v1alpha1.HorizontalRunnerAutoscaler) *runnerOwnerRef {
+	return &runnerOwnerRef{
+		kind: hra.Kind,
+		ns:   hra.Namespace,
+		name: hra.Name,
+	}
+}

controllers/actions.summerwind.net/persistent_volume_claim_controller.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2022 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// RunnerPersistentVolumeClaimReconciler reconciles a PersistentVolume object
+type RunnerPersistentVolumeClaimReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
+}
+
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerPersistentVolumeClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("pvc", req.NamespacedName)
+
+	var pvc corev1.PersistentVolumeClaim
+	if err := r.Get(ctx, req.NamespacedName, &pvc); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	res, err := syncPVC(ctx, r.Client, log, req.Namespace, &pvc)
+
+	if res == nil {
+		res = &ctrl.Result{}
+	}
+
+	return *res, err
+}
+
+func (r *RunnerPersistentVolumeClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerpersistentvolumeclaim-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.PersistentVolumeClaim{}).
+		Named(name).
+		Complete(r)
+}

controllers/actions.summerwind.net/persistent_volume_controller.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2022 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// RunnerPersistentVolumeReconciler reconciles a PersistentVolume object
+type RunnerPersistentVolumeReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
+}
+
+// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerPersistentVolumeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("pv", req.NamespacedName)
+
+	var pv corev1.PersistentVolume
+	if err := r.Get(ctx, req.NamespacedName, &pv); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	res, err := syncPV(ctx, r.Client, log, req.Namespace, &pv)
+	if res == nil {
+		res = &ctrl.Result{}
+	}
+
+	return *res, err
+}
+
+func (r *RunnerPersistentVolumeReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerpersistentvolume-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.PersistentVolume{}).
+		Named(name).
+		Complete(r)
+}

controllers/actions.summerwind.net/pod_runner_token_injector.go

@@ -0,0 +1,134 @@
+package controllers
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/go-logr/logr"
+	"gomodules.xyz/jsonpatch/v2"
+	admissionv1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+const (
+	AnnotationKeyTokenExpirationDate = "actions-runner-controller/token-expires-at"
+)
+
+// +kubebuilder:webhook:path=/mutate-runner-set-pod,mutating=true,failurePolicy=ignore,groups="",resources=pods,verbs=create,versions=v1,name=mutate-runner-pod.webhook.actions.summerwind.dev,sideEffects=None,admissionReviewVersions=v1beta1
+
+type PodRunnerTokenInjector struct {
+	client.Client
+
+	Name         string
+	Log          logr.Logger
+	Recorder     record.EventRecorder
+	GitHubClient *MultiGitHubClient
+	decoder      *admission.Decoder
+}
+
+func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Request) admission.Response {
+	var pod corev1.Pod
+	err := t.decoder.Decode(req, &pod)
+	if err != nil {
+		t.Log.Error(err, "Failed to decode request object")
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	if pod.Annotations == nil {
+		pod.Annotations = map[string]string{}
+	}
+
+	var runnerContainer *corev1.Container
+
+	for i := range pod.Spec.Containers {
+		c := pod.Spec.Containers[i]
+
+		if c.Name == "runner" {
+			runnerContainer = &c
+		}
+	}
+
+	if runnerContainer == nil {
+		return newEmptyResponse()
+	}
+
+	enterprise, okEnterprise := getEnv(runnerContainer, EnvVarEnterprise)
+	repo, okRepo := getEnv(runnerContainer, EnvVarRepo)
+	org, okOrg := getEnv(runnerContainer, EnvVarOrg)
+	if !okRepo || !okOrg || !okEnterprise {
+		return newEmptyResponse()
+	}
+
+	ghc, err := t.GitHubClient.InitForRunnerPod(ctx, &pod)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	rt, err := ghc.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
+	if err != nil {
+		t.Log.Error(err, "Failed to get new registration token")
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	ts := rt.GetExpiresAt().Format(time.RFC3339)
+
+	updated := mutatePod(&pod, *rt.Token)
+
+	updated.Annotations[AnnotationKeyTokenExpirationDate] = ts
+
+	forceRunnerPodRestartPolicyNever(updated)
+
+	buf, err := json.Marshal(updated)
+	if err != nil {
+		t.Log.Error(err, "Failed to encode new object")
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	res := admission.PatchResponseFromRaw(req.Object.Raw, buf)
+	return res
+}
+
+func getEnv(container *corev1.Container, key string) (string, bool) {
+	for _, env := range container.Env {
+		if env.Name == key {
+			return env.Value, true
+		}
+	}
+
+	return "", false
+}
+
+func (t *PodRunnerTokenInjector) InjectDecoder(d *admission.Decoder) error {
+	t.decoder = d
+	return nil
+}
+
+func newEmptyResponse() admission.Response {
+	pt := admissionv1.PatchTypeJSONPatch
+	return admission.Response{
+		Patches: []jsonpatch.Operation{},
+		AdmissionResponse: admissionv1.AdmissionResponse{
+			Allowed:   true,
+			PatchType: &pt,
+		},
+	}
+}
+
+func (r *PodRunnerTokenInjector) SetupWithManager(mgr ctrl.Manager) error {
+	name := "pod-runner-token-injector"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	mgr.GetWebhookServer().Register("/mutate-runner-set-pod", &admission.Webhook{Handler: r})
+
+	return nil
+}

controllers/actions.summerwind.net/runner_graceful_stop.go

@@ -0,0 +1,473 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/actions/actions-runner-controller/github"
+	"github.com/go-logr/logr"
+	gogithub "github.com/google/go-github/v47/github"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// tickRunnerGracefulStop reconciles the runner and the runner pod in a way so that
+// we can delete the runner pod without disrupting a workflow job.
+//
+// This function returns a non-nil pointer to corev1.Pod as the first return value
+// if the runner is considered to have gracefully stopped, hence it's pod is safe for deletion.
+//
+// It's a "tick" operation so a graceful stop can take multiple calls to complete.
+// This function is designed to complete a lengthy graceful stop process in a unblocking way.
+// When it wants to be retried later, the function returns a non-nil *ctrl.Result as the second return value, may or may not populating the error in the second return value.
+// The caller is expected to return the returned ctrl.Result and error to postpone the current reconcilation loop and trigger a scheduled retry.
+func tickRunnerGracefulStop(ctx context.Context, retryDelay time.Duration, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*corev1.Pod, *ctrl.Result, error) {
+	pod, err := annotatePodOnce(ctx, c, log, pod, AnnotationKeyUnregistrationStartTimestamp, time.Now().Format(time.RFC3339))
+	if err != nil {
+		return nil, &ctrl.Result{}, err
+	}
+
+	if res, err := ensureRunnerUnregistration(ctx, retryDelay, log, ghClient, c, enterprise, organization, repository, runner, pod); res != nil {
+		return nil, res, err
+	}
+
+	pod, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyUnregistrationCompleteTimestamp, time.Now().Format(time.RFC3339))
+	if err != nil {
+		return nil, &ctrl.Result{}, err
+	}
+
+	return pod, nil, nil
+}
+
+// annotatePodOnce annotates the pod if it wasn't.
+// Returns the provided pod as-is if it was already annotated.
+// Returns the updated pod if the pod was missing the annotation and the update to add the annotation succeeded.
+func annotatePodOnce(ctx context.Context, c client.Client, log logr.Logger, pod *corev1.Pod, k, v string) (*corev1.Pod, error) {
+	if pod == nil {
+		return nil, nil
+	}
+
+	if _, ok := getAnnotation(pod, k); ok {
+		return pod, nil
+	}
+
+	updated := pod.DeepCopy()
+	setAnnotation(&updated.ObjectMeta, k, v)
+	if err := c.Patch(ctx, updated, client.MergeFrom(pod)); err != nil {
+		log.Error(err, fmt.Sprintf("Failed to patch pod to have %s annotation", k))
+		return nil, err
+	}
+
+	log.V(2).Info("Annotated pod", "key", k, "value", v)
+
+	return updated, nil
+}
+
+// If the first return value is nil, it's safe to delete the runner pod.
+func ensureRunnerUnregistration(ctx context.Context, retryDelay time.Duration, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*ctrl.Result, error) {
+	var runnerID *int64
+
+	if id, ok := getAnnotation(pod, AnnotationKeyRunnerID); ok {
+		v, err := strconv.ParseInt(id, 10, 64)
+		if err != nil {
+			return &ctrl.Result{}, err
+		}
+
+		runnerID = &v
+	}
+
+	if runnerID == nil {
+		runner, err := getRunner(ctx, ghClient, enterprise, organization, repository, runner)
+		if err != nil {
+			return &ctrl.Result{}, err
+		}
+
+		if runner != nil && runner.ID != nil {
+			runnerID = runner.ID
+		}
+	}
+
+	code := runnerContainerExitCode(pod)
+
+	if pod != nil && pod.Annotations[AnnotationKeyUnregistrationCompleteTimestamp] != "" {
+		// If it's already unregistered in the previous reconcilation loop,
+		// you can safely assume that it won't get registered again so it's safe to delete the runner pod.
+		log.Info("Runner pod is marked as already unregistered.")
+	} else if runnerID == nil && !runnerPodOrContainerIsStopped(pod) && !podConditionTransitionTimeAfter(pod, corev1.PodReady, registrationTimeout) &&
+		!podIsPending(pod) {
+
+		log.Info(
+			"Unregistration started before runner obtains ID. Waiting for the registration timeout to elapse, or the runner to obtain ID, or the runner pod to stop",
+			"registrationTimeout", registrationTimeout,
+		)
+		return &ctrl.Result{RequeueAfter: retryDelay}, nil
+	} else if runnerID == nil && podIsPending(pod) {
+		// Note: This logic is here to prevent a dead-lock between ARC and the PV provider.
+		//
+		// The author of this logic thinks that some (or all?) of CSI plugins and PV providers
+		// do not support provisioning dynamic PVs for a pod that is already marked for deletion.
+		// If we didn't handle this case here, ARC would end up with waiting forever until the
+		// PV provider(s) provision PVs for the pod, which seems to never happen.
+		//
+		// For reference, the below is an eaxmple of pod.status that you might see when it happened:
+		// status:
+		//  conditions:
+		//  - lastProbeTime: null
+		//    lastTransitionTime: "2022-11-04T00:04:05Z"
+		//    message: 'binding rejected: running Bind plugin "DefaultBinder": Operation cannot
+		//      be fulfilled on pods/binding "org-runnerdeploy-xv2lg-pm6t2": pod org-runnerdeploy-xv2lg-pm6t2
+		//      is being deleted, cannot be assigned to a host'
+		//    reason: SchedulerError
+		//    status: "False"
+		//    type: PodScheduled
+		//  phase: Pending
+		//  qosClass: BestEffort
+		log.Info(
+			"Unregistration started before runner pod gets scheduled onto a node. "+
+				"Perhaps the runner is taking a long time due to e.g. slow CSI slugin not giving us a PV in a timely manner, or your Kubernetes cluster is overloaded? "+
+				"Marking unregistration as completed anyway because there's nothing ARC can do.",
+			"registrationTimeout", registrationTimeout,
+		)
+	} else if runnerID == nil && runnerPodOrContainerIsStopped(pod) {
+		log.Info(
+			"Unregistration started before runner ID is assigned and the runner stopped before obtaining ID within registration timeout. "+
+				"Perhaps the runner successfully ran the job and stopped normally before the runner ID becomes visible via GitHub API? "+
+				"Perhaps the runner pod was terminated by anyone other than ARC? Was it OOM killed? "+
+				"Marking unregistration as completed anyway because there's nothing ARC can do.",
+			"registrationTimeout", registrationTimeout,
+		)
+	} else if runnerID == nil && podConditionTransitionTimeAfter(pod, corev1.PodReady, registrationTimeout) {
+		log.Info(
+			"Unregistration started before runner ID is assigned and the runner was unable to obtain ID within registration timeout. "+
+				"Perhaps the runner has communication issue, or a firewall egress rule is dropping traffic to GitHub API, or GitHub API is unavailable? "+
+				"Marking unregistration as completed anyway because there's nothing ARC can do. "+
+				"This may result in in cancelling the job depending on your terminationGracePeriodSeconds and RUNNER_GRACEFUL_STOP_TIMEOUT settings.",
+			"registrationTimeout", registrationTimeout,
+		)
+	} else if pod != nil && runnerPodOrContainerIsStopped(pod) {
+		// If it's an ephemeral runner with the actions/runner container exited with 0,
+		// we can safely assume that it has unregistered itself from GitHub Actions
+		// so it's natural that RemoveRunner fails due to 404.
+
+		// If pod has ended up succeeded we need to restart it
+		// Happens e.g. when dind is in runner and run completes
+		log.Info("Runner pod has been stopped with a successful status.")
+	} else if pod != nil && pod.Annotations[AnnotationKeyRunnerCompletionWaitStartTimestamp] != "" {
+		ct := ephemeralRunnerContainerStatus(pod)
+		if ct == nil {
+			log.Info("Runner pod is annotated to wait for completion, and the runner container is not ephemeral")
+
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		lts := ct.LastTerminationState.Terminated
+		if lts == nil {
+			log.Info("Runner pod is annotated to wait for completion, and the runner container is not restarting")
+
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		// Prevent runner pod from stucking in Terminating.
+		// See https://github.com/actions/actions-runner-controller/issues/1369
+		log.Info("Deleting runner pod anyway because it has stopped prematurely. This may leave a dangling runner resource in GitHub Actions",
+			"lastState.exitCode", lts.ExitCode,
+			"lastState.message", lts.Message,
+			"pod.phase", pod.Status.Phase,
+		)
+	} else if ok, err := unregisterRunner(ctx, ghClient, enterprise, organization, repository, *runnerID); err != nil {
+		if errors.Is(err, &gogithub.RateLimitError{}) {
+			// We log the underlying error when we failed calling GitHub API to list or unregisters,
+			// or the runner is still busy.
+			log.Error(
+				err,
+				fmt.Sprintf(
+					"Failed to unregister runner due to GitHub API rate limits. Delaying retry for %s to avoid excessive GitHub API calls",
+					retryDelayOnGitHubAPIRateLimitError,
+				),
+			)
+
+			return &ctrl.Result{RequeueAfter: retryDelayOnGitHubAPIRateLimitError}, err
+		}
+
+		log.V(1).Info("Failed to unregister runner before deleting the pod.", "error", err)
+
+		var (
+			runnerBusy                         bool
+			runnerUnregistrationFailureMessage string
+		)
+
+		errRes := &gogithub.ErrorResponse{}
+		if errors.As(err, &errRes) {
+			if errRes.Response.StatusCode == 403 {
+				log.Error(err, "Unable to unregister due to permission error. "+
+					"Perhaps you've changed the permissions of PAT or GitHub App, or you updated authentication method of ARC in a wrong way? "+
+					"ARC considers it as already unregistered and continue removing the pod. "+
+					"You may need to remove the runner on GitHub UI.")
+
+				return nil, nil
+			}
+
+			runner, _ := getRunner(ctx, ghClient, enterprise, organization, repository, runner)
+
+			var runnerID int64
+
+			if runner != nil && runner.ID != nil {
+				runnerID = *runner.ID
+			}
+
+			runnerBusy = errRes.Response.StatusCode == 422
+			runnerUnregistrationFailureMessage = errRes.Message
+
+			if runnerBusy && code != nil {
+				log.V(2).Info("Runner container has already stopped but the unregistration attempt failed. "+
+					"This can happen when the runner container crashed due to an unhandled error, OOM, etc. "+
+					"ARC terminates the pod anyway. You'd probably need to manually delete the runner later by calling the GitHub API",
+					"runnerExitCode", *code,
+					"runnerID", runnerID,
+				)
+
+				return nil, nil
+			}
+		}
+
+		if runnerBusy {
+			_, err := annotatePodOnce(ctx, c, log, pod, AnnotationKeyUnregistrationFailureMessage, runnerUnregistrationFailureMessage)
+			if err != nil {
+				return &ctrl.Result{}, err
+			}
+
+			// We want to prevent spamming the deletion attemps but returning ctrl.Result with RequeueAfter doesn't
+			// work as the reconcilation can happen earlier due to pod status update.
+			// For ephemeral runners, we can expect it to stop and unregister itself on completion.
+			// So we can just wait for the completion without actively retrying unregistration.
+			ephemeral := getRunnerEnv(pod, EnvVarEphemeral)
+			if ephemeral == "true" {
+				_, err = annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerCompletionWaitStartTimestamp, time.Now().Format(time.RFC3339))
+				if err != nil {
+					return &ctrl.Result{}, err
+				}
+
+				return &ctrl.Result{}, nil
+			}
+
+			log.V(2).Info("Retrying runner unregistration because the static runner is still busy")
+			// Otherwise we may end up spamming 422 errors,
+			// each call consuming GitHub API rate limit
+			// https://github.com/actions/actions-runner-controller/pull/1167#issuecomment-1064213271
+			return &ctrl.Result{RequeueAfter: retryDelay}, nil
+		}
+
+		return &ctrl.Result{}, err
+	} else if ok {
+		log.Info("Runner has just been unregistered.")
+	} else if pod == nil {
+		// `r.unregisterRunner()` will returns `false, nil` if the runner is not found on GitHub.
+		// However, that doesn't always mean the pod can be safely removed.
+		//
+		// If the pod does not exist for the runner,
+		// it may be due to that the runner pod has never been created.
+		// In that case we can safely assume that the runner will never be registered.
+
+		log.Info("Runner was not found on GitHub and the runner pod was not found on Kuberntes.")
+	} else if ts := pod.Annotations[AnnotationKeyUnregistrationStartTimestamp]; ts != "" {
+		log.Info("Runner unregistration is in-progress. It can take forever to complete if if it's a static runner constantly running jobs."+
+			" It can also take very long time if it's an ephemeral runner that is running a log-running job.", "error", err)
+
+		return &ctrl.Result{RequeueAfter: retryDelay}, nil
+	} else {
+		// A runner and a runner pod that is created by this version of ARC should match
+		// any of the above branches.
+		//
+		// But we leave this match all branch for potential backward-compatibility.
+		// The caller is expected to take appropriate actions, like annotating the pod as started the unregistration process,
+		// and retry later.
+		log.V(1).Info("Runner unregistration is being retried later.")
+
+		return &ctrl.Result{RequeueAfter: retryDelay}, nil
+	}
+
+	return nil, nil
+}
+
+func ensureRunnerPodRegistered(ctx context.Context, log logr.Logger, ghClient *github.Client, c client.Client, enterprise, organization, repository, runner string, pod *corev1.Pod) (*corev1.Pod, *ctrl.Result, error) {
+	_, hasRunnerID := getAnnotation(pod, AnnotationKeyRunnerID)
+	if runnerPodOrContainerIsStopped(pod) || hasRunnerID {
+		return pod, nil, nil
+	}
+
+	r, err := getRunner(ctx, ghClient, enterprise, organization, repository, runner)
+	if err != nil {
+		return nil, &ctrl.Result{RequeueAfter: 10 * time.Second}, err
+	}
+
+	if r == nil || r.ID == nil {
+		return nil, &ctrl.Result{RequeueAfter: 10 * time.Second}, err
+	}
+
+	id := *r.ID
+
+	updated, err := annotatePodOnce(ctx, c, log, pod, AnnotationKeyRunnerID, fmt.Sprintf("%d", id))
+	if err != nil {
+		return nil, &ctrl.Result{RequeueAfter: 10 * time.Second}, err
+	}
+
+	return updated, nil, nil
+}
+
+func getAnnotation(obj client.Object, key string) (string, bool) {
+	if obj.GetAnnotations() == nil {
+		return "", false
+	}
+
+	v, ok := obj.GetAnnotations()[key]
+
+	return v, ok
+}
+
+func setAnnotation(meta *metav1.ObjectMeta, key, value string) {
+	if meta.Annotations == nil {
+		meta.Annotations = map[string]string{}
+	}
+
+	meta.Annotations[key] = value
+}
+
+func podConditionTransitionTime(pod *corev1.Pod, tpe corev1.PodConditionType, v corev1.ConditionStatus) *metav1.Time {
+	for _, c := range pod.Status.Conditions {
+		if c.Type == tpe && c.Status == v {
+			return &c.LastTransitionTime
+		}
+	}
+
+	return nil
+}
+
+func podConditionTransitionTimeAfter(pod *corev1.Pod, tpe corev1.PodConditionType, d time.Duration) bool {
+	c := podConditionTransitionTime(pod, tpe, corev1.ConditionTrue)
+	if c == nil {
+		return false
+	}
+
+	return c.Add(d).Before(time.Now())
+}
+
+func podIsPending(pod *corev1.Pod) bool {
+	return pod.Status.Phase == corev1.PodPending
+}
+
+func podRunnerID(pod *corev1.Pod) string {
+	id, _ := getAnnotation(pod, AnnotationKeyRunnerID)
+	return id
+}
+
+func getRunnerEnv(pod *corev1.Pod, key string) string {
+	for _, c := range pod.Spec.Containers {
+		if c.Name == containerName {
+			for _, e := range c.Env {
+				if e.Name == key {
+					return e.Value
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func setRunnerEnv(pod *corev1.Pod, key, value string) {
+	for i := range pod.Spec.Containers {
+		c := pod.Spec.Containers[i]
+		if c.Name == containerName {
+			for j, env := range c.Env {
+				if env.Name == key {
+					pod.Spec.Containers[i].Env[j].Value = value
+					return
+				}
+			}
+			pod.Spec.Containers[i].Env = append(c.Env, corev1.EnvVar{Name: key, Value: value})
+		}
+	}
+}
+
+// unregisterRunner unregisters the runner from GitHub Actions by name.
+//
+// This function returns:
+//
+// Case 1. (true, nil) when it has successfully unregistered the runner.
+// Case 2. (false, nil) when (2-1.) the runner has been already unregistered OR (2-2.) the runner will never be created OR (2-3.) the runner is not created yet and it is about to be registered(hence we couldn't see it's existence from GitHub Actions API yet)
+// Case 3. (false, err) when it postponed unregistration due to the runner being busy, or it tried to unregister the runner but failed due to
+//
+//	an error returned by GitHub API.
+//
+// When the returned values is "Case 2. (false, nil)", the caller must handle the three possible sub-cases appropriately.
+// In other words, all those three sub-cases cannot be distinguished by this function alone.
+//
+//   - Case "2-1." can happen when e.g. ARC has successfully unregistered in a previous reconcilation loop or it was an ephemeral runner that finished it's job run(an ephemeral runner is designed to stop after a job run).
+//     You'd need to maintain the runner state(i.e. if it's already unregistered or not) somewhere,
+//     so that you can either not call this function at all if the runner state says it's already unregistered, or determine that it's case "2-1." when you got (false, nil).
+//
+//   - Case "2-2." can happen when e.g. the runner registration token was somehow broken so that `config.sh` within the runner container was never meant to succeed.
+//     Waiting and retrying forever on this case is not a solution, because `config.sh` won't succeed with a wrong token hence the runner gets stuck in this state forever.
+//     There isn't a perfect solution to this, but a practical workaround would be implement a "grace period" in the caller side.
+//
+//   - Case "2-3." can happen when e.g. ARC recreated an ephemral runner pod in a previous reconcilation loop and then it was requested to delete the runner before the runner comes up.
+//     If handled inappropriately, this can cause a race condition betweeen a deletion of the runner pod and GitHub scheduling a workflow job onto the runner.
+//
+// Once successfully detected case "2-1." or "2-2.", you can safely delete the runner pod because you know that the runner won't come back
+// as long as you recreate the runner pod.
+//
+// If it was "2-3.", you need a workaround to avoid the race condition.
+//
+// You shall introduce a "grace period" mechanism, similar or equal to that is required for "Case 2-2.", so that you ever
+// start the runner pod deletion only after it's more and more likely that the runner pod is not coming up.
+//
+// Beware though, you need extra care to set an appropriate grace period depending on your environment.
+// There isn't a single right grace period that works for everyone.
+// The longer the grace period is, the earlier a cluster resource shortage can occur due to throttoled runner pod deletions,
+// while the shorter the grace period is, the more likely you may encounter the race issue.
+func unregisterRunner(ctx context.Context, client *github.Client, enterprise, org, repo string, id int64) (bool, error) {
+	// For the record, historically ARC did not try to call RemoveRunner on a busy runner, but it's no longer true.
+	// The reason ARC did so was to let a runner running a job to not stop prematurely.
+	//
+	// However, we learned that RemoveRunner already has an ability to prevent stopping a busy runner,
+	// so ARC doesn't need to do anything special for a graceful runner stop.
+	// It can just call RemoveRunner, and if it returned 200 you're guaranteed that the runner will not automatically come back and
+	// the runner pod is safe for deletion.
+	//
+	// Trying to remove a busy runner can result in errors like the following:
+	//    failed to remove runner: DELETE https://api.github.com/repos/actions-runner-controller/mumoshu-actions-test/actions/runners/47: 422 Bad request - Runner \"example-runnerset-0\" is still running a job\" []
+	//
+	// # NOTES
+	//
+	// - It can be "status=offline" at the same time but that's another story.
+	// - After https://github.com/actions/actions-runner-controller/pull/1127, ListRunners responses that are used to
+	//   determine if the runner is busy can be more outdated than before, as those responeses are now cached for 60 seconds.
+	// - Note that 60 seconds is controlled by the Cache-Control response header provided by GitHub so we don't have a strict control on it but we assume it won't
+	//   change from 60 seconds.
+	//
+	// TODO: Probably we can just remove the runner by ID without seeing if the runner is busy, by treating it as busy when a remove-runner call failed with 422?
+	if err := client.RemoveRunner(ctx, enterprise, org, repo, id); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+func getRunner(ctx context.Context, client *github.Client, enterprise, org, repo, name string) (*gogithub.Runner, error) {
+	runners, err := client.ListRunners(ctx, enterprise, org, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, runner := range runners {
+		if runner.GetName() == name {
+			return runner, nil
+		}
+	}
+
+	return nil, nil
+}

controllers/actions.summerwind.net/runner_pod.go

@@ -0,0 +1,22 @@
+package controllers
+
+import corev1 "k8s.io/api/core/v1"
+
+// Force the runner pod managed by either RunnerDeployment and RunnerSet to have restartPolicy=Never.
+// See https://github.com/actions/actions-runner-controller/issues/1369 for more context.
+//
+// This is to prevent runner pods from stucking in Terminating when a K8s node disappeared along with the runnr pod and the runner container within it.
+//
+// Previously, we used restartPolicy of OnFailure, it turned wrong later, and therefore we now set Never.
+//
+// When the restartPolicy is OnFailure and the node disappeared, runner pods on the node seem to stuck in state.terminated==nil, state.waiting!=nil, and state.lastTerminationState!=nil,
+// and will ever become Running.
+// It's probably due to that the node onto which the pods have been scheduled will ever come back, hence the container restart attempt swill ever succeed,
+// the pods stuck waiting for successful restarts forever.
+//
+// By forcing runner pods to never restart, we hope there will be no chances of pods being stuck waiting.
+func forceRunnerPodRestartPolicyNever(pod *corev1.Pod) {
+	if pod.Spec.RestartPolicy != corev1.RestartPolicyNever {
+		pod.Spec.RestartPolicy = corev1.RestartPolicyNever
+	}
+}

controllers/actions.summerwind.net/runner_pod_controller.go

@@ -0,0 +1,371 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/go-logr/logr"
+
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	arcv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// RunnerPodReconciler reconciles a Runner object
+type RunnerPodReconciler struct {
+	client.Client
+	Log                         logr.Logger
+	Recorder                    record.EventRecorder
+	Scheme                      *runtime.Scheme
+	GitHubClient                *MultiGitHubClient
+	Name                        string
+	RegistrationRecheckInterval time.Duration
+	RegistrationRecheckJitter   time.Duration
+
+	UnregistrationRetryDelay time.Duration
+}
+
+// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerPodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("runnerpod", req.NamespacedName)
+
+	var runnerPod corev1.Pod
+	if err := r.Get(ctx, req.NamespacedName, &runnerPod); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	_, isRunnerPod := runnerPod.Labels[LabelKeyRunner]
+	_, isRunnerSetPod := runnerPod.Labels[LabelKeyRunnerSetName]
+	_, isRunnerDeploymentPod := runnerPod.Labels[LabelKeyRunnerDeploymentName]
+
+	if !isRunnerPod && !isRunnerSetPod && !isRunnerDeploymentPod {
+		return ctrl.Result{}, nil
+	}
+
+	var envvars []corev1.EnvVar
+	for _, container := range runnerPod.Spec.Containers {
+		if container.Name == "runner" {
+			envvars = container.Env
+		}
+	}
+
+	if len(envvars) == 0 {
+		return ctrl.Result{}, errors.New("Could not determine env vars for runner Pod")
+	}
+
+	var enterprise, org, repo string
+	var isContainerMode bool
+
+	for _, e := range envvars {
+		switch e.Name {
+		case EnvVarEnterprise:
+			enterprise = e.Value
+		case EnvVarOrg:
+			org = e.Value
+		case EnvVarRepo:
+			repo = e.Value
+		case "ACTIONS_RUNNER_CONTAINER_HOOKS":
+			isContainerMode = true
+		}
+	}
+
+	ghc, err := r.GitHubClient.InitForRunnerPod(ctx, &runnerPod)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if runnerPod.ObjectMeta.DeletionTimestamp.IsZero() {
+		finalizers, added := addFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
+
+		var cleanupFinalizersAdded bool
+		if isContainerMode {
+			finalizers, cleanupFinalizersAdded = addFinalizer(finalizers, runnerLinkedResourcesFinalizerName)
+		}
+
+		if added || cleanupFinalizersAdded {
+			newRunner := runnerPod.DeepCopy()
+			newRunner.ObjectMeta.Finalizers = finalizers
+
+			if err := r.Patch(ctx, newRunner, client.MergeFrom(&runnerPod)); err != nil {
+				log.Error(err, "Failed to update runner")
+				return ctrl.Result{}, err
+			}
+
+			log.V(2).Info("Added finalizer")
+
+			return ctrl.Result{}, nil
+		}
+	} else {
+		log.V(2).Info("Seen deletion-timestamp is already set")
+
+		// Mark the parent Runner resource for deletion before deleting this runner pod from the cluster.
+		// Otherwise the runner controller can recreate the runner pod thinking it has not created any runner pod yet.
+		var (
+			key    = types.NamespacedName{Namespace: runnerPod.Namespace, Name: runnerPod.Name}
+			runner arcv1alpha1.Runner
+		)
+		if err := r.Get(ctx, key, &runner); err == nil {
+			if runner.Name != "" && runner.DeletionTimestamp == nil {
+				log.Info("This runner pod seems to have been deleted directly, bypassing the parent Runner resource. Marking the runner for deletion to not let it recreate this pod.")
+				if err := r.Delete(ctx, &runner); err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+		}
+
+		if finalizers, removed := removeFinalizer(runnerPod.ObjectMeta.Finalizers, runnerLinkedResourcesFinalizerName); removed {
+			if err := r.cleanupRunnerLinkedPods(ctx, &runnerPod, log); err != nil {
+				log.Info("Runner-linked pods clean up that has failed due to an error. If this persists, please manually remove the runner-linked pods to unblock ARC", "err", err.Error())
+				return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+			}
+			if err := r.cleanupRunnerLinkedSecrets(ctx, &runnerPod, log); err != nil {
+				log.Info("Runner-linked secrets clean up that has failed due to an error. If this persists, please manually remove the runner-linked secrets to unblock ARC", "err", err.Error())
+				return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
+			}
+			patchedPod := runnerPod.DeepCopy()
+			patchedPod.ObjectMeta.Finalizers = finalizers
+
+			if err := r.Patch(ctx, patchedPod, client.MergeFrom(&runnerPod)); err != nil {
+				log.Error(err, "Failed to update runner for finalizer linked resources removal")
+				return ctrl.Result{}, err
+			}
+
+			// Otherwise the subsequent patch request can revive the removed finalizer and it will trigger a unnecessary reconcilation
+			runnerPod = *patchedPod
+		}
+
+		finalizers, removed := removeFinalizer(runnerPod.ObjectMeta.Finalizers, runnerPodFinalizerName)
+
+		if removed {
+			// In a standard scenario, the upstream controller, like runnerset-controller, ensures this runner to be gracefully stopped before the deletion timestamp is set.
+			// But for the case that the user manually deleted it for whatever reason,
+			// we have to ensure it to gracefully stop now.
+			updatedPod, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+			if res != nil {
+				return *res, err
+			}
+
+			patchedPod := updatedPod.DeepCopy()
+			patchedPod.ObjectMeta.Finalizers = finalizers
+
+			// We commit the removal of the finalizer so that Kuberenetes notices it and delete the pod resource from the cluster.
+			if err := r.Patch(ctx, patchedPod, client.MergeFrom(&runnerPod)); err != nil {
+				log.Error(err, "Failed to update runner for finalizer removal")
+				return ctrl.Result{}, err
+			}
+
+			log.V(2).Info("Removed finalizer")
+
+			r.GitHubClient.DeinitForRunnerPod(updatedPod)
+
+			return ctrl.Result{}, nil
+		}
+
+		deletionTimeout := 1 * time.Minute
+		currentTime := time.Now()
+		deletionDidTimeout := currentTime.Sub(runnerPod.DeletionTimestamp.Add(deletionTimeout)) > 0
+
+		if deletionDidTimeout {
+			log.Info(
+				fmt.Sprintf("Failed to delete pod within %s. ", deletionTimeout)+
+					"This is typically the case when a Kubernetes node became unreachable "+
+					"and the kube controller started evicting nodes. Forcefully deleting the pod to not get stuck.",
+				"podDeletionTimestamp", runnerPod.DeletionTimestamp,
+				"currentTime", currentTime,
+				"configuredDeletionTimeout", deletionTimeout,
+			)
+
+			var force int64 = 0
+			// forcefully delete runner as we would otherwise get stuck if the node stays unreachable
+			if err := r.Delete(ctx, &runnerPod, &client.DeleteOptions{GracePeriodSeconds: &force}); err != nil {
+				// probably
+				if !kerrors.IsNotFound(err) {
+					log.Error(err, "Failed to forcefully delete pod resource ...")
+					return ctrl.Result{}, err
+				}
+				// forceful deletion finally succeeded
+				return ctrl.Result{Requeue: true}, nil
+			}
+
+			r.Recorder.Event(&runnerPod, corev1.EventTypeNormal, "PodDeleted", fmt.Sprintf("Forcefully deleted pod '%s'", runnerPod.Name))
+			log.Info("Forcefully deleted runner pod", "repository", repo)
+			// give kube manager a little time to forcefully delete the stuck pod
+			return ctrl.Result{RequeueAfter: 3 * time.Second}, nil
+		}
+
+		return ctrl.Result{}, nil
+	}
+
+	po, res, err := ensureRunnerPodRegistered(ctx, log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+	if res != nil {
+		return *res, err
+	}
+
+	runnerPod = *po
+
+	if _, unregistrationRequested := getAnnotation(&runnerPod, AnnotationKeyUnregistrationRequestTimestamp); unregistrationRequested {
+		log.V(2).Info("Progressing unregistration because unregistration-request timestamp is set")
+
+		// At this point we're sure that DeletionTimestamp is not set yet, but the unregistration process is triggered by an upstream controller like runnerset-controller.
+		//
+		// In a standard scenario, ARC starts the unregistration process before marking the pod for deletion at all,
+		// so that it isn't subject to terminationGracePeriod and can safely take hours to finish it's work.
+		_, res, err := tickRunnerGracefulStop(ctx, r.unregistrationRetryDelay(), log, ghc, r.Client, enterprise, org, repo, runnerPod.Name, &runnerPod)
+		if res != nil {
+			return *res, err
+		}
+
+		// At this point we are sure that the runner has successfully unregistered, hence is safe to be deleted.
+		// But we don't delete the pod here. Instead, let the upstream controller/parent object to delete this pod as
+		// a part of a cascade deletion.
+		// This is to avoid a parent object, like statefulset, to recreate the deleted pod.
+		// If the pod was recreated, it will start a registration process and that may race with the statefulset deleting the pod.
+		log.V(2).Info("Unregistration seems complete")
+
+		return ctrl.Result{}, nil
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *RunnerPodReconciler) unregistrationRetryDelay() time.Duration {
+	retryDelay := DefaultUnregistrationRetryDelay
+
+	if r.UnregistrationRetryDelay > 0 {
+		retryDelay = r.UnregistrationRetryDelay
+	}
+	return retryDelay
+}
+
+func (r *RunnerPodReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerpod-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Pod{}).
+		Named(name).
+		Complete(r)
+}
+
+func (r *RunnerPodReconciler) cleanupRunnerLinkedPods(ctx context.Context, pod *corev1.Pod, log logr.Logger) error {
+	var runnerLinkedPodList corev1.PodList
+	if err := r.List(ctx, &runnerLinkedPodList, client.InNamespace(pod.Namespace), client.MatchingLabels(
+		map[string]string{
+			"runner-pod": pod.ObjectMeta.Name,
+		},
+	)); err != nil {
+		return fmt.Errorf("failed to list runner-linked pods: %w", err)
+	}
+
+	var (
+		wg   sync.WaitGroup
+		errs []error
+	)
+	for _, p := range runnerLinkedPodList.Items {
+		if !p.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		p := p
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := r.Delete(ctx, &p); err != nil {
+				if kerrors.IsNotFound(err) || kerrors.IsGone(err) {
+					return
+				}
+				errs = append(errs, fmt.Errorf("delete pod %q error: %v", p.ObjectMeta.Name, err))
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			log.Error(err, "failed to remove runner-linked pod")
+		}
+		return errors.New("failed to remove some runner linked pods")
+	}
+
+	return nil
+}
+
+func (r *RunnerPodReconciler) cleanupRunnerLinkedSecrets(ctx context.Context, pod *corev1.Pod, log logr.Logger) error {
+	log.V(2).Info("Listing runner-linked secrets to be deleted", "ns", pod.Namespace)
+
+	var runnerLinkedSecretList corev1.SecretList
+	if err := r.List(ctx, &runnerLinkedSecretList, client.InNamespace(pod.Namespace), client.MatchingLabels(
+		map[string]string{
+			"runner-pod": pod.ObjectMeta.Name,
+		},
+	)); err != nil {
+		return fmt.Errorf("failed to list runner-linked secrets: %w", err)
+	}
+
+	var (
+		wg   sync.WaitGroup
+		errs []error
+	)
+	for _, s := range runnerLinkedSecretList.Items {
+		if !s.ObjectMeta.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		s := s
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := r.Delete(ctx, &s); err != nil {
+				if kerrors.IsNotFound(err) || kerrors.IsGone(err) {
+					return
+				}
+				errs = append(errs, fmt.Errorf("delete secret %q error: %v", s.ObjectMeta.Name, err))
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			log.Error(err, "failed to remove runner-linked secret")
+		}
+		return errors.New("failed to remove some runner linked secrets")
+	}
+
+	return nil
+}

controllers/actions.summerwind.net/runner_pod_owner.go

@@ -0,0 +1,600 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type podsForOwner struct {
+	total        int
+	completed    int
+	running      int
+	terminating  int
+	regTimeout   int
+	pending      int
+	templateHash string
+	runner       *v1alpha1.Runner
+	statefulSet  *appsv1.StatefulSet
+	owner        owner
+	object       client.Object
+	synced       bool
+	pods         []corev1.Pod
+}
+
+type owner interface {
+	client.Object
+
+	pods(context.Context, client.Client) ([]corev1.Pod, error)
+	templateHash() (string, bool)
+	withAnnotation(k, v string) client.Object
+	synced() bool
+}
+
+type ownerRunner struct {
+	client.Object
+
+	Log    logr.Logger
+	Runner *v1alpha1.Runner
+}
+
+var _ owner = (*ownerRunner)(nil)
+
+func (r *ownerRunner) pods(ctx context.Context, c client.Client) ([]corev1.Pod, error) {
+	var pod corev1.Pod
+
+	if err := c.Get(ctx, types.NamespacedName{Namespace: r.Runner.Namespace, Name: r.Runner.Name}, &pod); err != nil {
+		if errors.IsNotFound(err) {
+			return nil, nil
+		}
+		r.Log.Error(err, "Failed to get pod managed by runner")
+		return nil, err
+	}
+
+	return []corev1.Pod{pod}, nil
+}
+
+func (r *ownerRunner) templateHash() (string, bool) {
+	return getRunnerTemplateHash(r.Runner)
+}
+
+func (r *ownerRunner) withAnnotation(k, v string) client.Object {
+	copy := r.Runner.DeepCopy()
+	setAnnotation(&copy.ObjectMeta, k, v)
+	return copy
+}
+
+func (r *ownerRunner) synced() bool {
+	return r.Runner.Status.Phase != ""
+}
+
+type ownerStatefulSet struct {
+	client.Object
+
+	Log         logr.Logger
+	StatefulSet *appsv1.StatefulSet
+}
+
+var _ owner = (*ownerStatefulSet)(nil)
+
+func (s *ownerStatefulSet) pods(ctx context.Context, c client.Client) ([]corev1.Pod, error) {
+	var podList corev1.PodList
+
+	if err := c.List(ctx, &podList, client.MatchingLabels(s.StatefulSet.Spec.Template.ObjectMeta.Labels)); err != nil {
+		s.Log.Error(err, "Failed to list pods managed by statefulset")
+		return nil, err
+	}
+
+	var pods []corev1.Pod
+
+	for _, pod := range podList.Items {
+		if owner := metav1.GetControllerOf(&pod); owner == nil || owner.Kind != "StatefulSet" || owner.Name != s.StatefulSet.Name {
+			continue
+		}
+
+		pods = append(pods, pod)
+	}
+
+	return pods, nil
+}
+
+func (s *ownerStatefulSet) templateHash() (string, bool) {
+	return getRunnerTemplateHash(s.StatefulSet)
+}
+
+func (s *ownerStatefulSet) withAnnotation(k, v string) client.Object {
+	copy := s.StatefulSet.DeepCopy()
+	setAnnotation(&copy.ObjectMeta, k, v)
+	return copy
+}
+
+func (s *ownerStatefulSet) synced() bool {
+	var replicas int32 = 1
+	if s.StatefulSet.Spec.Replicas != nil {
+		replicas = *s.StatefulSet.Spec.Replicas
+	}
+
+	if s.StatefulSet.Status.Replicas != replicas {
+		s.Log.V(2).Info("Waiting for statefulset to sync", "desiredReplicas", replicas, "currentReplicas", s.StatefulSet.Status.Replicas)
+		return false
+	}
+
+	return true
+}
+
+func getPodsForOwner(ctx context.Context, c client.Client, log logr.Logger, o client.Object) (*podsForOwner, error) {
+	var (
+		owner       owner
+		runner      *v1alpha1.Runner
+		statefulSet *appsv1.StatefulSet
+		object      client.Object
+	)
+
+	switch v := o.(type) {
+	case *v1alpha1.Runner:
+		owner = &ownerRunner{
+			Log:    log,
+			Runner: v,
+			Object: v,
+		}
+		runner = v
+		object = v
+	case *appsv1.StatefulSet:
+		owner = &ownerStatefulSet{
+			Log:         log,
+			StatefulSet: v,
+			Object:      v,
+		}
+		statefulSet = v
+		object = v
+	default:
+		return nil, fmt.Errorf("BUG: Unsupported runner pods owner %v(%T)", v, v)
+	}
+
+	pods, err := owner.pods(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	var completed, running, terminating, regTimeout, pending, total int
+
+	for _, pod := range pods {
+		total++
+
+		if runnerPodOrContainerIsStopped(&pod) {
+			completed++
+		} else if pod.Status.Phase == corev1.PodRunning {
+			if podRunnerID(&pod) == "" && podConditionTransitionTimeAfter(&pod, corev1.PodReady, registrationTimeout) {
+				log.Info(
+					"Runner failed to register itself to GitHub in timely manner. "+
+						"Recreating the pod to see if it resolves the issue. "+
+						"CAUTION: If you see this a lot, you should investigate the root cause. "+
+						"See https://github.com/actions/actions-runner-controller/issues/288",
+					"creationTimestamp", pod.CreationTimestamp,
+					"readyTransitionTime", podConditionTransitionTime(&pod, corev1.PodReady, corev1.ConditionTrue),
+					"configuredRegistrationTimeout", registrationTimeout,
+				)
+
+				regTimeout++
+			} else {
+				running++
+			}
+		} else if !pod.DeletionTimestamp.IsZero() {
+			terminating++
+		} else {
+			// pending includes running but timedout runner's pod too
+			pending++
+		}
+	}
+
+	templateHash, ok := owner.templateHash()
+	if !ok {
+		log.Info("Failed to get template hash of statefulset. It must be in an invalid state. Please manually delete the statefulset so that it is recreated")
+
+		return nil, nil
+	}
+
+	synced := owner.synced()
+
+	return &podsForOwner{
+		total:        total,
+		completed:    completed,
+		running:      running,
+		terminating:  terminating,
+		regTimeout:   regTimeout,
+		pending:      pending,
+		templateHash: templateHash,
+		runner:       runner,
+		statefulSet:  statefulSet,
+		owner:        owner,
+		object:       object,
+		synced:       synced,
+		pods:         pods,
+	}, nil
+}
+
+func getRunnerTemplateHash(r client.Object) (string, bool) {
+	hash, ok := r.GetLabels()[LabelKeyRunnerTemplateHash]
+
+	return hash, ok
+}
+
+type state struct {
+	podsForOwners map[string][]*podsForOwner
+	lastSyncTime  *time.Time
+}
+
+type result struct {
+	currentObjects []*podsForOwner
+}
+
+// Why `create` must be a function rather than a client.Object? That's becase we use it to create one or more objects on scale up.
+//
+// We use client.Create to create a necessary number of client.Object. client.Create mutates the passed object on a successful creation.
+// It seems to set .Revision at least, and the existence of .Revision let client.Create fail due to K8s restriction that an object being just created
+// can't have .Revision.
+// Now, imagine that you are to add 2 runner replicas on scale up.
+// We create one resource object per a replica that ends up calling 2 client.Create calls.
+// If we were reusing client.Object to be passed to client.Create calls, only the first call suceeeds.
+// The second call fails due to the first call mutated the client.Object to have .Revision.
+// Passing a factory function of client.Object and creating a brand-new client.Object per a client.Create call resolves this issue,
+// allowing us to create two or more replicas in one reconcilation loop without being rejected by K8s.
+func syncRunnerPodsOwners(ctx context.Context, c client.Client, log logr.Logger, effectiveTime *metav1.Time, newDesiredReplicas int, create func() client.Object, ephemeral bool, owners []client.Object) (*result, error) {
+	state, err := collectPodsForOwners(ctx, c, log, owners)
+	if err != nil || state == nil {
+		return nil, err
+	}
+
+	podsForOwnersPerTemplateHash, lastSyncTime := state.podsForOwners, state.lastSyncTime
+
+	// # Why do we recreate statefulsets instead of updating their desired replicas?
+	//
+	// A statefulset cannot add more pods when not all the pods are running.
+	// Our ephemeral runners' pods that have finished running become Completed(Phase=Succeeded).
+	// So creating one statefulset per a batch of ephemeral runners is the only way for us to add more replicas.
+	//
+	// # Why do we recreate statefulsets instead of updating fields other than replicas?
+	//
+	// That's because Kubernetes doesn't allow updating anything other than replicas, template, and updateStrategy.
+	// And the nature of ephemeral runner pods requires you to create a statefulset per a batch of new runner pods so
+	// we have really no other choice.
+	//
+	// If you're curious, the below is the error message you will get when you tried to update forbidden StatefulSet field(s):
+	//
+	// 2021-06-13T07:19:52.760Z        ERROR   actions-runner-controller.runnerset     Failed to patch statefulset
+	// {"runnerset": "default/example-runnerset", "error": "StatefulSet.apps \"example-runnerset\" is invalid: s
+	// pec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'template', and 'updateStrategy'
+	// are forbidden"}
+	//
+	// Even though the error message includes "Forbidden", this error's reason is "Invalid".
+	// So we used to match these errors by using errors.IsInvalid. But that's another story...
+
+	desiredTemplateHash, ok := getRunnerTemplateHash(create())
+	if !ok {
+		log.Info("Failed to get template hash of desired owner resource. It must be in an invalid state. Please manually delete the owner so that it is recreated")
+
+		return nil, nil
+	}
+
+	currentObjects := podsForOwnersPerTemplateHash[desiredTemplateHash]
+
+	sort.SliceStable(currentObjects, func(i, j int) bool {
+		return currentObjects[i].owner.GetCreationTimestamp().Time.Before(currentObjects[j].owner.GetCreationTimestamp().Time)
+	})
+
+	if len(currentObjects) > 0 {
+		timestampFirst := currentObjects[0].owner.GetCreationTimestamp()
+		timestampLast := currentObjects[len(currentObjects)-1].owner.GetCreationTimestamp()
+		var names []string
+		for _, ss := range currentObjects {
+			names = append(names, ss.owner.GetName())
+		}
+		log.V(2).Info("Detected some current object(s)", "creationTimestampFirst", timestampFirst, "creationTimestampLast", timestampLast, "names", names)
+	}
+
+	var total, terminating, pending, running, regTimeout int
+
+	for _, ss := range currentObjects {
+		total += ss.total
+		terminating += ss.terminating
+		pending += ss.pending
+		running += ss.running
+		regTimeout += ss.regTimeout
+	}
+
+	numOwners := len(owners)
+
+	var hashes []string
+	for h := range state.podsForOwners {
+		hashes = append(hashes, h)
+	}
+
+	log.V(2).Info(
+		"Found some pods across owner(s)",
+		"total", total,
+		"terminating", terminating,
+		"pending", pending,
+		"running", running,
+		"regTimeout", regTimeout,
+		"desired", newDesiredReplicas,
+		"owners", numOwners,
+	)
+
+	maybeRunning := pending + running
+
+	wantMoreRunners := newDesiredReplicas > maybeRunning
+	alreadySyncedAfterEffectiveTime := ephemeral && lastSyncTime != nil && effectiveTime != nil && lastSyncTime.After(effectiveTime.Time)
+	runnerPodRecreationDelayAfterWebhookScale := lastSyncTime != nil && time.Now().Before(lastSyncTime.Add(DefaultRunnerPodRecreationDelayAfterWebhookScale))
+
+	log = log.WithValues(
+		"lastSyncTime", lastSyncTime,
+		"effectiveTime", effectiveTime,
+		"templateHashDesired", desiredTemplateHash,
+		"replicasDesired", newDesiredReplicas,
+		"replicasPending", pending,
+		"replicasRunning", running,
+		"replicasMaybeRunning", maybeRunning,
+		"templateHashObserved", hashes,
+	)
+
+	if wantMoreRunners && alreadySyncedAfterEffectiveTime && runnerPodRecreationDelayAfterWebhookScale {
+		// This is our special handling of the situation for ephemeral runners only.
+		//
+		// Handling static runners this way results in scale-up to not work at all,
+		// because then any scale up attempts for static runenrs fall within this condition, for two reasons.
+		// First, static(persistent) runners will never restart on their own.
+		// Second, we don't update EffectiveTime for static runners.
+		//
+		// We do need to skip this condition for static runners, and that's why we take the `ephemeral` flag into account when
+		// computing `alreadySyncedAfterEffectiveTime``.
+
+		log.V(2).Info(
+			"Detected that some ephemeral runners have disappeared. " +
+				"Usually this is due to that ephemeral runner completions " +
+				"so ARC does not create new runners until EffectiveTime is updated, or DefaultRunnerPodRecreationDelayAfterWebhookScale is elapsed.")
+	} else if wantMoreRunners {
+		if alreadySyncedAfterEffectiveTime && !runnerPodRecreationDelayAfterWebhookScale {
+			log.V(2).Info("Adding more replicas because DefaultRunnerPodRecreationDelayAfterWebhookScale has been passed")
+		}
+
+		num := newDesiredReplicas - maybeRunning
+
+		for i := 0; i < num; i++ {
+			// Add more replicas
+			if err := c.Create(ctx, create()); err != nil {
+				return nil, err
+			}
+		}
+
+		log.V(1).Info("Created replica(s)",
+			"created", num,
+		)
+
+		return nil, nil
+	} else if newDesiredReplicas <= running {
+		// If you use ephemeral runners with webhook-based autoscaler and the runner controller is working normally,
+		// you're unlikely to fall into this branch.
+		//
+		// That's because all the stakeholders work like this:
+		//
+		// 1. A runner pod completes with the runner container exiting with code 0
+		// 2. ARC runner controller detects the pod completion, marks the owner(runner or statefulset) resource on k8s for deletion (=Runner.DeletionTimestamp becomes non-zero)
+		// 3. GitHub triggers a corresponding workflow_job "complete" webhook event
+		// 4. ARC github-webhook-server (webhook-based autoscaler) receives the webhook event updates HRA with removing the oldest capacity reservation
+		// 5. ARC horizontalrunnerautoscaler updates RunnerDeployment's desired replicas based on capacity reservations
+		// 6. ARC runnerdeployment controller updates RunnerReplicaSet's desired replicas
+		// 7. (We're here) ARC runnerset or runnerreplicaset controller starts reconciling the owner resource (statefulset or runner)
+		//
+		// In a normally working ARC installation, the runner that was used to run the workflow job should already have been
+		// marked for deletion by the runner controller.
+		// This runnerreplicaset controller doesn't count marked runners into the `running` value, hence you're unlikely to
+		// fall into this branch when you're using ephemeral runners with webhook-based-autoscaler.
+
+		var retained int
+
+		var delete []*podsForOwner
+		for i := len(currentObjects) - 1; i >= 0; i-- {
+			ss := currentObjects[i]
+
+			if ss.running == 0 || retained >= newDesiredReplicas {
+				// In case the desired replicas is satisfied until i-1, or this owner has no running pods,
+				// this owner can be considered safe for deletion.
+				// Note that we already waited on this owner to create pods by waiting for
+				// `.Status.Replicas`(=total number of pods managed by owner, regardless of the runner is Running or Completed) to match the desired replicas in a previous step.
+				// So `.running == 0` means "the owner has created the desired number of pods before, and all of them are completed now".
+				delete = append(delete, ss)
+			} else if retained < newDesiredReplicas {
+				retained += ss.running
+			}
+		}
+
+		if retained == newDesiredReplicas {
+			for _, ss := range delete {
+				log := log.WithValues("owner", types.NamespacedName{Namespace: ss.owner.GetNamespace(), Name: ss.owner.GetName()})
+				// Statefulset termination process 1/4: Set unregistrationRequestTimestamp only after all the pods managed by the statefulset have
+				// started unregistreation process.
+				//
+				// NOTE: We just mark it instead of immediately starting the deletion process.
+				// Otherwise, the runner pod may hit termiationGracePeriod before the unregistration completes(the max terminationGracePeriod is limited to 1h by K8s and a job can be run for more than that),
+				// or actions/runner may potentially misbehave on SIGTERM immediately sent by K8s.
+				// We'd better unregister first and then start a pod deletion process.
+				// The annotation works as a mark to start the pod unregistration and deletion process of ours.
+
+				if _, ok := getAnnotation(ss.owner, AnnotationKeyUnregistrationRequestTimestamp); ok {
+					log.V(2).Info("Still waiting for runner pod(s) unregistration to complete")
+
+					continue
+				}
+
+				for _, po := range ss.pods {
+					if _, err := annotatePodOnce(ctx, c, log, &po, AnnotationKeyUnregistrationRequestTimestamp, time.Now().Format(time.RFC3339)); err != nil {
+						return nil, err
+					}
+				}
+
+				updated := ss.owner.withAnnotation(AnnotationKeyUnregistrationRequestTimestamp, time.Now().Format(time.RFC3339))
+				if err := c.Patch(ctx, updated, client.MergeFrom(ss.owner)); err != nil {
+					log.Error(err, fmt.Sprintf("Failed to patch owner to have %s annotation", AnnotationKeyUnregistrationRequestTimestamp))
+					return nil, err
+				}
+
+				log.V(2).Info("Redundant owner has been annotated to start the unregistration before deletion")
+			}
+		} else if retained > newDesiredReplicas {
+			log.V(2).Info("Waiting sync before scale down", "retained", retained, "newDesiredReplicas", newDesiredReplicas)
+
+			return nil, nil
+		} else {
+			log.Info("Invalid state", "retained", retained, "newDesiredReplicas", newDesiredReplicas)
+			panic("crashed due to invalid state")
+		}
+	}
+
+	for _, sss := range podsForOwnersPerTemplateHash {
+		for _, ss := range sss {
+			if ss.templateHash != desiredTemplateHash {
+				if ss.owner.GetDeletionTimestamp().IsZero() {
+					if err := c.Delete(ctx, ss.object); err != nil {
+						log.Error(err, "Unable to delete object")
+						return nil, err
+					}
+
+					log.V(2).Info("Deleted redundant and outdated object")
+				}
+
+				return nil, nil
+			}
+		}
+	}
+
+	return &result{
+		currentObjects: currentObjects,
+	}, nil
+}
+
+func collectPodsForOwners(ctx context.Context, c client.Client, log logr.Logger, owners []client.Object) (*state, error) {
+	podsForOwnerPerTemplateHash := map[string][]*podsForOwner{}
+
+	// lastSyncTime becomes non-nil only when there are one or more owner(s) hence there are same number of runner pods.
+	// It's used to prevent runnerset-controller from recreating "completed ephemeral runners".
+	// This is needed to prevent runners from being terminated prematurely.
+	// See https://github.com/actions/actions-runner-controller/issues/911 for more context.
+	//
+	// This becomes nil when there are zero statefulset(s). That's fine because then there should be zero stateful(s) to be recreated either hence
+	// we don't need to guard with lastSyncTime.
+	var lastSyncTime *time.Time
+
+	for _, ss := range owners {
+		log := log.WithValues("owner", types.NamespacedName{Namespace: ss.GetNamespace(), Name: ss.GetName()})
+
+		res, err := getPodsForOwner(ctx, c, log, ss)
+		if err != nil {
+			return nil, err
+		}
+
+		if res.templateHash == "" {
+			log.Info("validation error: runner pod owner must have template hash", "object", res.object)
+
+			return nil, nil
+		}
+
+		// Statefulset termination process 4/4: Let Kubernetes cascade-delete the statefulset and the pods.
+		//
+		// If the runner is already marked for deletion(=has a non-zero deletion timestamp) by the runner controller (can be caused by an ephemeral runner completion)
+		// or by this controller (in case it was deleted in the previous reconcilation loop),
+		// we don't need to bother calling GitHub API to re-mark the runner for deletion.
+		// Just hold on, and runners will disappear as long as the runner controller is up and running.
+		if !res.owner.GetDeletionTimestamp().IsZero() {
+			continue
+		}
+
+		// Statefulset termination process 3/4: Set the deletionTimestamp to let Kubernetes start a cascade deletion of the statefulset and the pods.
+		if _, ok := getAnnotation(res.owner, AnnotationKeyUnregistrationCompleteTimestamp); ok {
+			if err := c.Delete(ctx, res.object); err != nil {
+				log.Error(err, "Failed to delete owner")
+				return nil, err
+			}
+
+			log.V(2).Info("Started deletion of owner")
+
+			continue
+		}
+
+		// Statefulset termination process 2/4: Set unregistrationCompleteTimestamp only if all the pods managed by the statefulset
+		// have either unregistered or being deleted.
+		if _, ok := getAnnotation(res.owner, AnnotationKeyUnregistrationRequestTimestamp); ok {
+			var deletionSafe int
+			for _, po := range res.pods {
+				if _, ok := getAnnotation(&po, AnnotationKeyUnregistrationCompleteTimestamp); ok {
+					deletionSafe++
+				} else if !po.DeletionTimestamp.IsZero() {
+					deletionSafe++
+				}
+			}
+
+			if deletionSafe == res.total {
+				log.V(2).Info("Marking owner for unregistration completion", "deletionSafe", deletionSafe, "total", res.total)
+
+				if _, ok := getAnnotation(res.owner, AnnotationKeyUnregistrationCompleteTimestamp); !ok {
+					updated := res.owner.withAnnotation(AnnotationKeyUnregistrationCompleteTimestamp, time.Now().Format(time.RFC3339))
+
+					if err := c.Patch(ctx, updated, client.MergeFrom(res.owner)); err != nil {
+						log.Error(err, fmt.Sprintf("Failed to patch owner to have %s annotation", AnnotationKeyUnregistrationCompleteTimestamp))
+						return nil, err
+					}
+
+					log.V(2).Info("Redundant owner has been annotated to start the deletion")
+				} else {
+					log.V(2).Info("BUG: Redundant owner was already annotated to start the deletion")
+				}
+
+				continue
+			}
+		}
+
+		if annotations := res.owner.GetAnnotations(); annotations != nil {
+			if a, ok := annotations[SyncTimeAnnotationKey]; ok {
+				t, err := time.Parse(time.RFC3339, a)
+				if err == nil {
+					if lastSyncTime == nil || lastSyncTime.Before(t) {
+						lastSyncTime = &t
+					}
+				}
+			}
+		}
+
+		// A completed owner and a completed runner pod can safely be deleted without
+		// a race condition so delete it here,
+		// so that the later process can be a bit simpler.
+		if res.total > 0 && res.total == res.completed {
+			if err := c.Delete(ctx, ss); err != nil {
+				log.Error(err, "Unable to delete owner")
+				return nil, err
+			}
+
+			log.V(2).Info("Deleted completed owner")
+
+			return nil, nil
+		}
+
+		if !res.synced {
+			log.V(1).Info("Skipped reconcilation because owner is not synced yet", "pods", res.pods)
+
+			return nil, nil
+		}
+
+		podsForOwnerPerTemplateHash[res.templateHash] = append(podsForOwnerPerTemplateHash[res.templateHash], res)
+	}
+
+	return &state{podsForOwnerPerTemplateHash, lastSyncTime}, nil
+}

controllers/actions.summerwind.net/runnerdeployment_controller.go

@@ -0,0 +1,508 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"reflect"
+	"sort"
+	"time"
+
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/controllers/actions.summerwind.net/metrics"
+)
+
+const (
+	LabelKeyRunnerTemplateHash   = "runner-template-hash"
+	LabelKeyRunnerDeploymentName = "runner-deployment-name"
+
+	runnerSetOwnerKey = ".metadata.controller"
+)
+
+// RunnerDeploymentReconciler reconciles a Runner object
+type RunnerDeploymentReconciler struct {
+	client.Client
+	Log                logr.Logger
+	Recorder           record.EventRecorder
+	Scheme             *runtime.Scheme
+	CommonRunnerLabels []string
+	Name               string
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("runnerdeployment", req.NamespacedName)
+
+	var rd v1alpha1.RunnerDeployment
+	if err := r.Get(ctx, req.NamespacedName, &rd); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !rd.ObjectMeta.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	metrics.SetRunnerDeployment(rd)
+
+	var myRunnerReplicaSetList v1alpha1.RunnerReplicaSetList
+	if err := r.List(ctx, &myRunnerReplicaSetList, client.InNamespace(req.Namespace), client.MatchingFields{runnerSetOwnerKey: req.Name}); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	myRunnerReplicaSets := myRunnerReplicaSetList.Items
+
+	sort.Slice(myRunnerReplicaSets, func(i, j int) bool {
+		return myRunnerReplicaSets[i].GetCreationTimestamp().After(myRunnerReplicaSets[j].GetCreationTimestamp().Time)
+	})
+
+	var newestSet *v1alpha1.RunnerReplicaSet
+
+	var oldSets []v1alpha1.RunnerReplicaSet
+
+	if len(myRunnerReplicaSets) > 0 {
+		newestSet = &myRunnerReplicaSets[0]
+	}
+
+	if len(myRunnerReplicaSets) > 1 {
+		oldSets = myRunnerReplicaSets[1:]
+	}
+
+	desiredRS, err := r.newRunnerReplicaSet(rd)
+	if err != nil {
+		r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
+
+		log.Error(err, "Could not create runnerreplicaset")
+
+		return ctrl.Result{}, err
+	}
+
+	if newestSet == nil {
+		if err := r.Client.Create(ctx, desiredRS); err != nil {
+			log.Error(err, "Failed to create runnerreplicaset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		log.Info("Created runnerreplicaset", "runnerreplicaset", desiredRS.Name)
+
+		return ctrl.Result{}, nil
+	}
+
+	newestTemplateHash, ok := getTemplateHash(newestSet)
+	if !ok {
+		log.Info("Failed to get template hash of newest runnerreplicaset resource. It must be in an invalid state. Please manually delete the runnerreplicaset so that it is recreated")
+
+		return ctrl.Result{}, nil
+	}
+
+	desiredTemplateHash, ok := getTemplateHash(desiredRS)
+	if !ok {
+		log.Info("Failed to get template hash of desired runnerreplicaset resource. It must be in an invalid state. Please manually delete the runnerreplicaset so that it is recreated")
+
+		return ctrl.Result{}, nil
+	}
+
+	if newestTemplateHash != desiredTemplateHash {
+		if err := r.Client.Create(ctx, desiredRS); err != nil {
+			log.Error(err, "Failed to create runnerreplicaset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		log.Info("Created runnerreplicaset", "runnerreplicaset", desiredRS.Name)
+
+		// We requeue in order to clean up old runner replica sets later.
+		// Otherwise, they aren't cleaned up until the next re-sync interval.
+		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+	}
+
+	if !reflect.DeepEqual(newestSet.Spec.Selector, desiredRS.Spec.Selector) {
+		updateSet := newestSet.DeepCopy()
+		updateSet.Spec = *desiredRS.Spec.DeepCopy()
+
+		// A selector update change doesn't trigger replicaset replacement,
+		// but we still need to update the existing replicaset with it.
+		// Otherwise selector-based runner query will never work on replicasets created before the controller v0.17.0
+		// See https://github.com/actions/actions-runner-controller/pull/355#discussion_r585379259
+		if err := r.Client.Update(ctx, updateSet); err != nil {
+			log.Error(err, "Failed to update runnerreplicaset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		log.V(1).Info("Updated runnerreplicaset due to selector change")
+
+		// At this point, we are already sure that there's no need to create a new replicaset
+		// as the runner template hash is not changed.
+		//
+		// But we still need to requeue for the (possibly rare) cases that there are still old replicasets that needs
+		// to be cleaned up.
+		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+	}
+
+	const defaultReplicas = 1
+
+	currentDesiredReplicas := getIntOrDefault(newestSet.Spec.Replicas, defaultReplicas)
+	newDesiredReplicas := getIntOrDefault(desiredRS.Spec.Replicas, defaultReplicas)
+
+	// Please add more conditions that we can in-place update the newest runnerreplicaset without disruption
+	//
+	// If we missed taking the EffectiveTime diff into account, you might end up experiencing scale-ups being delayed scale-down.
+	// See https://github.com/actions/actions-runner-controller/pull/1477#issuecomment-1164154496
+	var et1, et2 time.Time
+	if newestSet.Spec.EffectiveTime != nil {
+		et1 = newestSet.Spec.EffectiveTime.Time
+	}
+	if rd.Spec.EffectiveTime != nil {
+		et2 = rd.Spec.EffectiveTime.Time
+	}
+	if currentDesiredReplicas != newDesiredReplicas || et1 != et2 {
+		newestSet.Spec.Replicas = &newDesiredReplicas
+		newestSet.Spec.EffectiveTime = rd.Spec.EffectiveTime
+
+		if err := r.Client.Update(ctx, newestSet); err != nil {
+			log.Error(err, "Failed to update runnerreplicaset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		log.V(1).Info("Updated runnerreplicaset due to spec change",
+			"currentDesiredReplicas", currentDesiredReplicas,
+			"newDesiredReplicas", newDesiredReplicas,
+			"currentEffectiveTime", newestSet.Spec.EffectiveTime,
+			"newEffectiveTime", rd.Spec.EffectiveTime,
+		)
+
+		return ctrl.Result{}, err
+	}
+
+	// Do we have old runner replica sets that should eventually deleted?
+	if len(oldSets) > 0 {
+		var readyReplicas int
+		if newestSet.Status.ReadyReplicas != nil {
+			readyReplicas = *newestSet.Status.ReadyReplicas
+		}
+
+		oldSetsCount := len(oldSets)
+
+		logWithDebugInfo := log.WithValues(
+			"newest_runnerreplicaset", types.NamespacedName{
+				Namespace: newestSet.Namespace,
+				Name:      newestSet.Name,
+			},
+			"newest_runnerreplicaset_replicas_ready", readyReplicas,
+			"newest_runnerreplicaset_replicas_desired", currentDesiredReplicas,
+			"old_runnerreplicasets_count", oldSetsCount,
+		)
+
+		if readyReplicas < currentDesiredReplicas {
+			logWithDebugInfo.
+				Info("Waiting until the newest runnerreplicaset to be 100% available")
+
+			return ctrl.Result{}, nil
+		}
+
+		if oldSetsCount > 0 {
+			logWithDebugInfo.
+				Info("The newest runnerreplicaset is 100% available. Deleting old runnerreplicasets")
+		}
+
+		for i := range oldSets {
+			rs := oldSets[i]
+
+			rslog := log.WithValues("runnerreplicaset", rs.Name)
+
+			if rs.Status.Replicas != nil && *rs.Status.Replicas > 0 {
+				if rs.Spec.Replicas != nil && *rs.Spec.Replicas == 0 {
+					rslog.V(2).Info("Waiting for runnerreplicaset to scale to zero")
+
+					continue
+				}
+
+				updated := rs.DeepCopy()
+				zero := 0
+				updated.Spec.Replicas = &zero
+				if err := r.Client.Update(ctx, updated); err != nil {
+					rslog.Error(err, "Failed to scale runnerreplicaset to zero")
+
+					return ctrl.Result{}, err
+				}
+
+				rslog.Info("Scaled runnerreplicaset to zero")
+
+				continue
+			}
+
+			if err := r.Client.Delete(ctx, &rs); err != nil {
+				rslog.Error(err, "Failed to delete runnerreplicaset resource")
+
+				return ctrl.Result{}, err
+			}
+
+			r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerReplicaSetDeleted", fmt.Sprintf("Deleted runnerreplicaset '%s'", rs.Name))
+
+			rslog.Info("Deleted runnerreplicaset")
+		}
+	}
+
+	var replicaSets []v1alpha1.RunnerReplicaSet
+
+	replicaSets = append(replicaSets, *newestSet)
+	replicaSets = append(replicaSets, oldSets...)
+
+	var totalCurrentReplicas, totalStatusAvailableReplicas, updatedReplicas int
+
+	for _, rs := range replicaSets {
+		var current, available int
+
+		if rs.Status.Replicas != nil {
+			current = *rs.Status.Replicas
+		}
+
+		if rs.Status.AvailableReplicas != nil {
+			available = *rs.Status.AvailableReplicas
+		}
+
+		totalCurrentReplicas += current
+		totalStatusAvailableReplicas += available
+	}
+
+	if newestSet.Status.Replicas != nil {
+		updatedReplicas = *newestSet.Status.Replicas
+	}
+
+	var status v1alpha1.RunnerDeploymentStatus
+
+	status.AvailableReplicas = &totalStatusAvailableReplicas
+	status.ReadyReplicas = &totalStatusAvailableReplicas
+	status.DesiredReplicas = &newDesiredReplicas
+	status.Replicas = &totalCurrentReplicas
+	status.UpdatedReplicas = &updatedReplicas
+
+	if !reflect.DeepEqual(rd.Status, status) {
+		updated := rd.DeepCopy()
+		updated.Status = status
+
+		if err := r.Status().Patch(ctx, updated, client.MergeFrom(&rd)); err != nil {
+			log.Info("Failed to patch runnerdeployment status. Retrying immediately", "error", err.Error())
+			return ctrl.Result{
+				Requeue: true,
+			}, nil
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func getIntOrDefault(p *int, d int) int {
+	if p == nil {
+		return d
+	}
+
+	return *p
+}
+
+func getTemplateHash(rs *v1alpha1.RunnerReplicaSet) (string, bool) {
+	hash, ok := rs.Labels[LabelKeyRunnerTemplateHash]
+
+	return hash, ok
+}
+
+// ComputeHash returns a hash value calculated from pod template and
+// a collisionCount to avoid hash collision. The hash will be safe encoded to
+// avoid bad words.
+//
+// Proudly modified and adopted from k8s.io/kubernetes/pkg/util/hash.DeepHashObject and
+// k8s.io/kubernetes/pkg/controller.ComputeHash.
+func ComputeHash(template interface{}) string {
+	hasher := fnv.New32a()
+
+	hasher.Reset()
+
+	printer := spew.ConfigState{
+		Indent:         " ",
+		SortKeys:       true,
+		DisableMethods: true,
+		SpewKeys:       true,
+	}
+	printer.Fprintf(hasher, "%#v", template)
+
+	return rand.SafeEncodeString(fmt.Sprint(hasher.Sum32()))
+}
+
+// Clones the given map and returns a new map with the given key and value added.
+// Returns the given map, if labelKey is empty.
+//
+// Proudly copied from k8s.io/kubernetes/pkg/util/labels.CloneAndAddLabel
+func CloneAndAddLabel(labels map[string]string, labelKey, labelValue string) map[string]string {
+	if labelKey == "" {
+		// Don't need to add a label.
+		return labels
+	}
+	// Clone.
+	newLabels := map[string]string{}
+	for key, value := range labels {
+		newLabels[key] = value
+	}
+	newLabels[labelKey] = labelValue
+	return newLabels
+}
+
+// Clones the given selector and returns a new selector with the given key and value added.
+// Returns the given selector, if labelKey is empty.
+//
+// Proudly copied from k8s.io/kubernetes/pkg/util/labels.CloneSelectorAndAddLabel
+func CloneSelectorAndAddLabel(selector *metav1.LabelSelector, labelKey, labelValue string) *metav1.LabelSelector {
+	if labelKey == "" {
+		// Don't need to add a label.
+		return selector
+	}
+
+	// Clone.
+	newSelector := new(metav1.LabelSelector)
+
+	newSelector.MatchLabels = make(map[string]string)
+	if selector.MatchLabels != nil {
+		for key, val := range selector.MatchLabels {
+			newSelector.MatchLabels[key] = val
+		}
+	}
+	newSelector.MatchLabels[labelKey] = labelValue
+
+	if selector.MatchExpressions != nil {
+		newMExps := make([]metav1.LabelSelectorRequirement, len(selector.MatchExpressions))
+		for i, me := range selector.MatchExpressions {
+			newMExps[i].Key = me.Key
+			newMExps[i].Operator = me.Operator
+			if me.Values != nil {
+				newMExps[i].Values = make([]string, len(me.Values))
+				copy(newMExps[i].Values, me.Values)
+			} else {
+				newMExps[i].Values = nil
+			}
+		}
+		newSelector.MatchExpressions = newMExps
+	} else {
+		newSelector.MatchExpressions = nil
+	}
+
+	return newSelector
+}
+
+func (r *RunnerDeploymentReconciler) newRunnerReplicaSet(rd v1alpha1.RunnerDeployment) (*v1alpha1.RunnerReplicaSet, error) {
+	return newRunnerReplicaSet(&rd, r.CommonRunnerLabels, r.Scheme)
+}
+
+func getSelector(rd *v1alpha1.RunnerDeployment) *metav1.LabelSelector {
+	selector := rd.Spec.Selector
+	if selector == nil {
+		selector = &metav1.LabelSelector{MatchLabels: map[string]string{LabelKeyRunnerDeploymentName: rd.Name}}
+	}
+
+	return selector
+}
+
+func newRunnerReplicaSet(rd *v1alpha1.RunnerDeployment, commonRunnerLabels []string, scheme *runtime.Scheme) (*v1alpha1.RunnerReplicaSet, error) {
+	newRSTemplate := *rd.Spec.Template.DeepCopy()
+
+	newRSTemplate.Spec.Labels = append(newRSTemplate.Spec.Labels, commonRunnerLabels...)
+
+	templateHash := ComputeHash(&newRSTemplate)
+
+	// Add template hash label to selector.
+	newRSTemplate.ObjectMeta.Labels = CloneAndAddLabel(newRSTemplate.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash)
+
+	// This label selector is used by default when rd.Spec.Selector is empty.
+	newRSTemplate.ObjectMeta.Labels = CloneAndAddLabel(newRSTemplate.ObjectMeta.Labels, LabelKeyRunnerDeploymentName, rd.Name)
+
+	selector := getSelector(rd)
+
+	newRSSelector := CloneSelectorAndAddLabel(selector, LabelKeyRunnerTemplateHash, templateHash)
+
+	rs := v1alpha1.RunnerReplicaSet{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: rd.ObjectMeta.Name + "-",
+			Namespace:    rd.ObjectMeta.Namespace,
+			Labels:       newRSTemplate.ObjectMeta.Labels,
+		},
+		Spec: v1alpha1.RunnerReplicaSetSpec{
+			Replicas:      rd.Spec.Replicas,
+			Selector:      newRSSelector,
+			Template:      newRSTemplate,
+			EffectiveTime: rd.Spec.EffectiveTime,
+		},
+	}
+
+	if err := ctrl.SetControllerReference(rd, &rs, scheme); err != nil {
+		return &rs, err
+	}
+
+	return &rs, nil
+}
+
+func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerdeployment-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &v1alpha1.RunnerReplicaSet{}, runnerSetOwnerKey, func(rawObj client.Object) []string {
+		runnerSet := rawObj.(*v1alpha1.RunnerReplicaSet)
+		owner := metav1.GetControllerOf(runnerSet)
+		if owner == nil {
+			return nil
+		}
+
+		if owner.APIVersion != v1alpha1.GroupVersion.String() || owner.Kind != "RunnerDeployment" {
+			return nil
+		}
+
+		return []string{owner.Name}
+	}); err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.RunnerDeployment{}).
+		Owns(&v1alpha1.RunnerReplicaSet{}).
+		Named(name).
+		Complete(r)
+}

controllers/actions.summerwind.net/runnerdeployment_controller_test.go

@@ -0,0 +1,496 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+)
+
+func TestNewRunnerReplicaSet(t *testing.T) {
+	scheme := runtime.NewScheme()
+	if err := actionsv1alpha1.AddToScheme(scheme); err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	r := &RunnerDeploymentReconciler{
+		CommonRunnerLabels: []string{"dev"},
+		Scheme:             scheme,
+	}
+	rd := actionsv1alpha1.RunnerDeployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "example",
+		},
+		Spec: actionsv1alpha1.RunnerDeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"foo": "bar",
+				},
+			},
+			Template: actionsv1alpha1.RunnerTemplate{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"foo": "bar",
+					},
+				},
+				Spec: actionsv1alpha1.RunnerSpec{
+					RunnerConfig: actionsv1alpha1.RunnerConfig{
+						Labels: []string{"project1"},
+					},
+				},
+			},
+		},
+	}
+
+	rs, err := r.newRunnerReplicaSet(rd)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	if val, ok := rs.Labels["foo"]; ok {
+		if val != "bar" {
+			t.Errorf("foo label does not have bar but %v", val)
+		}
+	} else {
+		t.Errorf("foo label does not exist")
+	}
+
+	hash1, ok := rs.Labels[LabelKeyRunnerTemplateHash]
+	if !ok {
+		t.Errorf("missing runner-template-hash label")
+	}
+
+	runnerLabel := []string{"project1", "dev"}
+	if d := cmp.Diff(runnerLabel, rs.Spec.Template.Spec.Labels); d != "" {
+		t.Errorf("%s", d)
+	}
+
+	rd2 := rd.DeepCopy()
+	rd2.Spec.Template.Spec.Labels = []string{"project2"}
+
+	rs2, err := r.newRunnerReplicaSet(*rd2)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	hash2, ok := rs2.Labels[LabelKeyRunnerTemplateHash]
+	if !ok {
+		t.Errorf("missing runner-template-hash label")
+	}
+
+	if hash1 == hash2 {
+		t.Errorf(
+			"runner replica sets from runner deployments with varying labels must have different template hash, but got %s and %s",
+			hash1, hash2,
+		)
+	}
+
+	rd3 := rd.DeepCopy()
+	rd3.Spec.Template.Labels["foo"] = "baz"
+
+	rs3, err := r.newRunnerReplicaSet(*rd3)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	hash3, ok := rs3.Labels[LabelKeyRunnerTemplateHash]
+	if !ok {
+		t.Errorf("missing runner-template-hash label")
+	}
+
+	if hash1 == hash3 {
+		t.Errorf(
+			"runner replica sets from runner deployments with varying meta labels must have different template hash, but got %s and %s",
+			hash1, hash3,
+		)
+	}
+}
+
+// SetupDeploymentTest will set up a testing environment.
+// This includes:
+// * creating a Namespace to be used during the test
+// * starting the 'RunnerDeploymentReconciler'
+// * stopping the 'RunnerDeploymentReconciler" after the test ends
+// Call this function at the start of each of your tests.
+func SetupDeploymentTest(ctx2 context.Context) *corev1.Namespace {
+	var ctx context.Context
+	var cancel func()
+	ns := &corev1.Namespace{}
+
+	BeforeEach(func() {
+		ctx, cancel = context.WithCancel(ctx2)
+		*ns = corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)},
+		}
+
+		err := k8sClient.Create(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+			Namespace: ns.Name,
+		})
+		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
+
+		controller := &RunnerDeploymentReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     "runnerdeployment-" + ns.Name,
+		}
+		err = controller.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		go func() {
+			defer GinkgoRecover()
+
+			err := mgr.Start(ctx)
+			Expect(err).NotTo(HaveOccurred(), "failed to start manager")
+		}()
+	})
+
+	AfterEach(func() {
+		defer cancel()
+
+		err := k8sClient.Delete(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
+	})
+
+	return ns
+}
+
+var _ = Context("Inside of a new namespace", func() {
+	ctx := context.TODO()
+	ns := SetupDeploymentTest(ctx)
+
+	Describe("when no existing resources exist", func() {
+
+		It("should create a new RunnerReplicaSet resource from the specified template, add a another RunnerReplicaSet on template modification, and eventually removes old runnerreplicasets", func() {
+			name := "example-runnerdeploy-1"
+
+			{
+				rs := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
+						if err != nil {
+							return 0, err
+						}
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
+						if err != nil {
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas, nil
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+			}
+
+			{
+				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+				var rd actionsv1alpha1.RunnerDeployment
+				Eventually(func() error {
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
+					if err != nil {
+						return fmt.Errorf("failed to get test RunnerReplicaSet resource: %v\n", err)
+					}
+					rd.Spec.Replicas = intPtr(2)
+
+					return k8sClient.Update(ctx, &rd)
+				},
+					time.Second*1, time.Millisecond*500).Should(BeNil())
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rd.Spec.Selector)
+						if err != nil {
+							return 0, err
+						}
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
+						if err != nil {
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas, nil
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
+			}
+		})
+
+		It("should create a new RunnerReplicaSet resource from the specified template without labels and selector, add a another RunnerReplicaSet on template modification, and eventually removes old runnerreplicasets", func() {
+			name := "example-runnerdeploy-2"
+
+			{
+				rs := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
+						if err != nil {
+							return 0, err
+						}
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
+						if err != nil {
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas, nil
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+			}
+
+			{
+				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+				var rd actionsv1alpha1.RunnerDeployment
+				Eventually(func() error {
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
+					if err != nil {
+						return fmt.Errorf("failed to get test RunnerReplicaSet resource: %v\n", err)
+					}
+					rd.Spec.Replicas = intPtr(2)
+
+					return k8sClient.Update(ctx, &rd)
+				},
+					time.Second*1, time.Millisecond*500).Should(BeNil())
+
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() (int, error) {
+						selector, err := metav1.LabelSelectorAsSelector(rd.Spec.Selector)
+						if err != nil {
+							return 0, err
+						}
+						err = k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+							client.MatchingLabelsSelector{Selector: selector},
+						)
+						if err != nil {
+							return 0, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas, nil
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
+			}
+		})
+
+		It("should adopt RunnerReplicaSet created before 0.18.0 to have Spec.Selector", func() {
+			name := "example-runnerdeploy-2"
+
+			{
+				rd := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				createRDErr := k8sClient.Create(ctx, rd)
+				Expect(createRDErr).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				Eventually(
+					func() (int, error) {
+						runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+						err := k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+						)
+						if err != nil {
+							return 0, err
+						}
+
+						return len(runnerSets.Items), nil
+					},
+					time.Second*1, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				var rs17 *actionsv1alpha1.RunnerReplicaSet
+
+				Consistently(
+					func() (*metav1.LabelSelector, error) {
+						runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+						err := k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+						)
+						if err != nil {
+							return nil, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return nil, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						rs17 = &runnerSets.Items[0]
+
+						return runnerSets.Items[0].Spec.Selector, nil
+					},
+					time.Second*1, time.Millisecond*500).Should(Not(BeNil()))
+
+				// We simulate the old, pre 0.18.0 RunnerReplicaSet by updating it.
+				// I've tried to use controllerutil.Set{Owner,Controller}Reference and k8sClient.Create(rs17)
+				// but it didn't work due to missing RD UID, where UID is generated on K8s API server on k8sCLient.Create(rd)
+				rs17.Spec.Selector = nil
+
+				updateRSErr := k8sClient.Update(ctx, rs17)
+				Expect(updateRSErr).NotTo(HaveOccurred())
+
+				Eventually(
+					func() (*metav1.LabelSelector, error) {
+						runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+						err := k8sClient.List(
+							ctx,
+							&runnerSets,
+							client.InNamespace(ns.Name),
+						)
+						if err != nil {
+							return nil, err
+						}
+						if len(runnerSets.Items) != 1 {
+							return nil, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
+						}
+
+						return runnerSets.Items[0].Spec.Selector, nil
+					},
+					time.Second*1, time.Millisecond*500).Should(Not(BeNil()))
+			}
+		})
+
+	})
+})

controllers/actions.summerwind.net/runnerreplicaset_controller.go

@@ -0,0 +1,205 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"reflect"
+	"time"
+
+	"github.com/go-logr/logr"
+
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+)
+
+// RunnerReplicaSetReconciler reconciles a Runner object
+type RunnerReplicaSetReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+	Name     string
+}
+
+const (
+	SyncTimeAnnotationKey = "sync-time"
+)
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerreplicasets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runners/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+
+func (r *RunnerReplicaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("runnerreplicaset", req.NamespacedName)
+
+	var rs v1alpha1.RunnerReplicaSet
+	if err := r.Get(ctx, req.NamespacedName, &rs); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !rs.ObjectMeta.DeletionTimestamp.IsZero() {
+		// RunnerReplicaSet cannot be gracefuly removed.
+		// That means any runner that is running a job can be prematurely terminated.
+		// To gracefully remove a RunnerReplicaSet, scale it down to zero first, observe RunnerReplicaSet's status replicas,
+		// and remove it only after the status replicas becomes zero.
+		return ctrl.Result{}, nil
+	}
+
+	if rs.ObjectMeta.Labels == nil {
+		rs.ObjectMeta.Labels = map[string]string{}
+	}
+
+	// Template hash is usually set by the upstream controller(RunnerDeplloyment controller) on authoring
+	// RunerReplicaset resource, but it may be missing when the user directly created RunnerReplicaSet.
+	// As a template hash is required by by the runner replica management, we dynamically add it here without ever persisting it.
+	if rs.ObjectMeta.Labels[LabelKeyRunnerTemplateHash] == "" {
+		template := rs.Spec.DeepCopy()
+		template.Replicas = nil
+		template.EffectiveTime = nil
+		templateHash := ComputeHash(template)
+
+		log.Info("Using auto-generated template hash", "value", templateHash)
+
+		rs.ObjectMeta.Labels = CloneAndAddLabel(rs.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash)
+		rs.Spec.Template.ObjectMeta.Labels = CloneAndAddLabel(rs.Spec.Template.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash)
+	}
+
+	selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// Get the Runners managed by the target RunnerReplicaSet
+	var runnerList v1alpha1.RunnerList
+	if err := r.List(
+		ctx,
+		&runnerList,
+		client.InNamespace(req.Namespace),
+		client.MatchingLabelsSelector{Selector: selector},
+	); err != nil {
+		if !kerrors.IsNotFound(err) {
+			return ctrl.Result{}, err
+		}
+	}
+
+	replicas := 1
+	if rs.Spec.Replicas != nil {
+		replicas = *rs.Spec.Replicas
+	}
+
+	effectiveTime := rs.Spec.EffectiveTime
+	ephemeral := rs.Spec.Template.Spec.Ephemeral == nil || *rs.Spec.Template.Spec.Ephemeral
+
+	desired, err := r.newRunner(rs)
+	if err != nil {
+		log.Error(err, "Could not create runner")
+
+		return ctrl.Result{}, err
+	}
+
+	var live []client.Object
+	for _, r := range runnerList.Items {
+		r := r
+		live = append(live, &r)
+	}
+
+	res, err := syncRunnerPodsOwners(ctx, r.Client, log, effectiveTime, replicas, func() client.Object { return desired.DeepCopy() }, ephemeral, live)
+	if err != nil || res == nil {
+		return ctrl.Result{}, err
+	}
+
+	var (
+		status v1alpha1.RunnerReplicaSetStatus
+
+		current, available, ready int
+	)
+
+	for _, o := range res.currentObjects {
+		current += o.total
+		available += o.running
+		ready += o.running
+	}
+
+	status.Replicas = &current
+	status.AvailableReplicas = &available
+	status.ReadyReplicas = &ready
+
+	if !reflect.DeepEqual(rs.Status, status) {
+		updated := rs.DeepCopy()
+		updated.Status = status
+
+		if err := r.Status().Patch(ctx, updated, client.MergeFrom(&rs)); err != nil {
+			log.Info("Failed to update runnerreplicaset status. Retrying immediately", "error", err.Error())
+			return ctrl.Result{
+				Requeue: true,
+			}, nil
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *RunnerReplicaSetReconciler) newRunner(rs v1alpha1.RunnerReplicaSet) (v1alpha1.Runner, error) {
+	// Note that the upstream controller (runnerdeployment) is expected to add
+	// the "runner template hash" label to the template.meta which is necessary to make this controller work correctly
+	objectMeta := rs.Spec.Template.ObjectMeta.DeepCopy()
+
+	objectMeta.GenerateName = rs.ObjectMeta.Name + "-"
+	objectMeta.Namespace = rs.ObjectMeta.Namespace
+	if objectMeta.Annotations == nil {
+		objectMeta.Annotations = map[string]string{}
+	}
+	objectMeta.Annotations[SyncTimeAnnotationKey] = time.Now().Format(time.RFC3339)
+
+	runner := v1alpha1.Runner{
+		TypeMeta:   metav1.TypeMeta{},
+		ObjectMeta: *objectMeta,
+		Spec:       rs.Spec.Template.Spec,
+	}
+
+	if err := ctrl.SetControllerReference(&rs, &runner, r.Scheme); err != nil {
+		return runner, err
+	}
+
+	return runner, nil
+}
+
+func (r *RunnerReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerreplicaset-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.RunnerReplicaSet{}).
+		Owns(&v1alpha1.Runner{}).
+		Named(name).
+		Complete(r)
+}

controllers/actions.summerwind.net/runnerreplicaset_controller_test.go

@@ -0,0 +1,270 @@
+package controllers
+
+import (
+	"context"
+	"math/rand"
+	"net/http/httptest"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/github/fake"
+)
+
+var (
+	runnersList *fake.RunnersList
+	server      *httptest.Server
+)
+
+// SetupTest will set up a testing environment.
+// This includes:
+// * creating a Namespace to be used during the test
+// * starting the 'RunnerReconciler'
+// * stopping the 'RunnerReplicaSetReconciler" after the test ends
+// Call this function at the start of each of your tests.
+func SetupTest(ctx2 context.Context) *corev1.Namespace {
+	var ctx context.Context
+	var cancel func()
+	ns := &corev1.Namespace{}
+
+	BeforeEach(func() {
+		ctx, cancel = context.WithCancel(ctx2)
+		*ns = corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)},
+		}
+
+		err := k8sClient.Create(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+			Namespace: ns.Name,
+		})
+		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
+
+		runnersList = fake.NewRunnersList()
+		server = runnersList.GetServer()
+
+		controller := &RunnerReplicaSetReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerreplicaset-controller"),
+			Name:     "runnerreplicaset-" + ns.Name,
+		}
+		err = controller.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		go func() {
+			defer GinkgoRecover()
+
+			err := mgr.Start(ctx)
+			Expect(err).NotTo(HaveOccurred(), "failed to start manager")
+		}()
+	})
+
+	AfterEach(func() {
+		defer cancel()
+
+		server.Close()
+		err := k8sClient.Delete(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
+	})
+
+	return ns
+}
+
+var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890")
+
+func randStringRunes(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+	return string(b)
+}
+
+func intPtr(v int) *int {
+	return &v
+}
+
+var _ = Context("Inside of a new namespace", func() {
+	ctx := context.TODO()
+	ns := SetupTest(ctx)
+	name := "example-runnerreplicaset"
+
+	getRunnerCount := func() int {
+		runners := actionsv1alpha1.RunnerList{Items: []actionsv1alpha1.Runner{}}
+
+		selector, err := metav1.LabelSelectorAsSelector(
+			&metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"foo": "bar",
+				},
+			},
+		)
+		if err != nil {
+			logf.Log.Error(err, "failed to create labelselector")
+			return -1
+		}
+
+		err = k8sClient.List(
+			ctx,
+			&runners,
+			client.InNamespace(ns.Name),
+			client.MatchingLabelsSelector{Selector: selector},
+		)
+		if err != nil {
+			logf.Log.Error(err, "list runners")
+		}
+
+		runnersList.Sync(runners.Items)
+
+		return len(runners.Items)
+	}
+
+	Describe("RunnerReplicaSet", func() {
+		It("should create a new Runner resource from the specified template", func() {
+			{
+				rs := &actionsv1alpha1.RunnerReplicaSet{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerReplicaSetSpec{
+						Replicas: intPtr(1),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				Eventually(
+					getRunnerCount,
+					time.Second*5, time.Second).Should(BeEquivalentTo(1))
+			}
+		})
+
+		It("should create 2 runners when specified 2 replicas", func() {
+			{
+				rs := &actionsv1alpha1.RunnerReplicaSet{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerReplicaSetSpec{
+						Replicas: intPtr(2),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				Eventually(
+					getRunnerCount,
+					time.Second*5, time.Second).Should(BeEquivalentTo(2))
+			}
+		})
+
+		It("should not create any runners when specified 0 replicas", func() {
+			{
+				rs := &actionsv1alpha1.RunnerReplicaSet{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerReplicaSetSpec{
+						Replicas: intPtr(0),
+						Selector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"foo": "bar",
+							},
+						},
+						Template: actionsv1alpha1.RunnerTemplate{
+							ObjectMeta: metav1.ObjectMeta{
+								Labels: map[string]string{
+									"foo": "bar",
+								},
+							},
+							Spec: actionsv1alpha1.RunnerSpec{
+								RunnerConfig: actionsv1alpha1.RunnerConfig{
+									Repository: "test/valid",
+									Image:      "bar",
+								},
+								RunnerPodSpec: actionsv1alpha1.RunnerPodSpec{
+									Env: []corev1.EnvVar{
+										{Name: "FOO", Value: "FOOVALUE"},
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
+
+				Consistently(
+					getRunnerCount,
+					time.Second*5, time.Second).Should(BeEquivalentTo(0))
+			}
+		})
+	})
+})

controllers/actions.summerwind.net/runnerset_controller.go

@@ -0,0 +1,309 @@
+/*
+Copyright 2021 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"reflect"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/actions/actions-runner-controller/controllers/actions.summerwind.net/metrics"
+	"github.com/go-logr/logr"
+)
+
+// RunnerSetReconciler reconciles a Runner object
+type RunnerSetReconciler struct {
+	Name string
+
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+
+	CommonRunnerLabels        []string
+	GitHubClient              *MultiGitHubClient
+	RunnerImage               string
+	RunnerImagePullSecrets    []string
+	DockerImage               string
+	DockerRegistryMirror      string
+	UseRunnerStatusUpdateHook bool
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch
+// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update
+
+// Note that coordination.k8s.io/leases permission must be added to any of the controllers to avoid the following error:
+//   E0613 07:02:08.004278       1 leaderelection.go:325] error retrieving resource lock actions-runner-system/actions-runner-controller: leases.coordination.k8s.io "actions-runner-controller" is forbidden: User "system:serviceaccount:actions-runner-system:actions-runner-controller" cannot get resource "leases" in API group "coordination.k8s.io" in the namespace "actions-runner-system"
+
+func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("runnerset", req.NamespacedName)
+
+	runnerSet := &v1alpha1.RunnerSet{}
+	if err := r.Get(ctx, req.NamespacedName, runnerSet); err != nil {
+		err = client.IgnoreNotFound(err)
+
+		if err != nil {
+			log.Error(err, "Could not get RunnerSet")
+		}
+
+		return ctrl.Result{}, err
+	}
+
+	if !runnerSet.ObjectMeta.DeletionTimestamp.IsZero() {
+		r.GitHubClient.DeinitForRunnerSet(runnerSet)
+
+		return ctrl.Result{}, nil
+	}
+
+	metrics.SetRunnerSet(*runnerSet)
+
+	var statefulsetList appsv1.StatefulSetList
+	if err := r.List(ctx, &statefulsetList, client.InNamespace(req.Namespace), client.MatchingFields{runnerSetOwnerKey: req.Name}); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	statefulsets := statefulsetList.Items
+
+	if len(statefulsets) > 1000 {
+		log.Info("Postponed reconcilation to prevent potential infinite loop. If you're really scaling more than 1000 statefulsets, do change this hard-coded threshold!")
+		return ctrl.Result{}, nil
+	}
+
+	desiredStatefulSet, err := r.newStatefulSet(ctx, runnerSet)
+	if err != nil {
+		r.Recorder.Event(runnerSet, corev1.EventTypeNormal, "RunnerAutoscalingFailure", err.Error())
+
+		log.Error(err, "Could not create statefulset")
+
+		return ctrl.Result{}, err
+	}
+
+	addedReplicas := int32(1)
+	create := desiredStatefulSet.DeepCopy()
+	create.Spec.Replicas = &addedReplicas
+
+	const defaultReplicas = 1
+
+	var replicasOfDesiredStatefulSet *int
+	if desiredStatefulSet.Spec.Replicas != nil {
+		v := int(*desiredStatefulSet.Spec.Replicas)
+		replicasOfDesiredStatefulSet = &v
+	}
+
+	newDesiredReplicas := getIntOrDefault(replicasOfDesiredStatefulSet, defaultReplicas)
+
+	effectiveTime := runnerSet.Spec.EffectiveTime
+	ephemeral := runnerSet.Spec.Ephemeral == nil || *runnerSet.Spec.Ephemeral
+
+	var owners []client.Object
+
+	for _, ss := range statefulsets {
+		ss := ss
+		owners = append(owners, &ss)
+	}
+
+	if res, err := syncVolumes(ctx, r.Client, log, req.Namespace, runnerSet, statefulsets); err != nil {
+		return ctrl.Result{}, err
+	} else if res != nil {
+		return *res, nil
+	}
+
+	res, err := syncRunnerPodsOwners(ctx, r.Client, log, effectiveTime, newDesiredReplicas, func() client.Object { return create.DeepCopy() }, ephemeral, owners)
+	if err != nil || res == nil {
+		return ctrl.Result{}, err
+	}
+
+	var statusReplicas, statusReadyReplicas, totalCurrentReplicas, updatedReplicas int
+
+	for _, ss := range res.currentObjects {
+		statusReplicas += int(ss.statefulSet.Status.Replicas)
+		statusReadyReplicas += int(ss.statefulSet.Status.ReadyReplicas)
+		totalCurrentReplicas += int(ss.statefulSet.Status.CurrentReplicas)
+		updatedReplicas += int(ss.statefulSet.Status.UpdatedReplicas)
+	}
+
+	status := runnerSet.Status.DeepCopy()
+
+	status.CurrentReplicas = &totalCurrentReplicas
+	status.ReadyReplicas = &statusReadyReplicas
+	status.DesiredReplicas = &newDesiredReplicas
+	status.Replicas = &statusReplicas
+	status.UpdatedReplicas = &updatedReplicas
+
+	if !reflect.DeepEqual(runnerSet.Status, status) {
+		updated := runnerSet.DeepCopy()
+		updated.Status = *status
+
+		if err := r.Status().Patch(ctx, updated, client.MergeFrom(runnerSet)); err != nil {
+			log.Info("Failed to patch runnerset status. Retrying immediately", "error", err.Error())
+			return ctrl.Result{
+				Requeue: true,
+			}, nil
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func getRunnerSetSelector(runnerSet *v1alpha1.RunnerSet) *metav1.LabelSelector {
+	selector := runnerSet.Spec.Selector
+	if selector == nil {
+		selector = &metav1.LabelSelector{MatchLabels: map[string]string{LabelKeyRunnerSetName: runnerSet.Name}}
+	}
+
+	return selector
+}
+
+var LabelKeyPodMutation = "actions-runner-controller/inject-registration-token"
+var LabelValuePodMutation = "true"
+
+func (r *RunnerSetReconciler) newStatefulSet(ctx context.Context, runnerSet *v1alpha1.RunnerSet) (*appsv1.StatefulSet, error) {
+	runnerSetWithOverrides := *runnerSet.Spec.DeepCopy()
+
+	runnerSetWithOverrides.Labels = append(runnerSetWithOverrides.Labels, r.CommonRunnerLabels...)
+
+	template := corev1.Pod{
+		ObjectMeta: runnerSetWithOverrides.StatefulSetSpec.Template.ObjectMeta,
+		Spec:       runnerSetWithOverrides.StatefulSetSpec.Template.Spec,
+	}
+
+	if runnerSet.Spec.RunnerConfig.ContainerMode == "kubernetes" {
+		found := false
+		for i := range template.Spec.Containers {
+			if template.Spec.Containers[i].Name == containerName {
+				found = true
+			}
+		}
+		if !found {
+			template.Spec.Containers = append(template.Spec.Containers, corev1.Container{
+				Name: "runner",
+			})
+		}
+
+		workDir := runnerSet.Spec.RunnerConfig.WorkDir
+		if workDir == "" {
+			workDir = "/runner/_work"
+		}
+		if err := applyWorkVolumeClaimTemplateToPod(&template, runnerSet.Spec.WorkVolumeClaimTemplate, workDir); err != nil {
+			return nil, err
+		}
+
+		template.Spec.ServiceAccountName = runnerSet.Spec.ServiceAccountName
+	}
+
+	template.ObjectMeta.Labels = CloneAndAddLabel(template.ObjectMeta.Labels, LabelKeyRunnerSetName, runnerSet.Name)
+
+	ghc, err := r.GitHubClient.InitForRunnerSet(ctx, runnerSet)
+	if err != nil {
+		return nil, err
+	}
+
+	githubBaseURL := ghc.GithubBaseURL
+
+	pod, err := newRunnerPodWithContainerMode(runnerSet.Spec.RunnerConfig.ContainerMode, template, runnerSet.Spec.RunnerConfig, r.RunnerImage, r.RunnerImagePullSecrets, r.DockerImage, r.DockerRegistryMirror, githubBaseURL, r.UseRunnerStatusUpdateHook)
+	if err != nil {
+		return nil, err
+	}
+
+	runnerSetWithOverrides.StatefulSetSpec.Template.ObjectMeta = pod.ObjectMeta
+	runnerSetWithOverrides.StatefulSetSpec.Template.Spec = pod.Spec
+	// NOTE: Seems like the only supported restart policy for statefulset is "Always"?
+	// I got errosr like the below when tried to use "OnFailure":
+	//   StatefulSet.apps \"example-runnersetpg9rx\" is invalid: [spec.template.metadata.labels: Invalid value: map[string]string{\"runner-template-hash\"
+	//   :\"85d7578bd6\", \"runnerset-name\":\"example-runnerset\"}: `selector` does not match template `labels`, spec.
+	//   template.spec.restartPolicy: Unsupported value: \"OnFailure\": supported values: \"Always\"]
+	runnerSetWithOverrides.StatefulSetSpec.Template.Spec.RestartPolicy = corev1.RestartPolicyAlways
+
+	templateHash := ComputeHash(pod.Spec)
+
+	// Add template hash label to selector.
+	runnerSetWithOverrides.Template.ObjectMeta.Labels = CloneAndAddLabel(runnerSetWithOverrides.Template.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash)
+
+	selector := getRunnerSetSelector(runnerSet)
+	selector = CloneSelectorAndAddLabel(selector, LabelKeyRunnerTemplateHash, templateHash)
+	selector = CloneSelectorAndAddLabel(selector, LabelKeyRunnerSetName, runnerSet.Name)
+	selector = CloneSelectorAndAddLabel(selector, LabelKeyPodMutation, LabelValuePodMutation)
+
+	runnerSetWithOverrides.StatefulSetSpec.Selector = selector
+
+	rs := appsv1.StatefulSet{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: runnerSet.ObjectMeta.Name + "-",
+			Namespace:    runnerSet.ObjectMeta.Namespace,
+			Labels:       CloneAndAddLabel(runnerSet.ObjectMeta.Labels, LabelKeyRunnerTemplateHash, templateHash),
+			Annotations: map[string]string{
+				SyncTimeAnnotationKey: time.Now().Format(time.RFC3339),
+			},
+		},
+		Spec: runnerSetWithOverrides.StatefulSetSpec,
+	}
+
+	if err := ctrl.SetControllerReference(runnerSet, &rs, r.Scheme); err != nil {
+		return &rs, err
+	}
+
+	return &rs, nil
+}
+
+func (r *RunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	name := "runnerset-controller"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &appsv1.StatefulSet{}, runnerSetOwnerKey, func(rawObj client.Object) []string {
+		set := rawObj.(*appsv1.StatefulSet)
+		owner := metav1.GetControllerOf(set)
+		if owner == nil {
+			return nil
+		}
+
+		if owner.APIVersion != v1alpha1.GroupVersion.String() || owner.Kind != "RunnerSet" {
+			return nil
+		}
+
+		return []string{owner.Name}
+	}); err != nil {
+		return err
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.RunnerSet{}).
+		Owns(&appsv1.StatefulSet{}).
+		Named(name).
+		Complete(r)
+}

controllers/actions.summerwind.net/schedule.go

@@ -0,0 +1,122 @@
+package controllers
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/teambition/rrule-go"
+)
+
+type RecurrenceRule struct {
+	Frequency string
+	UntilTime time.Time
+}
+
+type Period struct {
+	StartTime time.Time
+	EndTime   time.Time
+}
+
+func (r *Period) String() string {
+	if r == nil {
+		return ""
+	}
+
+	return r.StartTime.Format(time.RFC3339) + "-" + r.EndTime.Format(time.RFC3339)
+}
+
+func MatchSchedule(now time.Time, startTime, endTime time.Time, recurrenceRule RecurrenceRule) (*Period, *Period, error) {
+	return calculateActiveAndUpcomingRecurringPeriods(
+		now,
+		startTime,
+		endTime,
+		recurrenceRule.Frequency,
+		recurrenceRule.UntilTime,
+	)
+}
+
+func calculateActiveAndUpcomingRecurringPeriods(now, startTime, endTime time.Time, frequency string, untilTime time.Time) (*Period, *Period, error) {
+	var freqValue rrule.Frequency
+
+	var freqDurationDay int
+	var freqDurationMonth int
+	var freqDurationYear int
+
+	switch frequency {
+	case "Daily":
+		freqValue = rrule.DAILY
+		freqDurationDay = 1
+	case "Weekly":
+		freqValue = rrule.WEEKLY
+		freqDurationDay = 7
+	case "Monthly":
+		freqValue = rrule.MONTHLY
+		freqDurationMonth = 1
+	case "Yearly":
+		freqValue = rrule.YEARLY
+		freqDurationYear = 1
+	case "":
+		if now.Before(startTime) {
+			return nil, &Period{StartTime: startTime, EndTime: endTime}, nil
+		}
+
+		if now.Before(endTime) {
+			return &Period{StartTime: startTime, EndTime: endTime}, nil, nil
+		}
+
+		return nil, nil, nil
+	default:
+		return nil, nil, fmt.Errorf(`invalid freq %q: It must be one of "Daily", "Weekly", "Monthly", and "Yearly"`, frequency)
+	}
+
+	freqDurationLater := time.Date(
+		now.Year()+freqDurationYear,
+		time.Month(int(now.Month())+freqDurationMonth),
+		now.Day()+freqDurationDay,
+		now.Hour(), now.Minute(), now.Second(), now.Nanosecond(), now.Location(),
+	)
+
+	freqDuration := freqDurationLater.Sub(now)
+
+	overrideDuration := endTime.Sub(startTime)
+	if overrideDuration > freqDuration {
+		return nil, nil, fmt.Errorf("override's duration %s must be equal to sor shorter than the duration implied by freq %q (%s)", overrideDuration, frequency, freqDuration)
+	}
+
+	rrule, err := rrule.NewRRule(rrule.ROption{
+		Freq:    freqValue,
+		Dtstart: startTime,
+		Until:   untilTime,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	overrideDurationBefore := now.Add(-overrideDuration + 1)
+	activeOverrideStarts := rrule.Between(overrideDurationBefore, now, true)
+
+	var active *Period
+
+	if len(activeOverrideStarts) > 1 {
+		return nil, nil, fmt.Errorf("[bug] unexpted number of active overrides found: %v", activeOverrideStarts)
+	} else if len(activeOverrideStarts) == 1 {
+		active = &Period{
+			StartTime: activeOverrideStarts[0],
+			EndTime:   activeOverrideStarts[0].Add(overrideDuration),
+		}
+	}
+
+	oneSecondLater := now.Add(1)
+	upcomingOverrideStarts := rrule.Between(oneSecondLater, freqDurationLater, true)
+
+	var next *Period
+
+	if len(upcomingOverrideStarts) > 0 {
+		next = &Period{
+			StartTime: upcomingOverrideStarts[0],
+			EndTime:   upcomingOverrideStarts[0].Add(overrideDuration),
+		}
+	}
+
+	return active, next, nil
+}

controllers/actions.summerwind.net/schedule_test.go

@@ -0,0 +1,617 @@
+package controllers
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCalculateActiveAndUpcomingRecurringPeriods(t *testing.T) {
+	type recurrence struct {
+		Start string
+		End   string
+		Freq  string
+		Until string
+	}
+
+	type testcase struct {
+		now string
+
+		recurrence recurrence
+
+		wantActive   string
+		wantUpcoming string
+	}
+
+	check := func(t *testing.T, tc testcase) {
+		t.Helper()
+
+		_, err := time.Parse(time.RFC3339, "2021-05-08T00:00:00Z")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		now, err := time.Parse(time.RFC3339, tc.now)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		active, upcoming, err := parseAndMatchRecurringPeriod(now, tc.recurrence.Start, tc.recurrence.End, tc.recurrence.Freq, tc.recurrence.Until)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if active.String() != tc.wantActive {
+			t.Errorf("unexpected active: want %q, got %q", tc.wantActive, active)
+		}
+
+		if upcoming.String() != tc.wantUpcoming {
+			t.Errorf("unexpected upcoming: want %q, got %q", tc.wantUpcoming, upcoming)
+		}
+	}
+
+	t.Run("onetime override about to start", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+			},
+
+			now: "2021-04-30T23:59:59+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("onetime override started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+			},
+
+			now: "2021-05-01T00:00:00+09:00",
+
+			wantActive:   "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("onetime override about to end", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+			},
+
+			now: "2021-05-02T23:59:59+09:00",
+
+			wantActive:   "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("onetime override ended", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+			},
+
+			now: "2021-05-03T00:00:00+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("weekly override about to start", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-04-30T23:59:59+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-01T00:00:00+09:00",
+
+			wantActive:   "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+			wantUpcoming: "2021-05-08T00:00:00+09:00-2021-05-10T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override about to end", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-02T23:59:59+09:00",
+
+			wantActive:   "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+			wantUpcoming: "2021-05-08T00:00:00+09:00-2021-05-10T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override ended", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-03T00:00:00+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2021-05-08T00:00:00+09:00-2021-05-10T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override reccurrence about to start", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-07T23:59:59+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2021-05-08T00:00:00+09:00-2021-05-10T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override reccurrence started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-08T00:00:00+09:00",
+
+			wantActive:   "2021-05-08T00:00:00+09:00-2021-05-10T00:00:00+09:00",
+			wantUpcoming: "2021-05-15T00:00:00+09:00-2021-05-17T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override reccurrence about to end", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-09T23:59:59+09:00",
+
+			wantActive:   "2021-05-08T00:00:00+09:00-2021-05-10T00:00:00+09:00",
+			wantUpcoming: "2021-05-15T00:00:00+09:00-2021-05-17T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override reccurrence ended", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-10T00:00:00+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2021-05-15T00:00:00+09:00-2021-05-17T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override's last reccurrence about to start", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-04-29T23:59:59+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2022-04-30T00:00:00+09:00-2022-05-02T00:00:00+09:00",
+		})
+	})
+
+	t.Run("weekly override reccurrence started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-04-30T00:00:00+09:00",
+
+			wantActive:   "2022-04-30T00:00:00+09:00-2022-05-02T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("weekly override reccurrence about to end", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-05-01T23:59:59+09:00",
+
+			wantActive:   "2022-04-30T00:00:00+09:00-2022-05-02T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("weekly override reccurrence ended", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-05-02T00:00:00+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("weekly override repeated forever started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Weekly",
+			},
+
+			now: "2021-05-08T00:00:00+09:00",
+
+			wantActive:   "2021-05-08T00:00:00+09:00-2021-05-10T00:00:00+09:00",
+			wantUpcoming: "2021-05-15T00:00:00+09:00-2021-05-17T00:00:00+09:00",
+		})
+	})
+
+	t.Run("monthly override started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Monthly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-01T00:00:00+09:00",
+
+			wantActive:   "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+			wantUpcoming: "2021-06-01T00:00:00+09:00-2021-06-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("monthly override recurrence started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Monthly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-06-01T00:00:00+09:00",
+
+			wantActive:   "2021-06-01T00:00:00+09:00-2021-06-03T00:00:00+09:00",
+			wantUpcoming: "2021-07-01T00:00:00+09:00-2021-07-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("monthly override's last reccurence about to start", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Monthly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-04-30T23:59:59+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2022-05-01T00:00:00+09:00-2022-05-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("monthly override's last reccurence started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Monthly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-05-01T00:00:00+09:00",
+
+			wantActive:   "2022-05-01T00:00:00+09:00-2022-05-03T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("monthly override's last reccurence started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Monthly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-05-01T00:00:01+09:00",
+
+			wantActive:   "2022-05-01T00:00:00+09:00-2022-05-03T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("monthly override's last reccurence ending", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Monthly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-05-02T23:59:59+09:00",
+
+			wantActive:   "2022-05-01T00:00:00+09:00-2022-05-03T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("monthly override's last reccurence ended", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Monthly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-05-03T00:00:00+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("yearly override started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Yearly",
+				Until: "2022-05-01T00:00:00+09:00",
+			},
+
+			now: "2021-05-01T00:00:00+09:00",
+
+			wantActive:   "2021-05-01T00:00:00+09:00-2021-05-03T00:00:00+09:00",
+			wantUpcoming: "2022-05-01T00:00:00+09:00-2022-05-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("yearly override reccurrence started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Yearly",
+				Until: "2023-05-01T00:00:00+09:00",
+			},
+
+			now: "2022-05-01T00:00:00+09:00",
+
+			wantActive:   "2022-05-01T00:00:00+09:00-2022-05-03T00:00:00+09:00",
+			wantUpcoming: "2023-05-01T00:00:00+09:00-2023-05-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("yearly override's last recurrence about to start", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Yearly",
+				Until: "2023-05-01T00:00:00+09:00",
+			},
+
+			now: "2023-04-30T23:59:59+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "2023-05-01T00:00:00+09:00-2023-05-03T00:00:00+09:00",
+		})
+	})
+
+	t.Run("yearly override's last recurrence started", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Yearly",
+				Until: "2023-05-01T00:00:00+09:00",
+			},
+
+			now: "2023-05-01T00:00:00+09:00",
+
+			wantActive:   "2023-05-01T00:00:00+09:00-2023-05-03T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("yearly override's last recurrence ending", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Yearly",
+				Until: "2023-05-01T00:00:00+09:00",
+			},
+
+			now: "2023-05-02T23:23:59+09:00",
+
+			wantActive:   "2023-05-01T00:00:00+09:00-2023-05-03T00:00:00+09:00",
+			wantUpcoming: "",
+		})
+	})
+
+	t.Run("yearly override's last recurrence ended", func(t *testing.T) {
+		t.Helper()
+
+		check(t, testcase{
+			recurrence: recurrence{
+				Start: "2021-05-01T00:00:00+09:00",
+				End:   "2021-05-03T00:00:00+09:00",
+				Freq:  "Yearly",
+				Until: "2023-05-01T00:00:00+09:00",
+			},
+
+			now: "2023-05-03T00:00:00+09:00",
+
+			wantActive:   "",
+			wantUpcoming: "",
+		})
+	})
+}
+
+func parseAndMatchRecurringPeriod(now time.Time, start, end, frequency, until string) (*Period, *Period, error) {
+	startTime, err := time.Parse(time.RFC3339, start)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	endTime, err := time.Parse(time.RFC3339, end)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var untilTime time.Time
+
+	if until != "" {
+		ut, err := time.Parse(time.RFC3339, until)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		untilTime = ut
+	}
+
+	return MatchSchedule(now, startTime, endTime, RecurrenceRule{Frequency: frequency, UntilTime: untilTime})
+}
+
+func FuzzMatchSchedule(f *testing.F) {
+	start := time.Now()
+	end := time.Now()
+	now := time.Now()
+	f.Fuzz(func(t *testing.T, freq string) {
+		// Verify that it never panics
+		_, _, _ = MatchSchedule(now, start, end, RecurrenceRule{Frequency: freq})
+	})
+}

controllers/actions.summerwind.net/suite_test.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/onsi/ginkgo/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	// +kubebuilder:scaffold:imports
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+var cfg *rest.Config
+var k8sClient client.Client
+var testEnv *envtest.Environment
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	config.GinkgoConfig.FocusStrings = append(config.GinkgoConfig.FocusStrings, os.Getenv("GINKGO_FOCUS"))
+
+	RunSpecsWithDefaultAndCustomReporters(t,
+		"Controller Suite",
+		[]Reporter{printer.NewlineReporter{}})
+}
+
+var _ = BeforeSuite(func(done Done) {
+	logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))
+
+	var apiServerFlags []string
+
+	apiServerFlags = append(apiServerFlags, envtest.DefaultKubeAPIServerFlags...)
+	// Avoids the following error:
+	// 2021-03-19T15:14:11.673+0900    ERROR   controller-runtime.controller   Reconciler error      {"controller": "testns-tvjzjrunner", "request": "testns-gdnyx/example-runnerdeploy-zps4z-j5562", "error": "Pod \"example-runnerdeploy-zps4z-j5562\" is invalid: [spec.containers[1].image: Required value, spec.containers[1].securityContext.privileged: Forbidden: disallowed by cluster policy]"}
+	apiServerFlags = append(apiServerFlags, "--allow-privileged=true")
+
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths:  []string{filepath.Join("../..", "config", "crd", "bases")},
+		KubeAPIServerFlags: apiServerFlags,
+	}
+
+	var err error
+	cfg, err = testEnv.Start()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(cfg).ToNot(BeNil())
+
+	err = actionsv1alpha1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	// +kubebuilder:scaffold:scheme
+
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).ToNot(HaveOccurred())
+	Expect(k8sClient).ToNot(BeNil())
+
+	close(done)
+}, 60)
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).ToNot(HaveOccurred())
+})

controllers/actions.summerwind.net/sync_volumes.go

@@ -0,0 +1,185 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	labelKeyCleanup               = "pending-cleanup"
+	labelKeyRunnerStatefulSetName = "runner-statefulset-name"
+)
+
+func syncVolumes(ctx context.Context, c client.Client, log logr.Logger, ns string, runnerSet *v1alpha1.RunnerSet, statefulsets []appsv1.StatefulSet) (*ctrl.Result, error) {
+	log = log.WithValues("ns", ns)
+
+	for _, t := range runnerSet.Spec.StatefulSetSpec.VolumeClaimTemplates {
+		for _, sts := range statefulsets {
+			pvcName := fmt.Sprintf("%s-%s-0", t.Name, sts.Name)
+
+			var pvc corev1.PersistentVolumeClaim
+			if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil {
+				if !kerrors.IsNotFound(err) {
+					return nil, err
+				}
+				continue
+			}
+
+			// TODO move this to statefulset reconciler so that we spam this less,
+			// by starting the loop only after the statefulset got deletionTimestamp set.
+			// Perhaps you can just wrap this in a finalizer here.
+			if pvc.Labels[labelKeyRunnerStatefulSetName] == "" {
+				updated := pvc.DeepCopy()
+				updated.Labels[labelKeyRunnerStatefulSetName] = sts.Name
+				if err := c.Update(ctx, updated); err != nil {
+					return nil, err
+				}
+				log.V(1).Info("Added runner-statefulset-name label to PVC", "sts", sts.Name, "pvc", pvcName)
+			}
+		}
+	}
+
+	// PVs are not namespaced hence we don't need client.InNamespace(ns).
+	// If we added that, c.List will silently return zero items.
+	//
+	// This `List` needs to be done in a dedicated reconciler that is registered to the manager via the `For` func.
+	// Otherwise the List func might return outdated contents(I saw status.phase being Bound even after K8s updated it to Released, and it lasted minutes).
+	//
+	// cleanupLabels := map[string]string{
+	// 	labelKeyCleanup: runnerSet.Name,
+	// }
+	// pvList := &corev1.PersistentVolumeList{}
+	// if err := c.List(ctx, pvList, client.MatchingLabels(cleanupLabels)); err != nil {
+	// 	log.Info("retrying pv listing", "ns", ns, "err", err)
+	// 	return nil, err
+	// }
+
+	return nil, nil
+}
+
+func syncPVC(ctx context.Context, c client.Client, log logr.Logger, ns string, pvc *corev1.PersistentVolumeClaim) (*ctrl.Result, error) {
+	stsName := pvc.Labels[labelKeyRunnerStatefulSetName]
+	if stsName == "" {
+		return nil, nil
+	}
+
+	log.V(2).Info("Reconciling runner PVC")
+
+	// TODO: Probably we'd better remove PVCs related to the RunnetSet that is nowhere now?
+	// Otherwise, a bunch of continuously recreated StatefulSet
+	// can leave dangling PVCs forever, which might stress the cluster.
+
+	var sts appsv1.StatefulSet
+	if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: stsName}, &sts); err != nil {
+		if !kerrors.IsNotFound(err) {
+			return nil, err
+		}
+	} else {
+		// We assume that the statefulset is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(1).Info("Retrying sync until statefulset gets removed", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	log = log.WithValues("sts", stsName)
+
+	pvName := pvc.Spec.VolumeName
+
+	if pvName != "" {
+		// If we deleted PVC before unsetting pv.spec.claimRef,
+		// K8s seems to revive the claimRef :thinking:
+		// So we need to mark PV for claimRef unset first, and delete PVC, and finally unset claimRef on PV.
+
+		var pv corev1.PersistentVolume
+		if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvName}, &pv); err != nil {
+			if !kerrors.IsNotFound(err) {
+				return nil, err
+			}
+			return nil, nil
+		}
+
+		pvCopy := pv.DeepCopy()
+		if pvCopy.Labels == nil {
+			pvCopy.Labels = map[string]string{}
+		}
+		pvCopy.Labels[labelKeyCleanup] = stsName
+
+		log.V(2).Info("Scheduling to unset PV's claimRef", "pv", pv.Name)
+
+		// Apparently K8s doesn't reconcile PV immediately after PVC deletion.
+		// So we start a relatively busy loop of PV reconcilation slightly before the PVC deletion,
+		// so that PV can be unbound as soon as possible after the PVC got deleted.
+		if err := c.Update(ctx, pvCopy); err != nil {
+			return nil, err
+		}
+
+		log.Info("Updated PV to unset claimRef")
+
+		// At this point, the PV is still Bound
+
+		log.V(2).Info("Deleting unused PVC")
+
+		if err := c.Delete(ctx, pvc); err != nil {
+			return nil, err
+		}
+
+		log.Info("Deleted unused PVC")
+
+		// At this point, the PV is still "Bound", but we are ready to unset pv.spec.claimRef in pv controller.
+		// Once the pv controller unsets claimRef, the PV becomes "Released", hence available for reuse by another eligible PVC.
+	}
+
+	return nil, nil
+}
+
+func syncPV(ctx context.Context, c client.Client, log logr.Logger, ns string, pv *corev1.PersistentVolume) (*ctrl.Result, error) {
+	if pv.Spec.ClaimRef == nil {
+		return nil, nil
+	}
+
+	log.V(2).Info("Reconciling PV")
+
+	if pv.Labels[labelKeyCleanup] == "" {
+		// We assume that the pvc is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(2).Info("Retrying sync to see if this PV needs to be managed by ARC", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	log.V(2).Info("checking pv phase", "phase", pv.Status.Phase)
+
+	if pv.Status.Phase != corev1.VolumeReleased {
+		// We assume that the pvc is shortly terminated, hence retry forever until it gets removed.
+		retry := 10 * time.Second
+		log.V(1).Info("Retrying sync until pvc gets released", "requeueAfter", retry)
+		return &ctrl.Result{RequeueAfter: retry}, nil
+	}
+
+	// At this point, the PV is still Released
+
+	pvCopy := pv.DeepCopy()
+	delete(pvCopy.Labels, labelKeyCleanup)
+	pvCopy.Spec.ClaimRef = nil
+	log.V(2).Info("Unsetting PV's claimRef", "pv", pv.Name)
+	if err := c.Update(ctx, pvCopy); err != nil {
+		return nil, err
+	}
+
+	log.Info("PV should be Available now")
+
+	// At this point, the PV becomes Available, if it's reclaim policy is "Retain".
+	// I have not yet tested it with "Delete" but perhaps it's deleted automatically after the update?
+	// https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain
+
+	return nil, nil
+}

controllers/actions.summerwind.net/testdata/org_webhook_check_run_payload.json

@@ -0,0 +1,373 @@
+{
+  "action": "created",
+  "check_run": {
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "head_sha": "1234567890123456789012345678901234567890",
+    "external_id": "92058b04-f16a-5035-546c-cae3ad5e2f5f",
+    "url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890",
+    "html_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "details_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "status": "queued",
+    "conclusion": null,
+    "started_at": "2021-02-18T06:16:31Z",
+    "completed_at": null,
+    "output": {
+      "title": null,
+      "summary": null,
+      "text": null,
+      "annotations_count": 0,
+      "annotations_url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890/annotations"
+    },
+    "name": "validate",
+    "check_suite": {
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "head_branch": "MYNAME/actions-runner-controller-webhook",
+      "head_sha": "1234567890123456789012345678901234567890",
+      "status": "queued",
+      "conclusion": null,
+      "url": "https://api.github.com/repos/MYORG/MYREPO/check-suites/1234567890",
+      "before": "1234567890123456789012345678901234567890",
+      "after": "1234567890123456789012345678901234567890",
+      "pull_requests": [
+        {
+          "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/2033",
+          "id": 1234567890,
+          "number": 1234567890,
+          "head": {
+            "ref": "feature",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          },
+          "base": {
+            "ref": "master",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          }
+        }
+      ],
+      "app": {
+        "id": 1234567890,
+        "slug": "github-actions",
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "owner": {
+          "login": "github",
+          "id": 1234567890,
+          "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+          "avatar_url": "https://avatars.githubusercontent.com/u/123467890?v=4",
+          "gravatar_id": "",
+          "url": "https://api.github.com/users/github",
+          "html_url": "https://github.com/github",
+          "followers_url": "https://api.github.com/users/github/followers",
+          "following_url": "https://api.github.com/users/github/following{/other_user}",
+          "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+          "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+          "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+          "organizations_url": "https://api.github.com/users/github/orgs",
+          "repos_url": "https://api.github.com/users/github/repos",
+          "events_url": "https://api.github.com/users/github/events{/privacy}",
+          "received_events_url": "https://api.github.com/users/github/received_events",
+          "type": "Organization",
+          "site_admin": false
+        },
+        "name": "GitHub Actions",
+        "description": "Automate your workflow from idea to production",
+        "external_url": "https://help.github.com/en/actions",
+        "html_url": "https://github.com/apps/github-actions",
+        "created_at": "2018-07-30T09:30:17Z",
+        "updated_at": "2019-12-10T19:04:12Z",
+        "permissions": {
+          "actions": "write",
+          "checks": "write",
+          "contents": "write",
+          "deployments": "write",
+          "issues": "write",
+          "metadata": "read",
+          "organization_packages": "write",
+          "packages": "write",
+          "pages": "write",
+          "pull_requests": "write",
+          "repository_hooks": "write",
+          "repository_projects": "write",
+          "security_events": "write",
+          "statuses": "write",
+          "vulnerability_alerts": "read"
+        },
+        "events": [
+          "check_run",
+          "check_suite",
+          "create",
+          "delete",
+          "deployment",
+          "deployment_status",
+          "fork",
+          "gollum",
+          "issues",
+          "issue_comment",
+          "label",
+          "milestone",
+          "page_build",
+          "project",
+          "project_card",
+          "project_column",
+          "public",
+          "pull_request",
+          "pull_request_review",
+          "pull_request_review_comment",
+          "push",
+          "registry_package",
+          "release",
+          "repository",
+          "repository_dispatch",
+          "status",
+          "watch",
+          "workflow_dispatch",
+          "workflow_run"
+        ]
+      },
+      "created_at": "2021-02-18T06:15:32Z",
+      "updated_at": "2021-02-18T06:16:31Z"
+    },
+    "app": {
+    "id": 1234567890,
+    "slug": "github-actions",
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "owner": {
+      "login": "github",
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/github",
+      "html_url": "https://github.com/github",
+      "followers_url": "https://api.github.com/users/github/followers",
+      "following_url": "https://api.github.com/users/github/following{/other_user}",
+      "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+      "organizations_url": "https://api.github.com/users/github/orgs",
+      "repos_url": "https://api.github.com/users/github/repos",
+      "events_url": "https://api.github.com/users/github/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/github/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "name": "GitHub Actions",
+    "description": "Automate your workflow from idea to production",
+    "external_url": "https://help.github.com/en/actions",
+    "html_url": "https://github.com/apps/github-actions",
+    "created_at": "2018-07-30T09:30:17Z",
+    "updated_at": "2019-12-10T19:04:12Z",
+    "permissions": {
+      "actions": "write",
+      "checks": "write",
+      "contents": "write",
+      "deployments": "write",
+      "issues": "write",
+      "metadata": "read",
+      "organization_packages": "write",
+      "packages": "write",
+      "pages": "write",
+      "pull_requests": "write",
+      "repository_hooks": "write",
+      "repository_projects": "write",
+      "security_events": "write",
+      "statuses": "write",
+      "vulnerability_alerts": "read"
+    },
+    "events": [
+      "check_run",
+      "check_suite",
+      "create",
+      "delete",
+      "deployment",
+      "deployment_status",
+      "fork",
+      "gollum",
+      "issues",
+      "issue_comment",
+      "label",
+      "milestone",
+      "page_build",
+      "project",
+      "project_card",
+      "project_column",
+      "public",
+      "pull_request",
+      "pull_request_review",
+      "pull_request_review_comment",
+      "push",
+      "registry_package",
+      "release",
+      "repository",
+      "repository_dispatch",
+      "status",
+      "watch",
+      "workflow_dispatch",
+      "workflow_run"
+    ]
+  },
+  "pull_requests": [
+    {
+      "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/1234567890",
+      "id": 1234567890,
+      "number": 1234567890,
+      "head": {
+        "ref": "feature",
+        "sha": "1234567890123456789012345678901234567890",
+        "repo": {
+          "id": 1234567890,
+          "url": "https://api.github.com/repos/MYORG/MYREPO",
+          "name": "MYREPO"
+        }
+      },
+      "base": {
+        "ref": "master",
+        "sha": "1234567890123456789012345678901234567890",
+        "repo": {
+          "id": 1234567890,
+          "url": "https://api.github.com/repos/MYORG/MYREPO",
+          "name": "MYREPO"
+        }
+      }
+    }
+  ]
+},
+"repository": {
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "name": "MYREPO",
+    "full_name": "MYORG/MYREPO",
+    "private": true,
+    "owner": {
+      "login": "MYORG",
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/MYORG",
+      "html_url": "https://github.com/MYORG",
+      "followers_url": "https://api.github.com/users/MYORG/followers",
+      "following_url": "https://api.github.com/users/MYORG/following{/other_user}",
+      "gists_url": "https://api.github.com/users/MYORG/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/MYORG/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/MYORG/subscriptions",
+      "organizations_url": "https://api.github.com/users/MYORG/orgs",
+      "repos_url": "https://api.github.com/users/MYORG/repos",
+      "events_url": "https://api.github.com/users/MYORG/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/MYORG/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "html_url": "https://github.com/MYORG/MYREPO",
+    "description": "MYREPO",
+    "fork": false,
+    "url": "https://api.github.com/repos/MYORG/MYREPO",
+    "forks_url": "https://api.github.com/repos/MYORG/MYREPO/forks",
+    "keys_url": "https://api.github.com/repos/MYORG/MYREPO/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/MYORG/MYREPO/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/MYORG/MYREPO/teams",
+    "hooks_url": "https://api.github.com/repos/MYORG/MYREPO/hooks",
+    "issue_events_url": "https://api.github.com/repos/MYORG/MYREPO/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/MYORG/MYREPO/events",
+    "assignees_url": "https://api.github.com/repos/MYORG/MYREPO/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/MYORG/MYREPO/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/MYORG/MYREPO/tags",
+    "blobs_url": "https://api.github.com/repos/MYORG/MYREPO/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/MYORG/MYREPO/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/MYORG/MYREPO/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/MYORG/MYREPO/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/MYORG/MYREPO/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/MYORG/MYREPO/languages",
+    "stargazers_url": "https://api.github.com/repos/MYORG/MYREPO/stargazers",
+    "contributors_url": "https://api.github.com/repos/MYORG/MYREPO/contributors",
+    "subscribers_url": "https://api.github.com/repos/MYORG/MYREPO/subscribers",
+    "subscription_url": "https://api.github.com/repos/MYORG/MYREPO/subscription",
+    "commits_url": "https://api.github.com/repos/MYORG/MYREPO/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/MYORG/MYREPO/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/MYORG/MYREPO/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/MYORG/MYREPO/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/MYORG/MYREPO/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/MYORG/MYREPO/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/MYORG/MYREPO/merges",
+    "archive_url": "https://api.github.com/repos/MYORG/MYREPO/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/MYORG/MYREPO/downloads",
+    "issues_url": "https://api.github.com/repos/MYORG/MYREPO/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/MYORG/MYREPO/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/MYORG/MYREPO/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/MYORG/MYREPO/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/MYORG/MYREPO/labels{/name}",
+    "releases_url": "https://api.github.com/repos/MYORG/MYREPO/releases{/id}",
+    "deployments_url": "https://api.github.com/repos/MYORG/MYREPO/deployments",
+    "created_at": "2017-08-10T02:21:10Z",
+    "updated_at": "2021-02-18T04:40:55Z",
+    "pushed_at": "2021-02-18T06:15:30Z",
+    "git_url": "git://github.com/MYORG/MYREPO.git",
+    "ssh_url": "git@github.com:MYORG/MYREPO.git",
+    "clone_url": "https://github.com/MYORG/MYREPO.git",
+    "svn_url": "https://github.com/MYORG/MYREPO",
+    "homepage": null,
+    "size": 30782,
+    "stargazers_count": 2,
+    "watchers_count": 2,
+    "language": "Shell",
+    "has_issues": false,
+    "has_projects": true,
+    "has_downloads": true,
+    "has_wiki": false,
+    "has_pages": false,
+    "forks_count": 0,
+    "mirror_url": null,
+    "archived": false,
+    "disabled": false,
+    "open_issues_count": 6,
+    "license": null,
+    "forks": 0,
+    "open_issues": 6,
+    "watchers": 2,
+    "default_branch": "master"
+  },
+  "organization": {
+    "login": "MYORG",
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "url": "https://api.github.com/orgs/MYORG",
+    "repos_url": "https://api.github.com/orgs/MYORG/repos",
+    "events_url": "https://api.github.com/orgs/MYORG/events",
+    "hooks_url": "https://api.github.com/orgs/MYORG/hooks",
+    "issues_url": "https://api.github.com/orgs/MYORG/issues",
+    "members_url": "https://api.github.com/orgs/MYORG/members{/member}",
+    "public_members_url": "https://api.github.com/orgs/MYORG/public_members{/member}",
+    "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+    "description": ""
+  },
+  "sender": {
+    "login": "MYNAME",
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/MYNAME",
+    "html_url": "https://github.com/MYNAME",
+    "followers_url": "https://api.github.com/users/MYNAME/followers",
+    "following_url": "https://api.github.com/users/MYNAME/following{/other_user}",
+    "gists_url": "https://api.github.com/users/MYNAME/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/MYNAME/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/MYNAME/subscriptions",
+    "organizations_url": "https://api.github.com/users/MYNAME/orgs",
+    "repos_url": "https://api.github.com/users/MYNAME/repos",
+    "events_url": "https://api.github.com/users/MYNAME/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/MYNAME/received_events",
+    "type": "User",
+    "site_admin": false
+  }
+}

controllers/actions.summerwind.net/testdata/org_webhook_workflow_job_payload.json

@@ -0,0 +1,151 @@
+{
+    "action": "queued",
+    "workflow_job": {
+        "id": 1234567890,
+        "run_id": 1234567890,
+        "run_url": "https://api.github.com/repos/MYORG/MYREPO/actions/runs/1234567890",
+        "node_id": "CR_kwDOGCados7e1x2g",
+        "head_sha": "1234567890123456789012345678901234567890",
+        "url": "https://api.github.com/repos/MYORG/MYREPO/actions/jobs/1234567890",
+        "html_url": "https://github.com/MYORG/MYREPO/runs/1234567890",
+        "status": "queued",
+        "conclusion": null,
+        "started_at": "2021-09-28T23:45:29Z",
+        "completed_at": null,
+        "name": "build",
+        "steps": [],
+        "check_run_url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/1234567890",
+        "labels": [
+            "label1"
+        ]
+    },
+    "repository": {
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ=",
+        "name": "MYREPO",
+        "full_name": "MYORG/MYREPO",
+        "private": true,
+        "owner": {
+            "login": "MYORG",
+            "id": 1234567890,
+            "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+            "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+            "gravatar_id": "",
+            "url": "https://api.github.com/users/MYORG",
+            "html_url": "https://github.com/MYORG",
+            "followers_url": "https://api.github.com/users/MYORG/followers",
+            "following_url": "https://api.github.com/users/MYORG/following{/other_user}",
+            "gists_url": "https://api.github.com/users/MYORG/gists{/gist_id}",
+            "starred_url": "https://api.github.com/users/MYORG/starred{/owner}{/repo}",
+            "subscriptions_url": "https://api.github.com/users/MYORG/subscriptions",
+            "organizations_url": "https://api.github.com/users/MYORG/orgs",
+            "repos_url": "https://api.github.com/users/MYORG/repos",
+            "events_url": "https://api.github.com/users/MYORG/events{/privacy}",
+            "received_events_url": "https://api.github.com/users/MYORG/received_events",
+            "type": "Organization",
+            "site_admin": false
+        },
+        "html_url": "https://github.com/MYORG/MYREPO",
+        "description": "MYREPO",
+        "fork": false,
+        "url": "https://api.github.com/repos/MYORG/MYREPO",
+        "forks_url": "https://api.github.com/repos/MYORG/MYREPO/forks",
+        "keys_url": "https://api.github.com/repos/MYORG/MYREPO/keys{/key_id}",
+        "collaborators_url": "https://api.github.com/repos/MYORG/MYREPO/collaborators{/collaborator}",
+        "teams_url": "https://api.github.com/repos/MYORG/MYREPO/teams",
+        "hooks_url": "https://api.github.com/repos/MYORG/MYREPO/hooks",
+        "issue_events_url": "https://api.github.com/repos/MYORG/MYREPO/issues/events{/number}",
+        "events_url": "https://api.github.com/repos/MYORG/MYREPO/events",
+        "assignees_url": "https://api.github.com/repos/MYORG/MYREPO/assignees{/user}",
+        "branches_url": "https://api.github.com/repos/MYORG/MYREPO/branches{/branch}",
+        "tags_url": "https://api.github.com/repos/MYORG/MYREPO/tags",
+        "blobs_url": "https://api.github.com/repos/MYORG/MYREPO/git/blobs{/sha}",
+        "git_tags_url": "https://api.github.com/repos/MYORG/MYREPO/git/tags{/sha}",
+        "git_refs_url": "https://api.github.com/repos/MYORG/MYREPO/git/refs{/sha}",
+        "trees_url": "https://api.github.com/repos/MYORG/MYREPO/git/trees{/sha}",
+        "statuses_url": "https://api.github.com/repos/MYORG/MYREPO/statuses/{sha}",
+        "languages_url": "https://api.github.com/repos/MYORG/MYREPO/languages",
+        "stargazers_url": "https://api.github.com/repos/MYORG/MYREPO/stargazers",
+        "contributors_url": "https://api.github.com/repos/MYORG/MYREPO/contributors",
+        "subscribers_url": "https://api.github.com/repos/MYORG/MYREPO/subscribers",
+        "subscription_url": "https://api.github.com/repos/MYORG/MYREPO/subscription",
+        "commits_url": "https://api.github.com/repos/MYORG/MYREPO/commits{/sha}",
+        "git_commits_url": "https://api.github.com/repos/MYORG/MYREPO/git/commits{/sha}",
+        "comments_url": "https://api.github.com/repos/MYORG/MYREPO/comments{/number}",
+        "issue_comment_url": "https://api.github.com/repos/MYORG/MYREPO/issues/comments{/number}",
+        "contents_url": "https://api.github.com/repos/MYORG/MYREPO/contents/{+path}",
+        "compare_url": "https://api.github.com/repos/MYORG/MYREPO/compare/{base}...{head}",
+        "merges_url": "https://api.github.com/repos/MYORG/MYREPO/merges",
+        "archive_url": "https://api.github.com/repos/MYORG/MYREPO/{archive_format}{/ref}",
+        "downloads_url": "https://api.github.com/repos/MYORG/MYREPO/downloads",
+        "issues_url": "https://api.github.com/repos/MYORG/MYREPO/issues{/number}",
+        "pulls_url": "https://api.github.com/repos/MYORG/MYREPO/pulls{/number}",
+        "milestones_url": "https://api.github.com/repos/MYORG/MYREPO/milestones{/number}",
+        "notifications_url": "https://api.github.com/repos/MYORG/MYREPO/notifications{?since,all,participating}",
+        "labels_url": "https://api.github.com/repos/MYORG/MYREPO/labels{/name}",
+        "releases_url": "https://api.github.com/repos/MYORG/MYREPO/releases{/id}",
+        "deployments_url": "https://api.github.com/repos/MYORG/MYREPO/deployments",
+        "created_at": "2021-09-10T18:55:38Z",
+        "updated_at": "2021-09-10T18:55:41Z",
+        "pushed_at": "2021-09-28T23:25:26Z",
+        "git_url": "git://github.com/MYORG/MYREPO.git",
+        "ssh_url": "git@github.com:MYORG/MYREPO.git",
+        "clone_url": "https://github.com/MYORG/MYREPO.git",
+        "svn_url": "https://github.com/MYORG/MYREPO",
+        "homepage": null,
+        "size": 121,
+        "stargazers_count": 0,
+        "watchers_count": 0,
+        "language": null,
+        "has_issues": true,
+        "has_projects": true,
+        "has_downloads": true,
+        "has_wiki": true,
+        "has_pages": false,
+        "forks_count": 0,
+        "mirror_url": null,
+        "archived": false,
+        "disabled": false,
+        "open_issues_count": 1,
+        "license": null,
+        "allow_forking": false,
+        "forks": 0,
+        "open_issues": 1,
+        "watchers": 0,
+        "default_branch": "master"
+    },
+    "organization": {
+        "login": "MYORG",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "url": "https://api.github.com/orgs/MYORG",
+        "repos_url": "https://api.github.com/orgs/MYORG/repos",
+        "events_url": "https://api.github.com/orgs/MYORG/events",
+        "hooks_url": "https://api.github.com/orgs/MYORG/hooks",
+        "issues_url": "https://api.github.com/orgs/MYORG/issues",
+        "members_url": "https://api.github.com/orgs/MYORG/members{/member}",
+        "public_members_url": "https://api.github.com/orgs/MYORG/public_members{/member}",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "description": ""
+    },
+    "sender": {
+        "login": "MYNAME",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/MYNAME",
+        "html_url": "https://github.com/MYNAME",
+        "followers_url": "https://api.github.com/users/MYNAME/followers",
+        "following_url": "https://api.github.com/users/MYNAME/following{/other_user}",
+        "gists_url": "https://api.github.com/users/MYNAME/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/MYNAME/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/MYNAME/subscriptions",
+        "organizations_url": "https://api.github.com/users/MYNAME/orgs",
+        "repos_url": "https://api.github.com/users/MYNAME/repos",
+        "events_url": "https://api.github.com/users/MYNAME/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/MYNAME/received_events",
+        "type": "User",
+        "site_admin": false
+    }
+}

controllers/actions.summerwind.net/testdata/org_webhook_workflow_job_with_self_hosted_label_payload.json

@@ -0,0 +1,152 @@
+{
+    "action": "queued",
+    "workflow_job": {
+        "id": 1234567890,
+        "run_id": 1234567890,
+        "run_url": "https://api.github.com/repos/MYORG/MYREPO/actions/runs/1234567890",
+        "node_id": "CR_kwDOGCados7e1x2g",
+        "head_sha": "1234567890123456789012345678901234567890",
+        "url": "https://api.github.com/repos/MYORG/MYREPO/actions/jobs/1234567890",
+        "html_url": "https://github.com/MYORG/MYREPO/runs/1234567890",
+        "status": "queued",
+        "conclusion": null,
+        "started_at": "2021-09-28T23:45:29Z",
+        "completed_at": null,
+        "name": "build",
+        "steps": [],
+        "check_run_url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/1234567890",
+        "labels": [
+            "self-hosted",
+            "label1"
+        ]
+    },
+    "repository": {
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ=",
+        "name": "MYREPO",
+        "full_name": "MYORG/MYREPO",
+        "private": true,
+        "owner": {
+            "login": "MYORG",
+            "id": 1234567890,
+            "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+            "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+            "gravatar_id": "",
+            "url": "https://api.github.com/users/MYORG",
+            "html_url": "https://github.com/MYORG",
+            "followers_url": "https://api.github.com/users/MYORG/followers",
+            "following_url": "https://api.github.com/users/MYORG/following{/other_user}",
+            "gists_url": "https://api.github.com/users/MYORG/gists{/gist_id}",
+            "starred_url": "https://api.github.com/users/MYORG/starred{/owner}{/repo}",
+            "subscriptions_url": "https://api.github.com/users/MYORG/subscriptions",
+            "organizations_url": "https://api.github.com/users/MYORG/orgs",
+            "repos_url": "https://api.github.com/users/MYORG/repos",
+            "events_url": "https://api.github.com/users/MYORG/events{/privacy}",
+            "received_events_url": "https://api.github.com/users/MYORG/received_events",
+            "type": "Organization",
+            "site_admin": false
+        },
+        "html_url": "https://github.com/MYORG/MYREPO",
+        "description": "MYREPO",
+        "fork": false,
+        "url": "https://api.github.com/repos/MYORG/MYREPO",
+        "forks_url": "https://api.github.com/repos/MYORG/MYREPO/forks",
+        "keys_url": "https://api.github.com/repos/MYORG/MYREPO/keys{/key_id}",
+        "collaborators_url": "https://api.github.com/repos/MYORG/MYREPO/collaborators{/collaborator}",
+        "teams_url": "https://api.github.com/repos/MYORG/MYREPO/teams",
+        "hooks_url": "https://api.github.com/repos/MYORG/MYREPO/hooks",
+        "issue_events_url": "https://api.github.com/repos/MYORG/MYREPO/issues/events{/number}",
+        "events_url": "https://api.github.com/repos/MYORG/MYREPO/events",
+        "assignees_url": "https://api.github.com/repos/MYORG/MYREPO/assignees{/user}",
+        "branches_url": "https://api.github.com/repos/MYORG/MYREPO/branches{/branch}",
+        "tags_url": "https://api.github.com/repos/MYORG/MYREPO/tags",
+        "blobs_url": "https://api.github.com/repos/MYORG/MYREPO/git/blobs{/sha}",
+        "git_tags_url": "https://api.github.com/repos/MYORG/MYREPO/git/tags{/sha}",
+        "git_refs_url": "https://api.github.com/repos/MYORG/MYREPO/git/refs{/sha}",
+        "trees_url": "https://api.github.com/repos/MYORG/MYREPO/git/trees{/sha}",
+        "statuses_url": "https://api.github.com/repos/MYORG/MYREPO/statuses/{sha}",
+        "languages_url": "https://api.github.com/repos/MYORG/MYREPO/languages",
+        "stargazers_url": "https://api.github.com/repos/MYORG/MYREPO/stargazers",
+        "contributors_url": "https://api.github.com/repos/MYORG/MYREPO/contributors",
+        "subscribers_url": "https://api.github.com/repos/MYORG/MYREPO/subscribers",
+        "subscription_url": "https://api.github.com/repos/MYORG/MYREPO/subscription",
+        "commits_url": "https://api.github.com/repos/MYORG/MYREPO/commits{/sha}",
+        "git_commits_url": "https://api.github.com/repos/MYORG/MYREPO/git/commits{/sha}",
+        "comments_url": "https://api.github.com/repos/MYORG/MYREPO/comments{/number}",
+        "issue_comment_url": "https://api.github.com/repos/MYORG/MYREPO/issues/comments{/number}",
+        "contents_url": "https://api.github.com/repos/MYORG/MYREPO/contents/{+path}",
+        "compare_url": "https://api.github.com/repos/MYORG/MYREPO/compare/{base}...{head}",
+        "merges_url": "https://api.github.com/repos/MYORG/MYREPO/merges",
+        "archive_url": "https://api.github.com/repos/MYORG/MYREPO/{archive_format}{/ref}",
+        "downloads_url": "https://api.github.com/repos/MYORG/MYREPO/downloads",
+        "issues_url": "https://api.github.com/repos/MYORG/MYREPO/issues{/number}",
+        "pulls_url": "https://api.github.com/repos/MYORG/MYREPO/pulls{/number}",
+        "milestones_url": "https://api.github.com/repos/MYORG/MYREPO/milestones{/number}",
+        "notifications_url": "https://api.github.com/repos/MYORG/MYREPO/notifications{?since,all,participating}",
+        "labels_url": "https://api.github.com/repos/MYORG/MYREPO/labels{/name}",
+        "releases_url": "https://api.github.com/repos/MYORG/MYREPO/releases{/id}",
+        "deployments_url": "https://api.github.com/repos/MYORG/MYREPO/deployments",
+        "created_at": "2021-09-10T18:55:38Z",
+        "updated_at": "2021-09-10T18:55:41Z",
+        "pushed_at": "2021-09-28T23:25:26Z",
+        "git_url": "git://github.com/MYORG/MYREPO.git",
+        "ssh_url": "git@github.com:MYORG/MYREPO.git",
+        "clone_url": "https://github.com/MYORG/MYREPO.git",
+        "svn_url": "https://github.com/MYORG/MYREPO",
+        "homepage": null,
+        "size": 121,
+        "stargazers_count": 0,
+        "watchers_count": 0,
+        "language": null,
+        "has_issues": true,
+        "has_projects": true,
+        "has_downloads": true,
+        "has_wiki": true,
+        "has_pages": false,
+        "forks_count": 0,
+        "mirror_url": null,
+        "archived": false,
+        "disabled": false,
+        "open_issues_count": 1,
+        "license": null,
+        "allow_forking": false,
+        "forks": 0,
+        "open_issues": 1,
+        "watchers": 0,
+        "default_branch": "master"
+    },
+    "organization": {
+        "login": "MYORG",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "url": "https://api.github.com/orgs/MYORG",
+        "repos_url": "https://api.github.com/orgs/MYORG/repos",
+        "events_url": "https://api.github.com/orgs/MYORG/events",
+        "hooks_url": "https://api.github.com/orgs/MYORG/hooks",
+        "issues_url": "https://api.github.com/orgs/MYORG/issues",
+        "members_url": "https://api.github.com/orgs/MYORG/members{/member}",
+        "public_members_url": "https://api.github.com/orgs/MYORG/public_members{/member}",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "description": ""
+    },
+    "sender": {
+        "login": "MYNAME",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/MYNAME",
+        "html_url": "https://github.com/MYNAME",
+        "followers_url": "https://api.github.com/users/MYNAME/followers",
+        "following_url": "https://api.github.com/users/MYNAME/following{/other_user}",
+        "gists_url": "https://api.github.com/users/MYNAME/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/MYNAME/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/MYNAME/subscriptions",
+        "organizations_url": "https://api.github.com/users/MYNAME/orgs",
+        "repos_url": "https://api.github.com/users/MYNAME/repos",
+        "events_url": "https://api.github.com/users/MYNAME/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/MYNAME/received_events",
+        "type": "User",
+        "site_admin": false
+    }
+}

controllers/actions.summerwind.net/testdata/repo_webhook_check_run_payload.json

@@ -0,0 +1,360 @@
+{
+  "action": "completed",
+  "check_run": {
+    "id": 1949438388,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "head_sha": "1234567890123456789012345678901234567890",
+    "external_id": "ca395085-040a-526b-2ce8-bdc85f692774",
+    "url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890",
+    "html_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "details_url": "https://github.com/MYORG/MYREPO/runs/123467890",
+    "status": "queued",
+    "conclusion": null,
+    "started_at": "2021-02-18T06:16:31Z",
+    "completed_at": null,
+    "output": {
+      "title": null,
+      "summary": null,
+      "text": null,
+      "annotations_count": 0,
+      "annotations_url": "https://api.github.com/repos/MYORG/MYREPO/check-runs/123467890/annotations"
+    },
+    "name": "build",
+    "name": "validate",
+    "check_suite": {
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "head_branch": "MYNAME/actions-runner-controller-webhook",
+      "head_sha": "1234567890123456789012345678901234567890",
+      "status": "queued",
+      "conclusion": null,
+      "url": "https://api.github.com/repos/MYORG/MYREPO/check-suites/1234567890",
+      "before": "1234567890123456789012345678901234567890",
+      "after": "1234567890123456789012345678901234567890",
+      "pull_requests": [
+        {
+          "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/2033",
+          "id": 1234567890,
+          "number": 1234567890,
+          "head": {
+            "ref": "feature",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          },
+          "base": {
+            "ref": "master",
+            "sha": "1234567890123456789012345678901234567890",
+            "repo": {
+              "id": 1234567890,
+              "url": "https://api.github.com/repos/MYORG/MYREPO",
+              "name": "MYREPO"
+            }
+          }
+        }
+      ],
+      "app": {
+        "id": 1234567890,
+        "slug": "github-actions",
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "owner": {
+          "login": "github",
+          "id": 1234567890,
+          "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+          "avatar_url": "https://avatars.githubusercontent.com/u/123467890?v=4",
+          "gravatar_id": "",
+          "url": "https://api.github.com/users/github",
+          "html_url": "https://github.com/github",
+          "followers_url": "https://api.github.com/users/github/followers",
+          "following_url": "https://api.github.com/users/github/following{/other_user}",
+          "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+          "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+          "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+          "organizations_url": "https://api.github.com/users/github/orgs",
+          "repos_url": "https://api.github.com/users/github/repos",
+          "events_url": "https://api.github.com/users/github/events{/privacy}",
+          "received_events_url": "https://api.github.com/users/github/received_events",
+          "type": "Organization",
+          "site_admin": false
+        },
+        "name": "GitHub Actions",
+        "description": "Automate your workflow from idea to production",
+        "external_url": "https://help.github.com/en/actions",
+        "html_url": "https://github.com/apps/github-actions",
+        "created_at": "2018-07-30T09:30:17Z",
+        "updated_at": "2019-12-10T19:04:12Z",
+        "permissions": {
+          "actions": "write",
+          "checks": "write",
+          "contents": "write",
+          "deployments": "write",
+          "issues": "write",
+          "metadata": "read",
+          "organization_packages": "write",
+          "packages": "write",
+          "pages": "write",
+          "pull_requests": "write",
+          "repository_hooks": "write",
+          "repository_projects": "write",
+          "security_events": "write",
+          "statuses": "write",
+          "vulnerability_alerts": "read"
+        },
+        "events": [
+          "check_run",
+          "check_suite",
+          "create",
+          "delete",
+          "deployment",
+          "deployment_status",
+          "fork",
+          "gollum",
+          "issues",
+          "issue_comment",
+          "label",
+          "milestone",
+          "page_build",
+          "project",
+          "project_card",
+          "project_column",
+          "public",
+          "pull_request",
+          "pull_request_review",
+          "pull_request_review_comment",
+          "push",
+          "registry_package",
+          "release",
+          "repository",
+          "repository_dispatch",
+          "status",
+          "watch",
+          "workflow_dispatch",
+          "workflow_run"
+        ]
+      },
+      "created_at": "2021-02-18T06:15:32Z",
+      "updated_at": "2021-02-18T06:16:31Z"
+    },
+    "app": {
+      "id": 1234567890,
+      "slug": "github-actions",
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "owner": {
+        "login": "github",
+        "id": 1234567890,
+        "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+        "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+        "gravatar_id": "",
+        "url": "https://api.github.com/users/github",
+        "html_url": "https://github.com/github",
+        "followers_url": "https://api.github.com/users/github/followers",
+        "following_url": "https://api.github.com/users/github/following{/other_user}",
+        "gists_url": "https://api.github.com/users/github/gists{/gist_id}",
+        "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}",
+        "subscriptions_url": "https://api.github.com/users/github/subscriptions",
+        "organizations_url": "https://api.github.com/users/github/orgs",
+        "repos_url": "https://api.github.com/users/github/repos",
+        "events_url": "https://api.github.com/users/github/events{/privacy}",
+        "received_events_url": "https://api.github.com/users/github/received_events",
+        "type": "Organization",
+        "site_admin": false
+      },
+      "name": "GitHub Actions",
+      "description": "Automate your workflow from idea to production",
+      "external_url": "https://help.github.com/en/actions",
+      "html_url": "https://github.com/apps/github-actions",
+      "created_at": "2018-07-30T09:30:17Z",
+      "updated_at": "2019-12-10T19:04:12Z",
+      "permissions": {
+        "actions": "write",
+        "checks": "write",
+        "contents": "write",
+        "deployments": "write",
+        "issues": "write",
+        "metadata": "read",
+        "organization_packages": "write",
+        "packages": "write",
+        "pages": "write",
+        "pull_requests": "write",
+        "repository_hooks": "write",
+        "repository_projects": "write",
+        "security_events": "write",
+        "statuses": "write",
+        "vulnerability_alerts": "read"
+      },
+      "events": [
+        "check_run",
+        "check_suite",
+        "create",
+        "delete",
+        "deployment",
+        "deployment_status",
+        "fork",
+        "gollum",
+        "issues",
+        "issue_comment",
+        "label",
+        "milestone",
+        "page_build",
+        "project",
+        "project_card",
+        "project_column",
+        "public",
+        "pull_request",
+        "pull_request_review",
+        "pull_request_review_comment",
+        "push",
+        "registry_package",
+        "release",
+        "repository",
+        "repository_dispatch",
+        "status",
+        "watch",
+        "workflow_dispatch",
+        "workflow_run"
+      ]
+    },
+    "pull_requests": [
+      {
+        "url": "https://api.github.com/repos/MYORG/MYREPO/pulls/1234567890",
+        "id": 1234567890,
+        "number": 1234567890,
+        "head": {
+          "ref": "feature",
+          "sha": "1234567890123456789012345678901234567890",
+          "repo": {
+            "id": 1234567890,
+            "url": "https://api.github.com/repos/MYORG/MYREPO",
+            "name": "MYREPO"
+          }
+        },
+        "base": {
+          "ref": "master",
+          "sha": "1234567890123456789012345678901234567890",
+          "repo": {
+            "id": 1234567890,
+            "url": "https://api.github.com/repos/MYORG/MYREPO",
+            "name": "MYREPO"
+          }
+        }
+      }
+    ]
+  },
+  "repository": {
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "name": "MYREPO",
+    "full_name": "MYORG/MYREPO",
+    "private": true,
+    "owner": {
+      "login": "MYUSER",
+      "id": 1234567890,
+      "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/MYUSER",
+      "html_url": "https://github.com/MYUSER",
+      "followers_url": "https://api.github.com/users/MYUSER/followers",
+      "following_url": "https://api.github.com/users/MYUSER/following{/other_user}",
+      "gists_url": "https://api.github.com/users/MYUSER/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/MYUSER/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/MYUSER/subscriptions",
+      "organizations_url": "https://api.github.com/users/MYUSER/orgs",
+      "repos_url": "https://api.github.com/users/MYUSER/repos",
+      "events_url": "https://api.github.com/users/MYUSER/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/MYUSER/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "html_url": "https://github.com/MYUSER/MYREPO",
+    "description": null,
+    "fork": false,
+    "url": "https://api.github.com/repos/MYUSER/MYREPO",
+    "forks_url": "https://api.github.com/repos/MYUSER/MYREPO/forks",
+    "keys_url": "https://api.github.com/repos/MYUSER/MYREPO/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/MYUSER/MYREPO/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/MYUSER/MYREPO/teams",
+    "hooks_url": "https://api.github.com/repos/MYUSER/MYREPO/hooks",
+    "issue_events_url": "https://api.github.com/repos/MYUSER/MYREPO/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/MYUSER/MYREPO/events",
+    "assignees_url": "https://api.github.com/repos/MYUSER/MYREPO/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/MYUSER/MYREPO/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/MYUSER/MYREPO/tags",
+    "blobs_url": "https://api.github.com/repos/MYUSER/MYREPO/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/MYUSER/MYREPO/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/MYUSER/MYREPO/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/MYUSER/MYREPO/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/MYUSER/MYREPO/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/MYUSER/MYREPO/languages",
+    "stargazers_url": "https://api.github.com/repos/MYUSER/MYREPO/stargazers",
+    "contributors_url": "https://api.github.com/repos/MYUSER/MYREPO/contributors",
+    "subscribers_url": "https://api.github.com/repos/MYUSER/MYREPO/subscribers",
+    "subscription_url": "https://api.github.com/repos/MYUSER/MYREPO/subscription",
+    "commits_url": "https://api.github.com/repos/MYUSER/MYREPO/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/MYUSER/MYREPO/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/MYUSER/MYREPO/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/MYUSER/MYREPO/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/MYUSER/MYREPO/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/MYUSER/MYREPO/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/MYUSER/MYREPO/merges",
+    "archive_url": "https://api.github.com/repos/MYUSER/MYREPO/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/MYUSER/MYREPO/downloads",
+    "issues_url": "https://api.github.com/repos/MYUSER/MYREPO/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/MYUSER/MYREPO/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/MYUSER/MYREPO/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/MYUSER/MYREPO/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/MYUSER/MYREPO/labels{/name}",
+    "releases_url": "https://api.github.com/repos/MYUSER/MYREPO/releases{/id}",
+    "deployments_url": "https://api.github.com/repos/MYUSER/MYREPO/deployments",
+    "created_at": "2021-02-18T06:16:31Z",
+    "updated_at": "2021-02-18T06:16:31Z",
+    "pushed_at": "2021-02-18T06:16:31Z",
+    "git_url": "git://github.com/MYUSER/MYREPO.git",
+    "ssh_url": "git@github.com:MYUSER/MYREPO.git",
+    "clone_url": "https://github.com/MYUSER/MYREPO.git",
+    "svn_url": "https://github.com/MYUSER/MYREPO",
+    "homepage": null,
+    "size": 4,
+    "stargazers_count": 0,
+    "watchers_count": 0,
+    "language": null,
+    "has_issues": true,
+    "has_projects": true,
+    "has_downloads": true,
+    "has_wiki": true,
+    "has_pages": false,
+    "forks_count": 0,
+    "mirror_url": null,
+    "archived": false,
+    "disabled": false,
+    "open_issues_count": 0,
+    "license": null,
+    "forks": 0,
+    "open_issues": 0,
+    "watchers": 0,
+    "default_branch": "main"
+  },
+  "sender": {
+    "login": "MYUSER",
+    "id": 1234567890,
+    "node_id": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    "avatar_url": "https://avatars.githubusercontent.com/u/1234567890?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/MYUSER",
+    "html_url": "https://github.com/MYUSER",
+    "followers_url": "https://api.github.com/users/MYUSER/followers",
+    "following_url": "https://api.github.com/users/MYUSER/following{/other_user}",
+    "gists_url": "https://api.github.com/users/MYUSER/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/MYUSER/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/MYUSER/subscriptions",
+    "organizations_url": "https://api.github.com/users/MYUSER/orgs",
+    "repos_url": "https://api.github.com/users/MYUSER/repos",
+    "events_url": "https://api.github.com/users/MYUSER/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/MYUSER/received_events",
+    "type": "User",
+    "site_admin": false
+  }
+}

controllers/actions.summerwind.net/testresourcereader.go

@@ -0,0 +1,32 @@
+package controllers
+
+import (
+	"context"
+	"errors"
+	"reflect"
+
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type testResourceReader struct {
+	objects map[types.NamespacedName]client.Object
+}
+
+func (r *testResourceReader) Get(_ context.Context, key client.ObjectKey, obj client.Object, _ ...client.GetOption) error {
+	nsName := types.NamespacedName{Namespace: key.Namespace, Name: key.Name}
+	ret, ok := r.objects[nsName]
+	if !ok {
+		return &kerrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonNotFound}}
+	}
+	v := reflect.ValueOf(obj)
+	if v.Kind() != reflect.Ptr {
+		return errors.New("obj must be a pointer")
+	}
+
+	v.Elem().Set(reflect.ValueOf(ret).Elem())
+
+	return nil
+}

controllers/actions.summerwind.net/testresourcereader_test.go

@@ -0,0 +1,35 @@
+package controllers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestResourceReader(t *testing.T) {
+	rr := &testResourceReader{
+		objects: map[types.NamespacedName]client.Object{
+			{Namespace: "default", Name: "sec1"}: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "sec1",
+				},
+				Data: map[string][]byte{
+					"foo": []byte("bar"),
+				},
+			},
+		},
+	}
+
+	var sec corev1.Secret
+
+	err := rr.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "sec1"}, &sec)
+	require.NoError(t, err)
+
+	require.Equal(t, []byte("bar"), sec.Data["foo"])
+}

controllers/actions.summerwind.net/utils.go

@@ -0,0 +1,13 @@
+package controllers
+
+func filterLabels(labels map[string]string, filter string) map[string]string {
+	filtered := map[string]string{}
+
+	for k, v := range labels {
+		if k != filter {
+			filtered[k] = v
+		}
+	}
+
+	return filtered
+}

controllers/actions.summerwind.net/utils_test.go

@@ -0,0 +1,128 @@
+package controllers
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/actions/actions-runner-controller/apis/actions.summerwind.net/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func Test_filterLabels(t *testing.T) {
+	type args struct {
+		labels map[string]string
+		filter string
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[string]string
+	}{
+		{
+			name: "ok",
+			args: args{
+				labels: map[string]string{LabelKeyRunnerTemplateHash: "abc", LabelKeyPodTemplateHash: "def"},
+				filter: LabelKeyRunnerTemplateHash,
+			},
+			want: map[string]string{LabelKeyPodTemplateHash: "def"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := filterLabels(tt.args.labels, tt.args.filter); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("filterLabels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_workVolumeClaimTemplateVolumeV1VolumeTransformation(t *testing.T) {
+	storageClassName := "local-storage"
+	workVolumeClaimTemplate := v1alpha1.WorkVolumeClaimTemplate{
+		StorageClassName: storageClassName,
+		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+		Resources:        corev1.ResourceRequirements{},
+	}
+	want := corev1.Volume{
+		Name: "work",
+		VolumeSource: corev1.VolumeSource{
+			Ephemeral: &corev1.EphemeralVolumeSource{
+				VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+					Spec: corev1.PersistentVolumeClaimSpec{
+						AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+						StorageClassName: &storageClassName,
+						Resources:        corev1.ResourceRequirements{},
+					},
+				},
+			},
+		},
+	}
+
+	got := workVolumeClaimTemplate.V1Volume()
+
+	if got.Name != want.Name {
+		t.Errorf("want name %q, got %q\n", want.Name, got.Name)
+	}
+
+	if got.VolumeSource.Ephemeral == nil {
+		t.Fatal("work volume claim template should transform itself into Ephemeral volume source\n")
+	}
+
+	if got.VolumeSource.Ephemeral.VolumeClaimTemplate == nil {
+		t.Fatal("work volume claim template should have ephemeral volume claim template set\n")
+	}
+
+	gotClassName := *got.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName
+	wantClassName := *want.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName
+	if gotClassName != wantClassName {
+		t.Errorf("expected storage class name %q, got %q\n", wantClassName, gotClassName)
+	}
+
+	gotAccessModes := got.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes
+	wantAccessModes := want.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes
+	if len(gotAccessModes) != len(wantAccessModes) {
+		t.Fatalf("access modes lengths missmatch: got %v, expected %v\n", gotAccessModes, wantAccessModes)
+	}
+
+	diff := make(map[corev1.PersistentVolumeAccessMode]int, len(wantAccessModes))
+	for _, am := range wantAccessModes {
+		diff[am]++
+	}
+
+	for _, am := range gotAccessModes {
+		_, ok := diff[am]
+		if !ok {
+			t.Errorf("got access mode %v that is not in the wanted access modes\n", am)
+		}
+
+		diff[am]--
+		if diff[am] == 0 {
+			delete(diff, am)
+		}
+	}
+
+	if len(diff) != 0 {
+		t.Fatalf("got access modes did not take every access mode into account\nactual: %v expected: %v\n", gotAccessModes, wantAccessModes)
+	}
+}
+
+func Test_workVolumeClaimTemplateV1VolumeMount(t *testing.T) {
+
+	workVolumeClaimTemplate := v1alpha1.WorkVolumeClaimTemplate{
+		StorageClassName: "local-storage",
+		AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany},
+		Resources:        corev1.ResourceRequirements{},
+	}
+
+	mountPath := "/test/_work"
+	want := corev1.VolumeMount{
+		MountPath: mountPath,
+		Name:      "work",
+	}
+
+	got := workVolumeClaimTemplate.V1VolumeMount(mountPath)
+
+	if want != got {
+		t.Fatalf("expected volume mount %+v, actual %+v\n", want, got)
+	}
+}