feat: HorizontalRunnerAutoscaler Webhook server (#282)

* feat: HorizontalRunnerAutoscaler Webhook server

This introduces a Webhook server that responds GitHub `check_run`, `pull_request`, and `push` events by scaling up matched HorizontalRunnerAutoscaler by 1 replica. This allows you to immediately add "resource slack" for future GitHub Actions job runs, without waiting next sync period to add insufficient runners.

This feature is highly inspired by https://github.com/philips-labs/terraform-aws-github-runner. terraform-aws-github-runner can manage one set of runners per deployment, where actions-runner-controller with this feature can manage as many sets of runners as you declare with HorizontalRunnerAutoscaler and RunnerDeployment pairs.

On each GitHub event received, the webhook server queries repository-wide and organizational runners from the cluster and searches for the single target to scale up. The webhook server tries to match HorizontalRunnerAutoscaler.Spec.ScaleUpTriggers[].GitHubEvent.[CheckRun|Push|PullRequest] against the event and if it finds only one HRA, it is the scale target. If none or two or more targets are found for repository-wide runners, it does the same on organizational runners.

Changes:

* Fix integration test
* Update manifests
* chart: Add support for github webhook server
* dockerfile: Include github-webhook-server binary
* Do not import unversioned go-github
* Update README

controllers/integration_test.go

@@ -2,6 +2,11 @@ package controllers
 
 import (
 	"context"
+	"github.com/google/go-github/v33/github"
+	github3 "github.com/google/go-github/v33/github"
+	github2 "github.com/summerwind/actions-runner-controller/github"
+	"net/http"
+	"net/http/httptest"
 	"time"
 
 	"github.com/summerwind/actions-runner-controller/github/fake"
@@ -30,6 +35,12 @@ var (
 	workflowRunsFor1Replicas = `{"total_count": 6, "workflow_runs":[{"status":"queued"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}, {"status":"completed"}]}"`
 )
 
+var webhookServer *httptest.Server
+
+var ghClient *github2.Client
+
+var fakeRunnerList *fake.RunnersList
+
 // SetupIntegrationTest will set up a testing environment.
 // This includes:
 // * creating a Namespace to be used during the test
@@ -41,10 +52,13 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 	ns := &corev1.Namespace{}
 
 	responses := &fake.FixedResponses{}
+	responses.ListRunners = fake.DefaultListRunnersHandler()
 	responses.ListRepositoryWorkflowRuns = &fake.Handler{
 		Status: 200,
 		Body:   workflowRunsFor3Replicas,
 	}
+	fakeRunnerList = fake.NewRunnersList()
+	responses.ListRunners = fakeRunnerList.HandleList()
 	fakeGithubServer := fake.NewServer(fake.WithFixedResponses(responses))
 
 	BeforeEach(func() {
@@ -59,9 +73,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
 		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
 
-		runnersList = fake.NewRunnersList()
-		server = runnersList.GetServer()
-		ghClient := newGithubClient(server)
+		ghClient = newGithubClient(fakeGithubServer)
 
 		replicasetController := &RunnerReplicaSetReconciler{
 			Client:       mgr.GetClient(),
@@ -85,15 +97,30 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		client := newGithubClient(fakeGithubServer)
 
 		autoscalerController := &HorizontalRunnerAutoscalerReconciler{
-			Client:       mgr.GetClient(),
-			Scheme:       scheme.Scheme,
-			Log:          logf.Log,
-			GitHubClient: client,
-			Recorder:     mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+			Client:        mgr.GetClient(),
+			Scheme:        scheme.Scheme,
+			Log:           logf.Log,
+			GitHubClient:  client,
+			Recorder:      mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+			CacheDuration: 1 * time.Second,
 		}
 		err = autoscalerController.SetupWithManager(mgr)
 		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
 
+		autoscalerWebhook := &HorizontalRunnerAutoscalerGitHubWebhook{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("horizontalrunnerautoscaler-controller"),
+		}
+		err = autoscalerWebhook.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup autoscaler webhook")
+
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", autoscalerWebhook.Handle)
+
+		webhookServer = httptest.NewServer(mux)
+
 		go func() {
 			defer GinkgoRecover()
 
@@ -106,6 +133,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 		close(stopCh)
 
 		fakeGithubServer.Close()
+		webhookServer.Close()
 
 		err := k8sClient.Delete(ctx, ns)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
@@ -114,7 +142,7 @@ func SetupIntegrationTest(ctx context.Context) *testEnvironment {
 	return &testEnvironment{Namespace: ns, Responses: responses}
 }
 
-var _ = Context("Inside of a new namespace", func() {
+var _ = Context("INTEGRATION: Inside of a new namespace", func() {
 	ctx := context.TODO()
 	env := SetupIntegrationTest(ctx)
 	ns := env.Namespace
@@ -235,8 +263,20 @@ var _ = Context("Inside of a new namespace", func() {
 						},
 						MinReplicas:                       intPtr(1),
 						MaxReplicas:                       intPtr(3),
-						ScaleDownDelaySecondsAfterScaleUp: nil,
+						ScaleDownDelaySecondsAfterScaleUp: intPtr(1),
 						Metrics:                           nil,
+						ScaleUpTriggers: []actionsv1alpha1.ScaleUpTrigger{
+							{
+								GitHubEvent: &actionsv1alpha1.GitHubEventScaleUpTriggerSpec{
+									PullRequest: &actionsv1alpha1.PullRequestSpec{
+										Types:    []string{"created"},
+										Branches: []string{"main"},
+									},
+								},
+								Amount:   1,
+								Duration: metav1.Duration{Duration: time.Minute},
+							},
+						},
 					},
 				}
 
@@ -274,8 +314,33 @@ var _ = Context("Inside of a new namespace", func() {
 					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(3))
 			}
 
+			{
+				var runnerList actionsv1alpha1.RunnerList
+
+				err := k8sClient.List(ctx, &runnerList, client.InNamespace(ns.Name))
+				if err != nil {
+					logf.Log.Error(err, "list runners")
+				}
+
+				for i, r := range runnerList.Items {
+					fakeRunnerList.Add(&github3.Runner{
+						ID:     github.Int64(int64(i)),
+						Name:   github.String(r.Name),
+						OS:     github.String("linux"),
+						Status: github.String("online"),
+						Busy:   github.Bool(false),
+					})
+				}
+
+				rs, err := ghClient.ListRunners(context.Background(), "", "", "test/valid")
+				Expect(err).NotTo(HaveOccurred(), "verifying list fake runners response")
+				Expect(len(rs)).To(Equal(3), "count of fake list runners")
+			}
+
 			// Scale-down to 1 replica
 			{
+				time.Sleep(time.Second)
+
 				responses.ListRepositoryWorkflowRuns.Body = workflowRunsFor1Replicas
 
 				var hra actionsv1alpha1.HorizontalRunnerAutoscaler
@@ -308,7 +373,60 @@ var _ = Context("Inside of a new namespace", func() {
 
 						return *runnerSets.Items[0].Spec.Replicas
 					},
-					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runners after HRA force update for scale-down")
+			}
+
+			{
+				resp, err := sendWebhook(webhookServer, "pull_request", &github.PullRequestEvent{
+					PullRequest: &github.PullRequest{
+						Base: &github.PullRequestBranch{
+							Ref: github.String("main"),
+						},
+					},
+					Repo: &github.Repository{
+						Name: github.String("test/valid"),
+						Organization: &github.Organization{
+							Name: github.String("test"),
+						},
+					},
+					Action: github.String("created"),
+				})
+
+				Expect(err).NotTo(HaveOccurred(), "failed to send pull_request event")
+
+				Expect(resp.StatusCode).To(Equal(200))
+			}
+
+			// Scale-up to 2 replicas
+			{
+				runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1), "runner sets after webhook")
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnerreplicasets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2), "runners after webhook")
 			}
 		})
 	})