feat: RunnerSet backed by StatefulSet (#629)

* feat: RunnerSet backed by StatefulSet

Unlike a runner deployment, a runner set can manage a set of stateful runners by combining a statefulset and an admission webhook that mutates statefulset-managed pods with required envvars and registration tokens.

Resolves #613
Ref #612

* Upgrade controller-runtime to 0.9.0

* Bump Go to 1.16.x following controller-runtime 0.9.0

* Upgrade kubebuilder to 2.3.2 for updated etcd and apiserver following local setup

* Fix startup failure due to missing LeaderElectionID

* Fix the issue that any pods become unable to start once actions-runner-controller got failed after the mutating webhook has been registered

* Allow force-updating statefulset

* Fix runner container missing work and certs-client volume mounts and DOCKER_HOST and DOCKER_TLS_VERIFY envvars when dockerdWithinRunner=false

* Fix runnerset-controller not applying statefulset.spec.template.spec changes when there were no changes in runnerset spec

* Enable running acceptance tests against arbitrary kind cluster

* RunnerSet supports non-ephemeral runners only today

* fix: docker-build from root Makefile on intel mac

* fix: arch check fixes for mac and ARM

* ci: aligning test data format and patching checks

* fix: removing namespace in test data

* chore: adding more ignores

* chore: removing leading space in shebang

* Re-add metrics to org hra testdata

* Bump cert-manager to v1.1.1 and fix deploy.sh

Co-authored-by: toast-gear <15716903+toast-gear@users.noreply.github.com>
Co-authored-by: Callum James Tait <callum.tait@photobox.com>

main.go

@@ -127,6 +127,7 @@ func main() {
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
+		LeaderElectionID:   "actions-runner-controller",
 		Port:               9443,
 		SyncPeriod:         &syncPeriod,
 		Namespace:          namespace,
@@ -150,14 +151,14 @@ func main() {
 		os.Exit(1)
 	}
 
-	runnerSetReconciler := &controllers.RunnerReplicaSetReconciler{
+	runnerReplicaSetReconciler := &controllers.RunnerReplicaSetReconciler{
 		Client:       mgr.GetClient(),
 		Log:          log.WithName("runnerreplicaset"),
 		Scheme:       mgr.GetScheme(),
 		GitHubClient: ghClient,
 	}
 
-	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {
+	if err = runnerReplicaSetReconciler.SetupWithManager(mgr); err != nil {
 		log.Error(err, "unable to create controller", "controller", "RunnerReplicaSet")
 		os.Exit(1)
 	}
@@ -174,6 +175,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	runnerSetReconciler := &controllers.RunnerSetReconciler{
+		Client:             mgr.GetClient(),
+		Log:                log.WithName("runnerset"),
+		Scheme:             mgr.GetScheme(),
+		CommonRunnerLabels: commonRunnerLabels,
+		RunnerImage:        runnerImage,
+		DockerImage:        dockerImage,
+		GitHubBaseURL:      ghClient.GithubBaseURL,
+	}
+
+	if err = runnerSetReconciler.SetupWithManager(mgr); err != nil {
+		log.Error(err, "unable to create controller", "controller", "RunnerSet")
+		os.Exit(1)
+	}
 	if gitHubAPICacheDuration == 0 {
 		gitHubAPICacheDuration = syncPeriod - 10*time.Second
 	}
@@ -219,6 +234,16 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	injector := &controllers.PodRunnerTokenInjector{
+		Client:       mgr.GetClient(),
+		GitHubClient: ghClient,
+		Log:          ctrl.Log.WithName("webhook").WithName("PodRunnerTokenInjector"),
+	}
+	if err = injector.SetupWithManager(mgr); err != nil {
+		log.Error(err, "unable to create webhook server", "webhook", "PodRunnerTokenInjector")
+		os.Exit(1)
+	}
+
 	log.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		log.Error(err, "problem running manager")