feat: RunnerSet backed by StatefulSet (#629)

* feat: RunnerSet backed by StatefulSet

Unlike a runner deployment, a runner set can manage a set of stateful runners by combining a statefulset and an admission webhook that mutates statefulset-managed pods with required envvars and registration tokens.

Resolves #613
Ref #612

* Upgrade controller-runtime to 0.9.0

* Bump Go to 1.16.x following controller-runtime 0.9.0

* Upgrade kubebuilder to 2.3.2 for updated etcd and apiserver following local setup

* Fix startup failure due to missing LeaderElectionID

* Fix the issue that any pods become unable to start once actions-runner-controller got failed after the mutating webhook has been registered

* Allow force-updating statefulset

* Fix runner container missing work and certs-client volume mounts and DOCKER_HOST and DOCKER_TLS_VERIFY envvars when dockerdWithinRunner=false

* Fix runnerset-controller not applying statefulset.spec.template.spec changes when there were no changes in runnerset spec

* Enable running acceptance tests against arbitrary kind cluster

* RunnerSet supports non-ephemeral runners only today

* fix: docker-build from root Makefile on intel mac

* fix: arch check fixes for mac and ARM

* ci: aligning test data format and patching checks

* fix: removing namespace in test data

* chore: adding more ignores

* chore: removing leading space in shebang

* Re-add metrics to org hra testdata

* Bump cert-manager to v1.1.1 and fix deploy.sh

Co-authored-by: toast-gear <15716903+toast-gear@users.noreply.github.com>
Co-authored-by: Callum James Tait <callum.tait@photobox.com>

controllers/pod_runner_token_injector.go

@@ -0,0 +1,128 @@
+package controllers
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/go-logr/logr"
+	"github.com/summerwind/actions-runner-controller/github"
+	"gomodules.xyz/jsonpatch/v2"
+	admissionv1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// +kubebuilder:webhook:path=/mutate-runner-set-pod,mutating=true,failurePolicy=ignore,groups="",resources=pods,verbs=create,versions=v1,name=mutate-runner-pod.webhook.actions.summerwind.dev,sideEffects=None,admissionReviewVersions=v1beta1
+
+type PodRunnerTokenInjector struct {
+	client.Client
+
+	Name         string
+	Log          logr.Logger
+	Recorder     record.EventRecorder
+	GitHubClient *github.Client
+	decoder      *admission.Decoder
+}
+
+func (t *PodRunnerTokenInjector) Handle(ctx context.Context, req admission.Request) admission.Response {
+	var pod corev1.Pod
+	err := t.decoder.Decode(req, &pod)
+	if err != nil {
+		t.Log.Error(err, "Failed to decode request object")
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	if pod.Annotations == nil {
+		pod.Annotations = map[string]string{}
+	}
+
+	var runnerContainer *corev1.Container
+
+	for i := range pod.Spec.Containers {
+		c := pod.Spec.Containers[i]
+
+		if c.Name == "runner" {
+			runnerContainer = &c
+		}
+	}
+
+	if runnerContainer == nil {
+		return newEmptyResponse()
+	}
+
+	enterprise, okEnterprise := getEnv(runnerContainer, "RUNNER_ENTERPRISE")
+	repo, okRepo := getEnv(runnerContainer, "RUNNER_REPO")
+	org, okOrg := getEnv(runnerContainer, "RUNNER_ORG")
+	if !okRepo || !okOrg || !okEnterprise {
+		return newEmptyResponse()
+	}
+
+	rt, err := t.GitHubClient.GetRegistrationToken(context.Background(), enterprise, org, repo, pod.Name)
+	if err != nil {
+		t.Log.Error(err, "Failed to get new registration token")
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	ts := rt.GetExpiresAt().Format(time.RFC3339)
+
+	updated := mutatePod(&pod, *rt.Token)
+
+	updated.Annotations["actions-runner-controller/token-expires-at"] = ts
+
+	if pod.Spec.RestartPolicy != corev1.RestartPolicyOnFailure {
+		updated.Spec.RestartPolicy = corev1.RestartPolicyOnFailure
+	}
+
+	buf, err := json.Marshal(updated)
+	if err != nil {
+		t.Log.Error(err, "Failed to encode new object")
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+
+	res := admission.PatchResponseFromRaw(req.Object.Raw, buf)
+	return res
+}
+
+func getEnv(container *corev1.Container, key string) (string, bool) {
+	for _, env := range container.Env {
+		if env.Name == key {
+			return env.Value, true
+		}
+	}
+
+	return "", false
+}
+
+func (t *PodRunnerTokenInjector) InjectDecoder(d *admission.Decoder) error {
+	t.decoder = d
+	return nil
+}
+
+func newEmptyResponse() admission.Response {
+	pt := admissionv1.PatchTypeJSONPatch
+	return admission.Response{
+		Patches: []jsonpatch.Operation{},
+		AdmissionResponse: admissionv1.AdmissionResponse{
+			Allowed:   true,
+			PatchType: &pt,
+		},
+	}
+}
+
+func (r *PodRunnerTokenInjector) SetupWithManager(mgr ctrl.Manager) error {
+	name := "pod-runner-token-injector"
+	if r.Name != "" {
+		name = r.Name
+	}
+
+	r.Recorder = mgr.GetEventRecorderFor(name)
+
+	mgr.GetWebhookServer().Register("/mutate-runner-set-pod", &admission.Webhook{Handler: r})
+
+	return nil
+}