feat: RunnerDeployment

Adds the initial version of RunnerDeployment that is intended to manage RunnerSets(#1), like Deployment manages ReplicaSets.

This is the initial version and therefore is bare bone. The only update strategy it supports is `Recreate`, which recreates the underlying RunnerSet when the runner template changes. I'd like to add `RollingUpdate` strategy once this is merged.

This depends on #1 so the diff contains that of #1, too. Please see only the latest commit for review.

Also see https://github.com/mumoshu/actions-runner-controller-ci/runs/471329823?check_suite_focus=true to confirm that `make tests` is passing after changes made in this commit.

controllers/runnerdeployment_controller.go

@@ -0,0 +1,253 @@
+/*
+Copyright 2020 The actions-runner-controller authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/go-logr/logr"
+	"hash/fnv"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sort"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+const (
+	LabelKeyRunnerTemplateHash = "runner-template-hash"
+)
+
+// RunnerDeploymentReconciler reconciles a Runner object
+type RunnerDeploymentReconciler struct {
+	client.Client
+	Log      logr.Logger
+	Recorder record.EventRecorder
+	Scheme   *runtime.Scheme
+}
+
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnerdeployments/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets/status,verbs=get;update;patch
+
+func (r *RunnerDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	ctx := context.Background()
+	log := r.Log.WithValues("runnerset", req.NamespacedName)
+
+	var rd v1alpha1.RunnerDeployment
+	if err := r.Get(ctx, req.NamespacedName, &rd); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if !rd.ObjectMeta.DeletionTimestamp.IsZero() {
+		return ctrl.Result{}, nil
+	}
+
+	var allRunnerSets v1alpha1.RunnerSetList
+	if err := r.List(ctx, &allRunnerSets, client.InNamespace(req.Namespace)); err != nil {
+		if !errors.IsNotFound(err) {
+			return ctrl.Result{}, err
+		}
+	}
+
+	var myRunnerSets []*v1alpha1.RunnerSet
+
+	for i := range allRunnerSets.Items {
+		rs := allRunnerSets.Items[i]
+
+		if metav1.IsControlledBy(&rs, &rd) {
+			myRunnerSets = append(myRunnerSets, &rs)
+		}
+	}
+
+	sort.Slice(myRunnerSets, func(i, j int) bool {
+		return myRunnerSets[i].GetCreationTimestamp().After(myRunnerSets[j].GetCreationTimestamp().Time)
+	})
+
+	var newestSet *v1alpha1.RunnerSet
+
+	var oldSets []*v1alpha1.RunnerSet
+
+	if len(myRunnerSets) > 0 {
+		newestSet = myRunnerSets[0]
+	}
+
+	if len(myRunnerSets) > 1 {
+		oldSets = myRunnerSets[1:]
+	}
+
+	desiredRS, err := r.newRunnerSet(rd)
+	if err != nil {
+		log.Error(err, "Could not create runnerset")
+
+		return ctrl.Result{}, err
+	}
+
+	if newestSet == nil {
+		if err := r.Client.Create(ctx, &desiredRS); err != nil {
+			log.Error(err, "Failed to create runnerset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		return ctrl.Result{}, nil
+	}
+
+	newestTemplateHash, ok := getTemplateHash(newestSet)
+	if !ok {
+		log.Info("Failed to get template hash of newest runnerset resource. It must be in an invalid state. Please manually delete the runnerset so that it is recreated")
+
+		return ctrl.Result{}, nil
+	}
+
+	desiredTemplateHash, ok := getTemplateHash(&desiredRS)
+	if !ok {
+		log.Info("Failed to get template hash of desired runnerset resource. It must be in an invalid state. Please manually delete the runnerset so that it is recreated")
+
+		return ctrl.Result{}, nil
+	}
+
+	if newestTemplateHash != desiredTemplateHash {
+		if err := r.Client.Create(ctx, &desiredRS); err != nil {
+			log.Error(err, "Failed to create runnerset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		return ctrl.Result{}, nil
+	}
+
+	// Please add more conditions that we can in-place update the newest runnerset without disruption
+	if newestSet.Spec.Replicas != desiredRS.Spec.Replicas {
+		newestSet.Spec.Replicas = desiredRS.Spec.Replicas
+
+		if err := r.Client.Update(ctx, newestSet); err != nil {
+			log.Error(err, "Failed to update runnerset resource")
+
+			return ctrl.Result{}, err
+		}
+
+		return ctrl.Result{}, nil
+	}
+
+	for i := range oldSets {
+		rs := oldSets[i]
+
+		if err := r.Client.Delete(ctx, rs); err != nil {
+			log.Error(err, "Failed to delete runner resource")
+
+			return ctrl.Result{}, err
+		}
+
+		r.Recorder.Event(&rd, corev1.EventTypeNormal, "RunnerSetDeleted", fmt.Sprintf("Deleted runnerset '%s'", rs.Name))
+		log.Info("Deleted runnerset", "runnerdeployment", rd.ObjectMeta.Name, "runnerset", rs.Name)
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func getTemplateHash(rs *v1alpha1.RunnerSet) (string, bool) {
+	hash, ok := rs.Labels[LabelKeyRunnerTemplateHash]
+
+	return hash, ok
+}
+
+// ComputeHash returns a hash value calculated from pod template and
+// a collisionCount to avoid hash collision. The hash will be safe encoded to
+// avoid bad words.
+//
+// Proudly modified and adopted from k8s.io/kubernetes/pkg/util/hash.DeepHashObject and
+// k8s.io/kubernetes/pkg/controller.ComputeHash.
+func ComputeHash(template interface{}) string {
+	hasher := fnv.New32a()
+
+	hasher.Reset()
+
+	printer := spew.ConfigState{
+		Indent:         " ",
+		SortKeys:       true,
+		DisableMethods: true,
+		SpewKeys:       true,
+	}
+	printer.Fprintf(hasher, "%#v", template)
+
+	return rand.SafeEncodeString(fmt.Sprint(hasher.Sum32()))
+}
+
+// Clones the given map and returns a new map with the given key and value added.
+// Returns the given map, if labelKey is empty.
+//
+// Proudly copied from k8s.io/kubernetes/pkg/util/labels.CloneAndAddLabel
+func CloneAndAddLabel(labels map[string]string, labelKey, labelValue string) map[string]string {
+	if labelKey == "" {
+		// Don't need to add a label.
+		return labels
+	}
+	// Clone.
+	newLabels := map[string]string{}
+	for key, value := range labels {
+		newLabels[key] = value
+	}
+	newLabels[labelKey] = labelValue
+	return newLabels
+}
+
+func (r *RunnerDeploymentReconciler) newRunnerSet(rd v1alpha1.RunnerDeployment) (v1alpha1.RunnerSet, error) {
+	newRSTemplate := *rd.Spec.Template.DeepCopy()
+	templateHash := ComputeHash(&newRSTemplate)
+	// Add template hash label to selector.
+	labels := CloneAndAddLabel(rd.Spec.Template.Labels, LabelKeyRunnerTemplateHash, templateHash)
+
+	newRSTemplate.Labels = labels
+
+	rs := v1alpha1.RunnerSet{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: rd.ObjectMeta.Name,
+			Namespace:    rd.ObjectMeta.Namespace,
+			Labels:       labels,
+		},
+		Spec: v1alpha1.RunnerSetSpec{
+			Replicas: rd.Spec.Replicas,
+			Template: newRSTemplate,
+		},
+	}
+
+	if err := ctrl.SetControllerReference(&rd, &rs, r.Scheme); err != nil {
+		return rs, err
+	}
+
+	return rs, nil
+}
+
+func (r *RunnerDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.Recorder = mgr.GetEventRecorderFor("runnerdeployment-controller")
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1alpha1.RunnerDeployment{}).
+		Owns(&v1alpha1.RunnerSet{}).
+		Complete(r)
+}

controllers/runnerdeployment_controller_test.go

@@ -0,0 +1,175 @@
+package controllers
+
+import (
+	"context"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	actionsv1alpha1 "github.com/summerwind/actions-runner-controller/api/v1alpha1"
+)
+
+// SetupDeploymentTest will set up a testing environment.
+// This includes:
+// * creating a Namespace to be used during the test
+// * starting the 'RunnerDeploymentReconciler'
+// * stopping the 'RunnerDeploymentReconciler" after the test ends
+// Call this function at the start of each of your tests.
+func SetupDeploymentTest(ctx context.Context) *corev1.Namespace {
+	var stopCh chan struct{}
+	ns := &corev1.Namespace{}
+
+	BeforeEach(func() {
+		stopCh = make(chan struct{})
+		*ns = corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)},
+		}
+
+		err := k8sClient.Create(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
+
+		mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
+		Expect(err).NotTo(HaveOccurred(), "failed to create manager")
+
+		controller := &RunnerDeploymentReconciler{
+			Client:   mgr.GetClient(),
+			Scheme:   scheme.Scheme,
+			Log:      logf.Log,
+			Recorder: mgr.GetEventRecorderFor("runnerset-controller"),
+		}
+		err = controller.SetupWithManager(mgr)
+		Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
+
+		go func() {
+			defer GinkgoRecover()
+
+			err := mgr.Start(stopCh)
+			Expect(err).NotTo(HaveOccurred(), "failed to start manager")
+		}()
+	})
+
+	AfterEach(func() {
+		close(stopCh)
+
+		err := k8sClient.Delete(ctx, ns)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
+	})
+
+	return ns
+}
+
+var _ = Context("Inside of a new namespace", func() {
+	ctx := context.TODO()
+	ns := SetupDeploymentTest(ctx)
+
+	Describe("when no existing resources exist", func() {
+
+		It("should create a new RunnerSet resource from the specified template, add a another RunnerSet on template modification, and eventually removes old runnersets", func() {
+			name := "example-runnerdeploy"
+
+			{
+				rs := &actionsv1alpha1.RunnerDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: ns.Name,
+					},
+					Spec: actionsv1alpha1.RunnerDeploymentSpec{
+						Replicas: intPtr(1),
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								Repository: "foo/bar",
+								Image:      "bar",
+								Env: []corev1.EnvVar{
+									{Name: "FOO", Value: "FOOVALUE"},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, rs)
+
+				Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerSet resource")
+
+				runnerSets := actionsv1alpha1.RunnerSetList{Items: []actionsv1alpha1.RunnerSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						if len(runnerSets.Items) == 0 {
+							logf.Log.Info("No runnersets exist yet")
+							return -1
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+			}
+
+			{
+				// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
+				// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
+				//   Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
+				Eventually(func() error {
+					var rd actionsv1alpha1.RunnerDeployment
+
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
+
+					Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerSet resource")
+
+					rd.Spec.Replicas = intPtr(2)
+
+					return k8sClient.Update(ctx, &rd)
+				},
+					time.Second*1, time.Millisecond*500).Should(BeNil())
+
+				runnerSets := actionsv1alpha1.RunnerSetList{Items: []actionsv1alpha1.RunnerSet{}}
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return len(runnerSets.Items)
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
+
+				Eventually(
+					func() int {
+						err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
+						if err != nil {
+							logf.Log.Error(err, "list runner sets")
+						}
+
+						return *runnerSets.Items[0].Spec.Replicas
+					},
+					time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
+			}
+		})
+	})
+})

controllers/runnerset_controller.go

@@ -138,13 +138,15 @@ func (r *RunnerSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 }
 
 func (r *RunnerSetReconciler) newRunner(rs v1alpha1.RunnerSet) (v1alpha1.Runner, error) {
+	objectMeta := rs.Spec.Template.ObjectMeta.DeepCopy()
+
+	objectMeta.GenerateName = rs.ObjectMeta.Name
+	objectMeta.Namespace = rs.ObjectMeta.Namespace
+
 	runner := v1alpha1.Runner{
-		TypeMeta: metav1.TypeMeta{},
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: rs.ObjectMeta.Name,
-			Namespace:    rs.ObjectMeta.Namespace,
-		},
-		Spec: rs.Spec.Template,
+		TypeMeta:   metav1.TypeMeta{},
+		ObjectMeta: *objectMeta,
+		Spec:       rs.Spec.Template.Spec,
 	}
 
 	if err := ctrl.SetControllerReference(&rs, &runner, r.Scheme); err != nil {

controllers/runnerset_controller_test.go

@@ -98,11 +98,13 @@ var _ = Context("Inside of a new namespace", func() {
 					},
 					Spec: actionsv1alpha1.RunnerSetSpec{
 						Replicas: intPtr(1),
-						Template: actionsv1alpha1.RunnerSpec{
-							Repository: "foo/bar",
-							Image:      "bar",
-							Env: []corev1.EnvVar{
-								{Name: "FOO", Value: "FOOVALUE"},
+						Template: actionsv1alpha1.RunnerTemplate{
+							Spec: actionsv1alpha1.RunnerSpec{
+								Repository: "foo/bar",
+								Image:      "bar",
+								Env: []corev1.EnvVar{
+									{Name: "FOO", Value: "FOOVALUE"},
+								},
 							},
 						},
 					},