actions/actions-runner-controller
1
0
Fork 0
mirror of https://github.com/actions/actions-runner-controller.git synced 2025-12-11 12:06:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Manage runner with label (#355)

Browse Source 
* Update RunnerDeploymentSpec to have Selector field

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Update RunnerReplicaSetSpec to have Selector field

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Add CloneSelectorAndAddLabel to add Selector field

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Fix tests

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Use label to find RunnerReplicaSet/Runner

Signed-off-by: binoue <banji-inoue@cybozu.co.jp>

* Update controller-gen versions in CRD

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Update autoscaler to list Pods with labels

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Add debug log

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Modify RunnerDeployment tests

Signed-off-by: binoue <banji-inoue@cybozu.co.jp>

* Modify RunnerReplicaset test

Signed-off-by: binoue <banji-inoue@cybozu.co.jp>

* Modify integration test

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Use RunnerDeployment Template Labels as the default selector for backward compatibility

* Fix labeling

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Update func in Eventually to return (int, error)

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Update RunnerDeployment controller not to use label selector

Signed-off-by: Hiroshi Muraoka <h.muraoka714@gmail.com>

* Fix potential replicaset controller breakage on replicaset created before v0.17.0

* Fix errors on existing runner replica sets

* Ensure RunnerReplicaSet Spec Selector addition does not break controller

* Ensure RunnerDeployment Template.Spec.Labels change does result in template hash change

* Fix comment

Co-authored-by: binoue <banji-inoue@cybozu.co.jp>
Co-authored-by: Yusuke Kuoka <ykuoka@gmail.com>
This commit is contained in:
Hiroshi Muraoka
2021-03-05 10:15:39 +09:00
committed by GitHub
parent f220fefe92
commit 11e58fcc41
14 changed files with 667 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

351
controllers/runnerdeployment_controller_test.go
View File

@@ -2,11 +2,13 @@ package controllers
import (
"context"
"github.com/google/go-cmp/cmp"
"k8s.io/apimachinery/pkg/runtime"
"fmt"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"k8s.io/apimachinery/pkg/runtime"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/scheme"
@@ -36,7 +38,17 @@ func TestNewRunnerReplicaSet(t *testing.T) {
Name: "example",
},
Spec: actionsv1alpha1.RunnerDeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"foo": "bar",
},
},
Template: actionsv1alpha1.RunnerTemplate{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"foo": "bar",
},
},
Spec: actionsv1alpha1.RunnerSpec{
Labels: []string{"project1"},
},
@@ -49,10 +61,63 @@ func TestNewRunnerReplicaSet(t *testing.T) {
t.Fatalf("%v", err)
}
want := []string{"project1", "dev"}
if d := cmp.Diff(want, rs.Spec.Template.Spec.Labels); d != "" {
if val, ok := rs.Labels["foo"]; ok {
if val != "bar" {
t.Errorf("foo label does not have bar but %v", val)
}
} else {
t.Errorf("foo label does not exist")
}
hash1, ok := rs.Labels[LabelKeyRunnerTemplateHash]
if !ok {
t.Errorf("missing runner-template-hash label")
}
runnerLabel := []string{"project1", "dev"}
if d := cmp.Diff(runnerLabel, rs.Spec.Template.Spec.Labels); d != "" {
t.Errorf("%s", d)
}
rd2 := rd.DeepCopy()
rd2.Spec.Template.Spec.Labels = []string{"project2"}
rs2, err := r.newRunnerReplicaSet(*rd2)
if err != nil {
t.Fatalf("%v", err)
}
hash2, ok := rs2.Labels[LabelKeyRunnerTemplateHash]
if !ok {
t.Errorf("missing runner-template-hash label")
}
if hash1 == hash2 {
t.Errorf(
"runner replica sets from runner deployments with varying labels must have different template hash, but got %s and %s",
hash1, hash2,
)
}
rd3 := rd.DeepCopy()
rd3.Spec.Template.Labels["foo"] = "baz"
rs3, err := r.newRunnerReplicaSet(*rd3)
if err != nil {
t.Fatalf("%v", err)
}
hash3, ok := rs3.Labels[LabelKeyRunnerTemplateHash]
if !ok {
t.Errorf("missing runner-template-hash label")
}
if hash1 == hash3 {
t.Errorf(
"runner replica sets from runner deployments with varying meta labels must have different template hash, but got %s and %s",
hash1, hash3,
)
}
}
// SetupDeploymentTest will set up a testing environment.
@@ -112,7 +177,113 @@ var _ = Context("Inside of a new namespace", func() {
Describe("when no existing resources exist", func() {
It("should create a new RunnerReplicaSet resource from the specified template, add a another RunnerReplicaSet on template modification, and eventually removes old runnerreplicasets", func() {
name := "example-runnerdeploy"
name := "example-runnerdeploy-1"
{
rs := &actionsv1alpha1.RunnerDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns.Name,
},
Spec: actionsv1alpha1.RunnerDeploymentSpec{
Replicas: intPtr(1),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"foo": "bar",
},
},
Template: actionsv1alpha1.RunnerTemplate{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"foo": "bar",
},
},
Spec: actionsv1alpha1.RunnerSpec{
Repository: "foo/bar",
Image: "bar",
Env: []corev1.EnvVar{
{Name: "FOO", Value: "FOOVALUE"},
},
},
},
},
}
err := k8sClient.Create(ctx, rs)
Expect(err).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
Eventually(
func() (int, error) {
selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
if err != nil {
return 0, err
}
err = k8sClient.List(
ctx,
&runnerSets,
client.InNamespace(ns.Name),
client.MatchingLabelsSelector{Selector: selector},
)
if err != nil {
return 0, err
}
if len(runnerSets.Items) != 1 {
return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
}
return *runnerSets.Items[0].Spec.Replicas, nil
},
time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
}
{
// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
// Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
var rd actionsv1alpha1.RunnerDeployment
Eventually(func() error {
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
if err != nil {
return fmt.Errorf("failed to get test RunnerReplicaSet resource: %v\n", err)
}
rd.Spec.Replicas = intPtr(2)
return k8sClient.Update(ctx, &rd)
},
time.Second*1, time.Millisecond*500).Should(BeNil())
runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
Eventually(
func() (int, error) {
selector, err := metav1.LabelSelectorAsSelector(rd.Spec.Selector)
if err != nil {
return 0, err
}
err = k8sClient.List(
ctx,
&runnerSets,
client.InNamespace(ns.Name),
client.MatchingLabelsSelector{Selector: selector},
)
if err != nil {
return 0, err
}
if len(runnerSets.Items) != 1 {
return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
}
return *runnerSets.Items[0].Spec.Replicas, nil
},
time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
}
})
It("should create a new RunnerReplicaSet resource from the specified template without labels and selector, add a another RunnerReplicaSet on template modification, and eventually removes old runnerreplicasets", func() {
name := "example-runnerdeploy-2"
{
rs := &actionsv1alpha1.RunnerDeployment{
@@ -141,29 +312,25 @@ var _ = Context("Inside of a new namespace", func() {
runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
Eventually(
func() int {
err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
func() (int, error) {
selector, err := metav1.LabelSelectorAsSelector(rs.Spec.Selector)
if err != nil {
logf.Log.Error(err, "list runner sets")
return 0, err
}
return len(runnerSets.Items)
},
time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
Eventually(
func() int {
err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
err = k8sClient.List(
ctx,
&runnerSets,
client.InNamespace(ns.Name),
client.MatchingLabelsSelector{Selector: selector},
)
if err != nil {
logf.Log.Error(err, "list runner sets")
return 0, err
}
if len(runnerSets.Items) != 1 {
return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
}
if len(runnerSets.Items) == 0 {
logf.Log.Info("No runnerreplicasets exist yet")
return -1
}
return *runnerSets.Items[0].Spec.Replicas
return *runnerSets.Items[0].Spec.Replicas, nil
},
time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
}
@@ -172,13 +339,12 @@ var _ = Context("Inside of a new namespace", func() {
// We wrap the update in the Eventually block to avoid the below error that occurs due to concurrent modification
// made by the controller to update .Status.AvailableReplicas and .Status.ReadyReplicas
// Operation cannot be fulfilled on runnersets.actions.summerwind.dev "example-runnerset": the object has been modified; please apply your changes to the latest version and try again
var rd actionsv1alpha1.RunnerDeployment
Eventually(func() error {
var rd actionsv1alpha1.RunnerDeployment
err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: name}, &rd)
Expect(err).NotTo(HaveOccurred(), "failed to get test RunnerReplicaSet resource")
if err != nil {
return fmt.Errorf("failed to get test RunnerReplicaSet resource: %v\n", err)
}
rd.Spec.Replicas = intPtr(2)
return k8sClient.Update(ctx, &rd)
@@ -188,27 +354,126 @@ var _ = Context("Inside of a new namespace", func() {
runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
Eventually(
func() int {
err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
func() (int, error) {
selector, err := metav1.LabelSelectorAsSelector(rd.Spec.Selector)
if err != nil {
logf.Log.Error(err, "list runner sets")
return 0, err
}
err = k8sClient.List(
ctx,
&runnerSets,
client.InNamespace(ns.Name),
client.MatchingLabelsSelector{Selector: selector},
)
if err != nil {
return 0, err
}
if len(runnerSets.Items) != 1 {
return 0, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
}
return len(runnerSets.Items)
},
time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(1))
Eventually(
func() int {
err := k8sClient.List(ctx, &runnerSets, client.InNamespace(ns.Name))
if err != nil {
logf.Log.Error(err, "list runner sets")
}
return *runnerSets.Items[0].Spec.Replicas
return *runnerSets.Items[0].Spec.Replicas, nil
},
time.Second*5, time.Millisecond*500).Should(BeEquivalentTo(2))
}
})
It("should adopt RunnerReplicaSet created before 0.18.0 to have Spec.Selector", func() {
name := "example-runnerdeploy-2"
{
rd := &actionsv1alpha1.RunnerDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns.Name,
},
Spec: actionsv1alpha1.RunnerDeploymentSpec{
Replicas: intPtr(1),
Template: actionsv1alpha1.RunnerTemplate{
Spec: actionsv1alpha1.RunnerSpec{
Repository: "foo/bar",
Image: "bar",
Env: []corev1.EnvVar{
{Name: "FOO", Value: "FOOVALUE"},
},
},
},
},
}
createRDErr := k8sClient.Create(ctx, rd)
Expect(createRDErr).NotTo(HaveOccurred(), "failed to create test RunnerReplicaSet resource")
Eventually(
func() (int, error) {
runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
err := k8sClient.List(
ctx,
&runnerSets,
client.InNamespace(ns.Name),
)
if err != nil {
return 0, err
}
return len(runnerSets.Items), nil
},
time.Second*1, time.Millisecond*500).Should(BeEquivalentTo(1))
var rs17 *actionsv1alpha1.RunnerReplicaSet
Consistently(
func() (*metav1.LabelSelector, error) {
runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
err := k8sClient.List(
ctx,
&runnerSets,
client.InNamespace(ns.Name),
)
if err != nil {
return nil, err
}
if len(runnerSets.Items) != 1 {
return nil, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
}
rs17 = &runnerSets.Items[0]
return runnerSets.Items[0].Spec.Selector, nil
},
time.Second*1, time.Millisecond*500).Should(Not(BeNil()))
// We simulate the old, pre 0.18.0 RunnerReplicaSet by updating it.
// I've tried to use controllerutil.Set{Owner,Controller}Reference and k8sClient.Create(rs17)
// but it didn't work due to missing RD UID, where UID is generated on K8s API server on k8sCLient.Create(rd)
rs17.Spec.Selector = nil
updateRSErr := k8sClient.Update(ctx, rs17)
Expect(updateRSErr).NotTo(HaveOccurred())
Eventually(
func() (*metav1.LabelSelector, error) {
runnerSets := actionsv1alpha1.RunnerReplicaSetList{Items: []actionsv1alpha1.RunnerReplicaSet{}}
err := k8sClient.List(
ctx,
&runnerSets,
client.InNamespace(ns.Name),
)
if err != nil {
return nil, err
}
if len(runnerSets.Items) != 1 {
return nil, fmt.Errorf("runnerreplicasets is not 1 but %d", len(runnerSets.Items))
}
return runnerSets.Items[0].Spec.Selector, nil
},
time.Second*1, time.Millisecond*500).Should(Not(BeNil()))
}
})
})
})