From 065655be7e0c4523073365dda5e3e7d8d5dc325d Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Wed, 22 Mar 2023 11:54:15 +0000 Subject: [PATCH] Experimenting with no ginkgo/gomega --- .../autoscalinglistener_controller_test.go | 1436 +++++----- .../autoscalingrunnerset_controller_test.go | 2204 ++++++++-------- .../ephemeralrunner_controller_test.go | 1884 ++++++------- .../ephemeralrunnerset_controller_test.go | 2320 ++++++++--------- .../actions.github.com/helpers_test.go | 9 +- controllers/actions.github.com/suite_test.go | 140 +- go.mod | 1 + go.sum | 2 + 8 files changed, 4067 insertions(+), 3929 deletions(-) diff --git a/controllers/actions.github.com/autoscalinglistener_controller_test.go b/controllers/actions.github.com/autoscalinglistener_controller_test.go index d4932797..e7640fb3 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller_test.go +++ b/controllers/actions.github.com/autoscalinglistener_controller_test.go @@ -1,720 +1,720 @@ package actionsgithubcom -import ( - "context" - "fmt" - "os" - "path/filepath" - "time" - - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" -) - -const ( - autoscalingListenerTestTimeout = time.Second * 5 - autoscalingListenerTestInterval = time.Millisecond * 250 - autoscalingListenerTestGitHubToken = "gh_token" -) - -var _ = Describe("Test AutoScalingListener controller", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet - var configSecret *corev1.Secret - var autoscalingListener *actionsv1alpha1.AutoscalingListener - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - controller := &AutoscalingListenerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - min := 1 - max := 10 - autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - MaxRunners: &max, - MinRunners: &min, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - autoscalingListener = &actionsv1alpha1.AutoscalingListener{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asl", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 1, - AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, - AutoscalingRunnerSetName: autoscalingRunnerSet.Name, - EphemeralRunnerSetName: "test-ers", - MaxRunners: 10, - MinRunners: 1, - Image: "ghcr.io/owner/repo", - }, - } - - err = k8sClient.Create(ctx, autoscalingListener) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener") - - startManagers(GinkgoT(), mgr) - }) - - Context("When creating a new AutoScalingListener", func() { - It("It should create/add all required resources for a new AutoScalingListener (finalizer, secret, service account, role, rolebinding, pod)", func() { - // Check if finalizer is added - created := new(actionsv1alpha1.AutoscalingListener) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, created) - if err != nil { - return "", err - } - if len(created.Finalizers) == 0 { - return "", nil - } - return created.Finalizers[0], nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListenerFinalizerName), "AutoScalingListener should have a finalizer") - - // Check if secret is created - mirrorSecret := new(corev1.Secret) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerSecretMirrorName(autoscalingListener), Namespace: autoscalingListener.Namespace}, mirrorSecret) - if err != nil { - return "", err - } - return string(mirrorSecret.Data["github_token"]), nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListenerTestGitHubToken), "Mirror secret should be created") - - // Check if service account is created - serviceAccount := new(corev1.ServiceAccount) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerServiceAccountName(autoscalingListener), Namespace: autoscalingListener.Namespace}, serviceAccount) - if err != nil { - return "", err - } - return serviceAccount.Name, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(scaleSetListenerServiceAccountName(autoscalingListener)), "Service account should be created") - - // Check if role is created - role := new(rbacv1.Role) - Eventually( - func() ([]rbacv1.PolicyRule, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerRoleName(autoscalingListener), Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, role) - if err != nil { - return nil, err - } - - return role.Rules, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName})), "Role should be created") - - // Check if rolebinding is created - roleBinding := new(rbacv1.RoleBinding) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerRoleName(autoscalingListener), Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, roleBinding) - if err != nil { - return "", err - } - - return roleBinding.RoleRef.Name, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(scaleSetListenerRoleName(autoscalingListener)), "Rolebinding should be created") - - // Check if pod is created - pod := new(corev1.Pod) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) - if err != nil { - return "", err - } - - return pod.Name, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") - }) - }) - - Context("When deleting a new AutoScalingListener", func() { - It("It should cleanup all resources for a deleting AutoScalingListener before removing it", func() { - // Waiting for the pod to be created - pod := new(corev1.Pod) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) - if err != nil { - return "", err - } - - return pod.Name, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") - - // Delete the AutoScalingListener - err := k8sClient.Delete(ctx, autoscalingListener) - Expect(err).NotTo(HaveOccurred(), "failed to delete test AutoScalingListener") - - // Cleanup the listener pod - Eventually( - func() error { - podList := new(corev1.PodList) - err := k8sClient.List(ctx, podList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{autoscalingRunnerSetOwnerKey: autoscalingListener.Name}) - if err != nil { - return err - } - - if len(podList.Items) > 0 { - return fmt.Errorf("pod still exists") - } - - return nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete pod") - - // Cleanup the listener service account - Eventually( - func() error { - serviceAccountList := new(corev1.ServiceAccountList) - err := k8sClient.List(ctx, serviceAccountList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{autoscalingRunnerSetOwnerKey: autoscalingListener.Name}) - if err != nil { - return err - } - - if len(serviceAccountList.Items) > 0 { - return fmt.Errorf("service account still exists") - } - - return nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete service account") - - // The AutoScalingListener should be deleted - Eventually( - func() error { - listenerList := new(actionsv1alpha1.AutoscalingListenerList) - err := k8sClient.List(ctx, listenerList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{".metadata.name": autoscalingListener.Name}) - if err != nil { - return err - } - - if len(listenerList.Items) > 0 { - return fmt.Errorf("AutoScalingListener still exists") - } - return nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete AutoScalingListener") - }) - }) - - Context("React to changes in the AutoScalingListener", func() { - It("It should update role to match EphemeralRunnerSet", func() { - // Waiting for the pod is created - pod := new(corev1.Pod) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) - if err != nil { - return "", err - } - - return pod.Name, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") - - // Update the AutoScalingListener - updated := autoscalingListener.DeepCopy() - updated.Spec.EphemeralRunnerSetName = "test-ers-updated" - err := k8sClient.Patch(ctx, updated, client.MergeFrom(autoscalingListener)) - Expect(err).NotTo(HaveOccurred(), "failed to update test AutoScalingListener") - - // Check if role is updated with right rules - role := new(rbacv1.Role) - Eventually( - func() ([]rbacv1.PolicyRule, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerRoleName(autoscalingListener), Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, role) - if err != nil { - return nil, err - } - - return role.Rules, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(rulesForListenerRole([]string{updated.Spec.EphemeralRunnerSetName})), "Role should be updated") - }) - - It("It should update mirror secrets to match secret used by AutoScalingRunnerSet", func() { - // Waiting for the pod is created - pod := new(corev1.Pod) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) - if err != nil { - return "", err - } - - return pod.Name, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") - - // Update the secret - updatedSecret := configSecret.DeepCopy() - updatedSecret.Data["github_token"] = []byte(autoscalingListenerTestGitHubToken + "_updated") - err := k8sClient.Update(ctx, updatedSecret) - Expect(err).NotTo(HaveOccurred(), "failed to update test secret") - - updatedPod := pod.DeepCopy() - updatedPod.Status.Phase = corev1.PodFailed - err = k8sClient.Status().Update(ctx, updatedPod) - Expect(err).NotTo(HaveOccurred(), "failed to update test pod to failed") - - // Check if mirror secret is updated with right data - mirrorSecret := new(corev1.Secret) - Eventually( - func() (map[string][]byte, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerSecretMirrorName(autoscalingListener), Namespace: autoscalingListener.Namespace}, mirrorSecret) - if err != nil { - return nil, err - } - - return mirrorSecret.Data, nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(BeEquivalentTo(updatedSecret.Data), "Mirror secret should be updated") - - // Check if we re-created a new pod - Eventually( - func() error { - latestPod := new(corev1.Pod) - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, latestPod) - if err != nil { - return err - } - if latestPod.UID == pod.UID { - return fmt.Errorf("Pod should be recreated") - } - - return nil - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(Succeed(), "Pod should be recreated") - }) - }) -}) - -var _ = Describe("Test AutoScalingListener controller with proxy", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet - var configSecret *corev1.Secret - var autoscalingListener *actionsv1alpha1.AutoscalingListener - - createRunnerSetAndListener := func(proxy *actionsv1alpha1.ProxyConfig) { - min := 1 - max := 10 - autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - MaxRunners: &max, - MinRunners: &min, - Proxy: proxy, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err := k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - autoscalingListener = &actionsv1alpha1.AutoscalingListener{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asl", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 1, - AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, - AutoscalingRunnerSetName: autoscalingRunnerSet.Name, - EphemeralRunnerSetName: "test-ers", - MaxRunners: 10, - MinRunners: 1, - Image: "ghcr.io/owner/repo", - Proxy: proxy, - }, - } - - err = k8sClient.Create(ctx, autoscalingListener) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener") - } - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - controller := &AutoscalingListenerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("should create a secret in the listener namespace containing proxy details, use it to populate env vars on the pod and should delete it as part of cleanup", func() { - proxyCredentials := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "proxy-credentials", - Namespace: autoscalingNS.Name, - }, - Data: map[string][]byte{ - "username": []byte("test"), - "password": []byte("password"), - }, - } - - err := k8sClient.Create(ctx, proxyCredentials) - Expect(err).NotTo(HaveOccurred(), "failed to create proxy credentials secret") - - proxy := &actionsv1alpha1.ProxyConfig{ - HTTP: &actionsv1alpha1.ProxyServerConfig{ - Url: "http://localhost:8080", - CredentialSecretRef: "proxy-credentials", - }, - HTTPS: &actionsv1alpha1.ProxyServerConfig{ - Url: "https://localhost:8443", - CredentialSecretRef: "proxy-credentials", - }, - NoProxy: []string{ - "example.com", - "example.org", - }, - } - - createRunnerSetAndListener(proxy) - - var proxySecret corev1.Secret - Eventually( - func(g Gomega) { - err := k8sClient.Get( - ctx, - types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name}, - &proxySecret, - ) - g.Expect(err).NotTo(HaveOccurred(), "failed to get secret") - expected, err := autoscalingListener.Spec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) { - var secret corev1.Secret - err := k8sClient.Get(ctx, types.NamespacedName{Name: s, Namespace: autoscalingNS.Name}, &secret) - if err != nil { - return nil, err - } - return &secret, nil - }) - g.Expect(err).NotTo(HaveOccurred(), "failed to convert proxy config to secret data") - g.Expect(proxySecret.Data).To(Equal(expected)) - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(Succeed(), "failed to create secret with proxy details") - - // wait for listener pod to be created - Eventually( - func(g Gomega) { - pod := new(corev1.Pod) - err := k8sClient.Get( - ctx, - client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, - pod, - ) - g.Expect(err).NotTo(HaveOccurred(), "failed to get pod") - - g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ - Name: "http_proxy", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, - Key: "http_proxy", - }, - }, - }), "http_proxy environment variable not found") - - g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ - Name: "https_proxy", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, - Key: "https_proxy", - }, - }, - }), "https_proxy environment variable not found") - - g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ - Name: "no_proxy", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, - Key: "no_proxy", - }, - }, - }), "no_proxy environment variable not found") - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(Succeed(), "failed to create listener pod with proxy details") - - // Delete the AutoScalingListener - err = k8sClient.Delete(ctx, autoscalingListener) - Expect(err).NotTo(HaveOccurred(), "failed to delete test AutoScalingListener") - - Eventually( - func(g Gomega) { - var proxySecret corev1.Secret - err := k8sClient.Get( - ctx, - types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name}, - &proxySecret, - ) - g.Expect(kerrors.IsNotFound(err)).To(BeTrue()) - }, - autoscalingListenerTestTimeout, - autoscalingListenerTestInterval).Should(Succeed(), "failed to delete secret with proxy details") - }) -}) - -var _ = Describe("Test GitHub Server TLS configuration", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet - var configSecret *corev1.Secret - var autoscalingListener *actionsv1alpha1.AutoscalingListener - var rootCAConfigMap *corev1.ConfigMap - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - cert, err := os.ReadFile(filepath.Join( - "../../", - "github", - "actions", - "testdata", - "rootCA.crt", - )) - Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") - rootCAConfigMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "root-ca-configmap", - Namespace: autoscalingNS.Name, - }, - Data: map[string]string{ - "rootCA.crt": string(cert), - }, - } - err = k8sClient.Create(ctx, rootCAConfigMap) - Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") - - controller := &AutoscalingListenerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - } - err = controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - min := 1 - max := 10 - autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{ - CertificateFrom: &actionsv1alpha1.TLSCertificateSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAConfigMap.Name, - }, - Key: "rootCA.crt", - }, - }, - }, - MaxRunners: &max, - MinRunners: &min, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - autoscalingListener = &actionsv1alpha1.AutoscalingListener{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asl", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.AutoscalingListenerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{ - CertificateFrom: &actionsv1alpha1.TLSCertificateSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAConfigMap.Name, - }, - Key: "rootCA.crt", - }, - }, - }, - RunnerScaleSetId: 1, - AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, - AutoscalingRunnerSetName: autoscalingRunnerSet.Name, - EphemeralRunnerSetName: "test-ers", - MaxRunners: 10, - MinRunners: 1, - Image: "ghcr.io/owner/repo", - }, - } - - err = k8sClient.Create(ctx, autoscalingListener) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener") - - startManagers(GinkgoT(), mgr) - }) - - Context("When creating a new AutoScalingListener", func() { - It("It should set the certificates as an environment variable on the pod", func() { - pod := new(corev1.Pod) - Eventually( - func(g Gomega) { - err := k8sClient.Get( - ctx, - client.ObjectKey{ - Name: autoscalingListener.Name, - Namespace: autoscalingListener.Namespace, - }, - pod, - ) - - g.Expect(err).NotTo(HaveOccurred(), "failed to get pod") - g.Expect(pod.Spec.Containers).NotTo(BeEmpty(), "pod should have containers") - g.Expect(pod.Spec.Containers[0].Env).NotTo(BeEmpty(), "pod should have env variables") - - var env *corev1.EnvVar - for _, e := range pod.Spec.Containers[0].Env { - if e.Name == "GITHUB_SERVER_ROOT_CA" { - env = &e - break - } - } - g.Expect(env).NotTo(BeNil(), "pod should have an env variable named GITHUB_SERVER_ROOT_CA_PATH") - - cert, err := os.ReadFile(filepath.Join( - "../../", - "github", - "actions", - "testdata", - "rootCA.crt", - )) - g.Expect(err).NotTo(HaveOccurred(), "failed to read rootCA.crt") - - g.Expect(env.Value).To( - BeEquivalentTo(string(cert)), - "GITHUB_SERVER_ROOT_CA should be the rootCA.crt", - ) - }). - WithTimeout(autoscalingRunnerSetTestTimeout). - WithPolling(autoscalingListenerTestInterval). - Should(Succeed(), "failed to create pod with volume and env variable") - }) - }) -}) +// import ( +// "context" +// "fmt" +// "os" +// "path/filepath" +// "time" +// +// corev1 "k8s.io/api/core/v1" +// rbacv1 "k8s.io/api/rbac/v1" +// ctrl "sigs.k8s.io/controller-runtime" +// "sigs.k8s.io/controller-runtime/pkg/client" +// logf "sigs.k8s.io/controller-runtime/pkg/log" +// +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// kerrors "k8s.io/apimachinery/pkg/api/errors" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// "k8s.io/apimachinery/pkg/types" +// +// actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" +// ) +// +// const ( +// autoscalingListenerTestTimeout = time.Second * 5 +// autoscalingListenerTestInterval = time.Millisecond * 250 +// autoscalingListenerTestGitHubToken = "gh_token" +// ) +// +// var _ = Describe("Test AutoScalingListener controller", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet +// var configSecret *corev1.Secret +// var autoscalingListener *actionsv1alpha1.AutoscalingListener +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller := &AutoscalingListenerReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// min := 1 +// max := 10 +// autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// MaxRunners: &max, +// MinRunners: &min, +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// autoscalingListener = &actionsv1alpha1.AutoscalingListener{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asl", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.AutoscalingListenerSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// RunnerScaleSetId: 1, +// AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, +// AutoscalingRunnerSetName: autoscalingRunnerSet.Name, +// EphemeralRunnerSetName: "test-ers", +// MaxRunners: 10, +// MinRunners: 1, +// Image: "ghcr.io/owner/repo", +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingListener) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// Context("When creating a new AutoScalingListener", func() { +// It("It should create/add all required resources for a new AutoScalingListener (finalizer, secret, service account, role, rolebinding, pod)", func() { +// // Check if finalizer is added +// created := new(actionsv1alpha1.AutoscalingListener) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, created) +// if err != nil { +// return "", err +// } +// if len(created.Finalizers) == 0 { +// return "", nil +// } +// return created.Finalizers[0], nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListenerFinalizerName), "AutoScalingListener should have a finalizer") +// +// // Check if secret is created +// mirrorSecret := new(corev1.Secret) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerSecretMirrorName(autoscalingListener), Namespace: autoscalingListener.Namespace}, mirrorSecret) +// if err != nil { +// return "", err +// } +// return string(mirrorSecret.Data["github_token"]), nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListenerTestGitHubToken), "Mirror secret should be created") +// +// // Check if service account is created +// serviceAccount := new(corev1.ServiceAccount) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerServiceAccountName(autoscalingListener), Namespace: autoscalingListener.Namespace}, serviceAccount) +// if err != nil { +// return "", err +// } +// return serviceAccount.Name, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(scaleSetListenerServiceAccountName(autoscalingListener)), "Service account should be created") +// +// // Check if role is created +// role := new(rbacv1.Role) +// Eventually( +// func() ([]rbacv1.PolicyRule, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerRoleName(autoscalingListener), Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, role) +// if err != nil { +// return nil, err +// } +// +// return role.Rules, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(rulesForListenerRole([]string{autoscalingListener.Spec.EphemeralRunnerSetName})), "Role should be created") +// +// // Check if rolebinding is created +// roleBinding := new(rbacv1.RoleBinding) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerRoleName(autoscalingListener), Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, roleBinding) +// if err != nil { +// return "", err +// } +// +// return roleBinding.RoleRef.Name, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(scaleSetListenerRoleName(autoscalingListener)), "Rolebinding should be created") +// +// // Check if pod is created +// pod := new(corev1.Pod) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) +// if err != nil { +// return "", err +// } +// +// return pod.Name, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") +// }) +// }) +// +// Context("When deleting a new AutoScalingListener", func() { +// It("It should cleanup all resources for a deleting AutoScalingListener before removing it", func() { +// // Waiting for the pod to be created +// pod := new(corev1.Pod) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) +// if err != nil { +// return "", err +// } +// +// return pod.Name, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") +// +// // Delete the AutoScalingListener +// err := k8sClient.Delete(ctx, autoscalingListener) +// Expect(err).NotTo(HaveOccurred(), "failed to delete test AutoScalingListener") +// +// // Cleanup the listener pod +// Eventually( +// func() error { +// podList := new(corev1.PodList) +// err := k8sClient.List(ctx, podList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{autoscalingRunnerSetOwnerKey: autoscalingListener.Name}) +// if err != nil { +// return err +// } +// +// if len(podList.Items) > 0 { +// return fmt.Errorf("pod still exists") +// } +// +// return nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete pod") +// +// // Cleanup the listener service account +// Eventually( +// func() error { +// serviceAccountList := new(corev1.ServiceAccountList) +// err := k8sClient.List(ctx, serviceAccountList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{autoscalingRunnerSetOwnerKey: autoscalingListener.Name}) +// if err != nil { +// return err +// } +// +// if len(serviceAccountList.Items) > 0 { +// return fmt.Errorf("service account still exists") +// } +// +// return nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete service account") +// +// // The AutoScalingListener should be deleted +// Eventually( +// func() error { +// listenerList := new(actionsv1alpha1.AutoscalingListenerList) +// err := k8sClient.List(ctx, listenerList, client.InNamespace(autoscalingListener.Namespace), client.MatchingFields{".metadata.name": autoscalingListener.Name}) +// if err != nil { +// return err +// } +// +// if len(listenerList.Items) > 0 { +// return fmt.Errorf("AutoScalingListener still exists") +// } +// return nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).ShouldNot(Succeed(), "failed to delete AutoScalingListener") +// }) +// }) +// +// Context("React to changes in the AutoScalingListener", func() { +// It("It should update role to match EphemeralRunnerSet", func() { +// // Waiting for the pod is created +// pod := new(corev1.Pod) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) +// if err != nil { +// return "", err +// } +// +// return pod.Name, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") +// +// // Update the AutoScalingListener +// updated := autoscalingListener.DeepCopy() +// updated.Spec.EphemeralRunnerSetName = "test-ers-updated" +// err := k8sClient.Patch(ctx, updated, client.MergeFrom(autoscalingListener)) +// Expect(err).NotTo(HaveOccurred(), "failed to update test AutoScalingListener") +// +// // Check if role is updated with right rules +// role := new(rbacv1.Role) +// Eventually( +// func() ([]rbacv1.PolicyRule, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerRoleName(autoscalingListener), Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, role) +// if err != nil { +// return nil, err +// } +// +// return role.Rules, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(rulesForListenerRole([]string{updated.Spec.EphemeralRunnerSetName})), "Role should be updated") +// }) +// +// It("It should update mirror secrets to match secret used by AutoScalingRunnerSet", func() { +// // Waiting for the pod is created +// pod := new(corev1.Pod) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, pod) +// if err != nil { +// return "", err +// } +// +// return pod.Name, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(autoscalingListener.Name), "Pod should be created") +// +// // Update the secret +// updatedSecret := configSecret.DeepCopy() +// updatedSecret.Data["github_token"] = []byte(autoscalingListenerTestGitHubToken + "_updated") +// err := k8sClient.Update(ctx, updatedSecret) +// Expect(err).NotTo(HaveOccurred(), "failed to update test secret") +// +// updatedPod := pod.DeepCopy() +// updatedPod.Status.Phase = corev1.PodFailed +// err = k8sClient.Status().Update(ctx, updatedPod) +// Expect(err).NotTo(HaveOccurred(), "failed to update test pod to failed") +// +// // Check if mirror secret is updated with right data +// mirrorSecret := new(corev1.Secret) +// Eventually( +// func() (map[string][]byte, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerSecretMirrorName(autoscalingListener), Namespace: autoscalingListener.Namespace}, mirrorSecret) +// if err != nil { +// return nil, err +// } +// +// return mirrorSecret.Data, nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(BeEquivalentTo(updatedSecret.Data), "Mirror secret should be updated") +// +// // Check if we re-created a new pod +// Eventually( +// func() error { +// latestPod := new(corev1.Pod) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, latestPod) +// if err != nil { +// return err +// } +// if latestPod.UID == pod.UID { +// return fmt.Errorf("Pod should be recreated") +// } +// +// return nil +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(Succeed(), "Pod should be recreated") +// }) +// }) +// }) +// +// var _ = Describe("Test AutoScalingListener controller with proxy", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet +// var configSecret *corev1.Secret +// var autoscalingListener *actionsv1alpha1.AutoscalingListener +// +// createRunnerSetAndListener := func(proxy *actionsv1alpha1.ProxyConfig) { +// min := 1 +// max := 10 +// autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// MaxRunners: &max, +// MinRunners: &min, +// Proxy: proxy, +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err := k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// autoscalingListener = &actionsv1alpha1.AutoscalingListener{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asl", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.AutoscalingListenerSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// RunnerScaleSetId: 1, +// AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, +// AutoscalingRunnerSetName: autoscalingRunnerSet.Name, +// EphemeralRunnerSetName: "test-ers", +// MaxRunners: 10, +// MinRunners: 1, +// Image: "ghcr.io/owner/repo", +// Proxy: proxy, +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingListener) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener") +// } +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller := &AutoscalingListenerReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("should create a secret in the listener namespace containing proxy details, use it to populate env vars on the pod and should delete it as part of cleanup", func() { +// proxyCredentials := &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "proxy-credentials", +// Namespace: autoscalingNS.Name, +// }, +// Data: map[string][]byte{ +// "username": []byte("test"), +// "password": []byte("password"), +// }, +// } +// +// err := k8sClient.Create(ctx, proxyCredentials) +// Expect(err).NotTo(HaveOccurred(), "failed to create proxy credentials secret") +// +// proxy := &actionsv1alpha1.ProxyConfig{ +// HTTP: &actionsv1alpha1.ProxyServerConfig{ +// Url: "http://localhost:8080", +// CredentialSecretRef: "proxy-credentials", +// }, +// HTTPS: &actionsv1alpha1.ProxyServerConfig{ +// Url: "https://localhost:8443", +// CredentialSecretRef: "proxy-credentials", +// }, +// NoProxy: []string{ +// "example.com", +// "example.org", +// }, +// } +// +// createRunnerSetAndListener(proxy) +// +// var proxySecret corev1.Secret +// Eventually( +// func(g Gomega) { +// err := k8sClient.Get( +// ctx, +// types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name}, +// &proxySecret, +// ) +// g.Expect(err).NotTo(HaveOccurred(), "failed to get secret") +// expected, err := autoscalingListener.Spec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) { +// var secret corev1.Secret +// err := k8sClient.Get(ctx, types.NamespacedName{Name: s, Namespace: autoscalingNS.Name}, &secret) +// if err != nil { +// return nil, err +// } +// return &secret, nil +// }) +// g.Expect(err).NotTo(HaveOccurred(), "failed to convert proxy config to secret data") +// g.Expect(proxySecret.Data).To(Equal(expected)) +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(Succeed(), "failed to create secret with proxy details") +// +// // wait for listener pod to be created +// Eventually( +// func(g Gomega) { +// pod := new(corev1.Pod) +// err := k8sClient.Get( +// ctx, +// client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace}, +// pod, +// ) +// g.Expect(err).NotTo(HaveOccurred(), "failed to get pod") +// +// g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ +// Name: "http_proxy", +// ValueFrom: &corev1.EnvVarSource{ +// SecretKeyRef: &corev1.SecretKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, +// Key: "http_proxy", +// }, +// }, +// }), "http_proxy environment variable not found") +// +// g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ +// Name: "https_proxy", +// ValueFrom: &corev1.EnvVarSource{ +// SecretKeyRef: &corev1.SecretKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, +// Key: "https_proxy", +// }, +// }, +// }), "https_proxy environment variable not found") +// +// g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ +// Name: "no_proxy", +// ValueFrom: &corev1.EnvVarSource{ +// SecretKeyRef: &corev1.SecretKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)}, +// Key: "no_proxy", +// }, +// }, +// }), "no_proxy environment variable not found") +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(Succeed(), "failed to create listener pod with proxy details") +// +// // Delete the AutoScalingListener +// err = k8sClient.Delete(ctx, autoscalingListener) +// Expect(err).NotTo(HaveOccurred(), "failed to delete test AutoScalingListener") +// +// Eventually( +// func(g Gomega) { +// var proxySecret corev1.Secret +// err := k8sClient.Get( +// ctx, +// types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name}, +// &proxySecret, +// ) +// g.Expect(kerrors.IsNotFound(err)).To(BeTrue()) +// }, +// autoscalingListenerTestTimeout, +// autoscalingListenerTestInterval).Should(Succeed(), "failed to delete secret with proxy details") +// }) +// }) +// +// var _ = Describe("Test GitHub Server TLS configuration", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var autoscalingRunnerSet *actionsv1alpha1.AutoscalingRunnerSet +// var configSecret *corev1.Secret +// var autoscalingListener *actionsv1alpha1.AutoscalingListener +// var rootCAConfigMap *corev1.ConfigMap +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// cert, err := os.ReadFile(filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// "rootCA.crt", +// )) +// Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") +// rootCAConfigMap = &corev1.ConfigMap{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "root-ca-configmap", +// Namespace: autoscalingNS.Name, +// }, +// Data: map[string]string{ +// "rootCA.crt": string(cert), +// }, +// } +// err = k8sClient.Create(ctx, rootCAConfigMap) +// Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") +// +// controller := &AutoscalingListenerReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// } +// err = controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// min := 1 +// max := 10 +// autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{ +// CertificateFrom: &actionsv1alpha1.TLSCertificateSource{ +// ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: rootCAConfigMap.Name, +// }, +// Key: "rootCA.crt", +// }, +// }, +// }, +// MaxRunners: &max, +// MinRunners: &min, +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// autoscalingListener = &actionsv1alpha1.AutoscalingListener{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asl", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.AutoscalingListenerSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{ +// CertificateFrom: &actionsv1alpha1.TLSCertificateSource{ +// ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: rootCAConfigMap.Name, +// }, +// Key: "rootCA.crt", +// }, +// }, +// }, +// RunnerScaleSetId: 1, +// AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace, +// AutoscalingRunnerSetName: autoscalingRunnerSet.Name, +// EphemeralRunnerSetName: "test-ers", +// MaxRunners: 10, +// MinRunners: 1, +// Image: "ghcr.io/owner/repo", +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingListener) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// Context("When creating a new AutoScalingListener", func() { +// It("It should set the certificates as an environment variable on the pod", func() { +// pod := new(corev1.Pod) +// Eventually( +// func(g Gomega) { +// err := k8sClient.Get( +// ctx, +// client.ObjectKey{ +// Name: autoscalingListener.Name, +// Namespace: autoscalingListener.Namespace, +// }, +// pod, +// ) +// +// g.Expect(err).NotTo(HaveOccurred(), "failed to get pod") +// g.Expect(pod.Spec.Containers).NotTo(BeEmpty(), "pod should have containers") +// g.Expect(pod.Spec.Containers[0].Env).NotTo(BeEmpty(), "pod should have env variables") +// +// var env *corev1.EnvVar +// for _, e := range pod.Spec.Containers[0].Env { +// if e.Name == "GITHUB_SERVER_ROOT_CA" { +// env = &e +// break +// } +// } +// g.Expect(env).NotTo(BeNil(), "pod should have an env variable named GITHUB_SERVER_ROOT_CA_PATH") +// +// cert, err := os.ReadFile(filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// "rootCA.crt", +// )) +// g.Expect(err).NotTo(HaveOccurred(), "failed to read rootCA.crt") +// +// g.Expect(env.Value).To( +// BeEquivalentTo(string(cert)), +// "GITHUB_SERVER_ROOT_CA should be the rootCA.crt", +// ) +// }). +// WithTimeout(autoscalingRunnerSetTestTimeout). +// WithPolling(autoscalingListenerTestInterval). +// Should(Succeed(), "failed to create pod with volume and env variable") +// }) +// }) +// }) diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go index 2a5fd780..9ebd2871 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller_test.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller_test.go @@ -2,32 +2,29 @@ package actionsgithubcom import ( "context" - "crypto/tls" - "encoding/base64" - "fmt" - "net/http" - "net/http/httptest" - "os" + "io" "path/filepath" - "strings" + "testing" "time" corev1 "k8s.io/api/core/v1" - ctrl "sigs.k8s.io/controller-runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/errors" + // . "github.com/onsi/ginkgo/v2" + // . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" - "github.com/actions/actions-runner-controller/github/actions" + actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" "github.com/actions/actions-runner-controller/github/actions/fake" - "github.com/actions/actions-runner-controller/github/actions/testserver" + "github.com/rentziass/eventually" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( @@ -36,1045 +33,1182 @@ const ( autoscalingRunnerSetTestGitHubToken = "gh_token" ) -var _ = Describe("Test AutoScalingRunnerSet controller", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet - var configSecret *corev1.Secret +func setupK8s(t *testing.T) (client.Client, *rest.Config) { + logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(io.Discard))) - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) + testEnv := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("../..", "config", "crd", "bases")}, + } - controller := &AutoscalingRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ControllerNamespace: autoscalingNS.Name, - DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", - ActionsClient: fake.NewMultiClient(), - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + // Avoids the following error: + // 2021-03-19T15:14:11.673+0900 ERROR controller-runtime.controller + // Reconciler error {"controller": "testns-tvjzjrunner", "request": + // "testns-gdnyx/example-runnerdeploy-zps4z-j5562", "error": "Pod + // \"example-runnerdeploy-zps4z-j5562\" is invalid: + // [spec.containers[1].image: Required value, + // spec.containers[1].securityContext.privileged: Forbidden: disallowed by + // cluster policy]"} + testEnv.ControlPlane.GetAPIServer().Configure(). + Append("allow-privileged", "true") - min := 1 - max := 10 - autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - MaxRunners: &max, - MinRunners: &min, - RunnerGroup: "testgroup", - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } + cfg, err := testEnv.Start() + require.NoError(t, err) + require.NotNil(t, cfg) - err = k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") + err = actionsv1alpha1.AddToScheme(scheme.Scheme) + require.NoError(t, err) - startManagers(GinkgoT(), mgr) + // +kubebuilder:scaffold:scheme + + k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + require.NoError(t, err) + require.NotNil(t, k8sClient) + + t.Cleanup(func() { + err := testEnv.Stop() + require.NoError(t, err) }) - Context("When creating a new AutoScalingRunnerSet", func() { - It("It should create/add all required resources for a new AutoScalingRunnerSet (finalizer, runnerscaleset, ephemeralrunnerset, listener)", func() { - // Check if finalizer is added - created := new(v1alpha1.AutoscalingRunnerSet) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created) - if err != nil { - return "", err - } - if len(created.Finalizers) == 0 { - return "", nil - } - return created.Finalizers[0], nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer") + return k8sClient, cfg +} - // Check if runner scale set is created on service - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created) - if err != nil { - return "", err - } +func TestAutoscalitRunnerSetReconciler_CreateRunnerScaleSet(t *testing.T) { + k8sClient, cfg := setupK8s(t) + ctx := context.Background() + autoscalingNS, mgr := createNamespace(t, k8sClient, cfg) + configSecret := createDefaultSecret(t, k8sClient, autoscalingNS.Name) - if _, ok := created.Annotations[runnerScaleSetIdKey]; !ok { - return "", nil - } + controller := &AutoscalingRunnerSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: logf.Log, + ControllerNamespace: autoscalingNS.Name, + DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", + ActionsClient: fake.NewMultiClient(), + } + err := controller.SetupWithManager(mgr) + require.NoError(t, err, "failed to setup controller") - if _, ok := created.Annotations[runnerScaleSetRunnerGroupNameKey]; !ok { - return "", nil - } - - return fmt.Sprintf("%s_%s", created.Annotations[runnerScaleSetIdKey], created.Annotations[runnerScaleSetRunnerGroupNameKey]), nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(BeEquivalentTo("1_testgroup"), "RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's annotation") - - // Check if ephemeral runner set is created - Eventually( - func() (int, error) { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - if err != nil { - return 0, err - } - - return len(runnerSetList.Items), nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created") - - // Check if listener is created - Eventually( - func() error { - return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") - - // Check if status is updated - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") - Expect(len(runnerSetList.Items)).To(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created") - }) - }) - - Context("When deleting a new AutoScalingRunnerSet", func() { - It("It should cleanup all resources for a deleting AutoScalingRunnerSet before removing it", func() { - // Wait till the listener is created - Eventually( - func() error { - return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") - - // Delete the AutoScalingRunnerSet - err := k8sClient.Delete(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to delete AutoScalingRunnerSet") - - // Check if the listener is deleted - Eventually( - func() error { - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) - if err != nil && errors.IsNotFound(err) { - return nil - } - - return fmt.Errorf("listener is not deleted") - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be deleted") - - // Check if all the EphemeralRunnerSet is deleted - Eventually( - func() error { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - if err != nil { - return err - } - - if len(runnerSetList.Items) != 0 { - return fmt.Errorf("EphemeralRunnerSet is not deleted, count=%v", len(runnerSetList.Items)) - } - - return nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(Succeed(), "All EphemeralRunnerSet should be deleted") - - // Check if the AutoScalingRunnerSet is deleted - Eventually( - func() error { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet)) - if err != nil && errors.IsNotFound(err) { - return nil - } - - return fmt.Errorf("AutoScalingRunnerSet is not deleted") - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(Succeed(), "AutoScalingRunnerSet should be deleted") - }) - }) - - Context("When updating a new AutoScalingRunnerSet", func() { - It("It should re-create EphemeralRunnerSet and Listener as needed when updating AutoScalingRunnerSet", func() { - // Wait till the listener is created - listener := new(v1alpha1.AutoscalingListener) - Eventually( - func() error { - return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") - - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") - Expect(len(runnerSetList.Items)).To(Equal(1), "There should be 1 EphemeralRunnerSet") - runnerSet := runnerSetList.Items[0] - - // Update the AutoScalingRunnerSet.Spec.Template - // This should trigger re-creation of EphemeralRunnerSet and Listener - patched := autoscalingRunnerSet.DeepCopy() - patched.Spec.Template.Spec.PriorityClassName = "test-priority-class" - err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") - autoscalingRunnerSet = patched.DeepCopy() - - // We should create a new EphemeralRunnerSet and delete the old one, eventually, we will have only one EphemeralRunnerSet - Eventually( - func() (string, error) { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - if err != nil { - return "", err - } - - if len(runnerSetList.Items) != 1 { - return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) - } - - return runnerSetList.Items[0].Labels[LabelKeyRunnerSpecHash], nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(runnerSet.Labels[LabelKeyRunnerSpecHash]), "New EphemeralRunnerSet should be created") - - // We should create a new listener - Eventually( - func() (string, error) { - listener := new(v1alpha1.AutoscalingListener) - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) - if err != nil { - return "", err - } - - return listener.Spec.EphemeralRunnerSetName, nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(runnerSet.Name), "New Listener should be created") - - // Only update the Spec for the AutoScalingListener - // This should trigger re-creation of the Listener only - runnerSetList = new(v1alpha1.EphemeralRunnerSetList) - err = k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") - Expect(len(runnerSetList.Items)).To(Equal(1), "There should be 1 EphemeralRunnerSet") - runnerSet = runnerSetList.Items[0] - - listener = new(v1alpha1.AutoscalingListener) - err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) - Expect(err).NotTo(HaveOccurred(), "failed to get Listener") - - patched = autoscalingRunnerSet.DeepCopy() - min := 10 - patched.Spec.MinRunners = &min - err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") - - // We should not re-create a new EphemeralRunnerSet - Consistently( - func() (string, error) { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - if err != nil { - return "", err - } - - if len(runnerSetList.Items) != 1 { - return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) - } - - return string(runnerSetList.Items[0].UID), nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(BeEquivalentTo(string(runnerSet.UID)), "New EphemeralRunnerSet should not be created") - - // We should only re-create a new listener - Eventually( - func() (string, error) { - listener := new(v1alpha1.AutoscalingListener) - err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) - if err != nil { - return "", err - } - - return string(listener.UID), nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(string(listener.UID)), "New Listener should be created") - }) - - It("It should update RunnerScaleSet's runner group on service when it changes", func() { - updated := new(v1alpha1.AutoscalingRunnerSet) - // Wait till the listener is created - Eventually( - func() error { - return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") - - patched := autoscalingRunnerSet.DeepCopy() - patched.Spec.RunnerGroup = "testgroup2" - err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") - - // Check if AutoScalingRunnerSet has the new runner group in its annotation - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) - if err != nil { - return "", err - } - - if _, ok := updated.Annotations[runnerScaleSetRunnerGroupNameKey]; !ok { - return "", nil - } - - return updated.Annotations[runnerScaleSetRunnerGroupNameKey], nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the new runner group in its annotation") - - // delete the annotation and it should be re-added - patched = autoscalingRunnerSet.DeepCopy() - delete(patched.Annotations, runnerScaleSetRunnerGroupNameKey) - err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") - - // Check if AutoScalingRunnerSet still has the runner group in its annotation - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) - if err != nil { - return "", err - } - - if _, ok := updated.Annotations[runnerScaleSetRunnerGroupNameKey]; !ok { - return "", nil - } - - return updated.Annotations[runnerScaleSetRunnerGroupNameKey], nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the runner group in its annotation") - }) - }) - - It("Should update Status on EphemeralRunnerSet status Update", func() { - ars := new(v1alpha1.AutoscalingRunnerSet) - Eventually( - func() (bool, error) { - err := k8sClient.Get( - ctx, - client.ObjectKey{ - Name: autoscalingRunnerSet.Name, - Namespace: autoscalingRunnerSet.Namespace, - }, - ars, - ) - if err != nil { - return false, err - } - return true, nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeTrue(), "AutoscalingRunnerSet should be created") - - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - Eventually(func() (int, error) { - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(ars.Namespace)) - if err != nil { - return 0, err - } - return len(runnerSetList.Items), nil + min := 1 + max := 10 + autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-asrs", + Namespace: autoscalingNS.Name, }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(1), "Failed to fetch runner set list") - - runnerSet := runnerSetList.Items[0] - statusUpdate := runnerSet.DeepCopy() - statusUpdate.Status.CurrentReplicas = 6 - statusUpdate.Status.FailedEphemeralRunners = 1 - statusUpdate.Status.RunningEphemeralRunners = 2 - statusUpdate.Status.PendingEphemeralRunners = 3 - - desiredStatus := v1alpha1.AutoscalingRunnerSetStatus{ - CurrentRunners: statusUpdate.Status.CurrentReplicas, - State: "", - PendingEphemeralRunners: statusUpdate.Status.PendingEphemeralRunners, - RunningEphemeralRunners: statusUpdate.Status.RunningEphemeralRunners, - FailedEphemeralRunners: statusUpdate.Status.FailedEphemeralRunners, - } - - err := k8sClient.Status().Patch(ctx, statusUpdate, client.MergeFrom(&runnerSet)) - Expect(err).NotTo(HaveOccurred(), "Failed to patch runner set status") - - Eventually( - func() (v1alpha1.AutoscalingRunnerSetStatus, error) { - updated := new(v1alpha1.AutoscalingRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) - if err != nil { - return v1alpha1.AutoscalingRunnerSetStatus{}, fmt.Errorf("failed to get AutoScalingRunnerSet: %w", err) - } - return updated.Status, nil + Spec: v1alpha1.AutoscalingRunnerSetSpec{ + GitHubConfigUrl: "https://github.com/owner/repo", + GitHubConfigSecret: configSecret.Name, + MaxRunners: &max, + MinRunners: &min, + RunnerGroup: "testgroup", + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runner", + Image: "ghcr.io/actions/runner", + }, + }, + }, }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(desiredStatus), "AutoScalingRunnerSet status should be updated") - }) -}) + }, + } -var _ = Describe("Test AutoScalingController updates", func() { - Context("Creating autoscaling runner set with RunnerScaleSetName set", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet - var configSecret *corev1.Secret + err = k8sClient.Create(ctx, autoscalingRunnerSet) + require.NoError(t, err, "failed to create AutoScalingRunnerSet") - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) + startManagers(t, mgr) - controller := &AutoscalingRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ControllerNamespace: autoscalingNS.Name, - DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", - ActionsClient: fake.NewMultiClient( - fake.WithDefaultClient( - fake.NewFakeClient( - fake.WithUpdateRunnerScaleSet( - &actions.RunnerScaleSet{ - Id: 1, - Name: "testset_update", - RunnerGroupId: 1, - RunnerGroupName: "testgroup", - Labels: []actions.Label{{Type: "test", Name: "test"}}, - RunnerSetting: actions.RunnerSetting{}, - CreatedOn: time.Now(), - RunnerJitConfigUrl: "test.test.test", - Statistics: nil, - }, - nil, - ), - ), - nil, - ), - ), - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") + // Check if finalizer is added + created := new(v1alpha1.AutoscalingRunnerSet) + eventually.Must(t, func(t testing.TB) { + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created) + require.NoError(t, err) + require.Len(t, created.Finalizers, 1) + assert.Equal(t, created.Finalizers[0], autoscalingRunnerSetFinalizerName) + }, + eventually.WithTimeout(autoscalingRunnerSetTestTimeout), + eventually.WithInterval(autoscalingRunnerSetTestInterval)) - startManagers(GinkgoT(), mgr) - }) + // Check if runner scale set is created on service + eventually.Must(t, func(t testing.TB) { + err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created) + require.NoError(t, err) - It("It should be create AutoScalingRunnerSet and has annotation for the RunnerScaleSetName", func() { - min := 1 - max := 10 - autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - MaxRunners: &max, - MinRunners: &min, - RunnerScaleSetName: "testset", - RunnerGroup: "testgroup", - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } + require.Contains(t, created.Annotations, runnerScaleSetIdKey) + assert.Equal(t, "1", created.Annotations[runnerScaleSetIdKey]) - err := k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") + require.Contains(t, created.Annotations, runnerScaleSetRunnerGroupNameKey) + assert.Equal(t, "testgroup", created.Annotations[runnerScaleSetRunnerGroupNameKey]) + }, + eventually.WithTimeout(autoscalingRunnerSetTestTimeout), + eventually.WithInterval(autoscalingRunnerSetTestInterval)) - // Wait for the AutoScalingRunnerSet to be created with right annotation - ars := new(v1alpha1.AutoscalingRunnerSet) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) - if err != nil { - return "", err - } + // Check if ephemeral runner set is created + eventually.Must(t, func(t testing.TB) { + runnerSetList := new(v1alpha1.EphemeralRunnerSetList) + err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) + require.NoError(t, err) + require.Len(t, runnerSetList.Items, 1) + }, + eventually.WithTimeout(autoscalingRunnerSetTestTimeout), + eventually.WithInterval(autoscalingRunnerSetTestInterval)) - if val, ok := ars.Annotations[runnerScaleSetNameKey]; ok { - return val, nil - } + // Check if listener is created + eventually.Must(t, func(t testing.TB) { + err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) + require.NoError(t, err) + }, + eventually.WithTimeout(autoscalingRunnerSetTestTimeout), + eventually.WithInterval(autoscalingRunnerSetTestInterval)) - return "", nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(autoscalingRunnerSet.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have annotation for the RunnerScaleSetName") + // Check if status is updated + runnerSetList := new(v1alpha1.EphemeralRunnerSetList) + err = k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) + require.NoError(t, err) + assert.Len(t, runnerSetList.Items, 1, "Only one EphemeralRunnerSet should be created") +} - update := autoscalingRunnerSet.DeepCopy() - update.Spec.RunnerScaleSetName = "testset_update" - err = k8sClient.Patch(ctx, update, client.MergeFrom(autoscalingRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to update AutoScalingRunnerSet") - - // Wait for the AutoScalingRunnerSet to be updated with right annotation - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) - if err != nil { - return "", err - } - - if val, ok := ars.Annotations[runnerScaleSetNameKey]; ok { - return val, nil - } - - return "", nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(update.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have a updated annotation for the RunnerScaleSetName") - }) - }) -}) - -var _ = Describe("Test AutoscalingController creation failures", func() { - Context("When autoscaling runner set creation fails on the client", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - - controller := &AutoscalingRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ControllerNamespace: autoscalingNS.Name, - DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", - ActionsClient: fake.NewMultiClient(), - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("It should be able to clean up if annotation related to scale set id does not exist", func() { - min := 1 - max := 10 - autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - MaxRunners: &max, - MinRunners: &min, - RunnerGroup: "testgroup", - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err := k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - // wait for the finalizer to be added - ars := new(v1alpha1.AutoscalingRunnerSet) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) - if err != nil { - return "", err - } - if len(ars.Finalizers) == 0 { - return "", nil - } - return ars.Finalizers[0], nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer") - - ars.ObjectMeta.Annotations = make(map[string]string) - err = k8sClient.Update(ctx, ars) - Expect(err).NotTo(HaveOccurred(), "Update autoscaling runner set without annotation should be successful") - - Eventually( - func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) - if err != nil { - return false, err - } - return len(ars.ObjectMeta.Annotations) == 0, nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(true), "Autoscaling runner set should be updated with empty annotations") - - err = k8sClient.Delete(ctx, ars) - Expect(err).NotTo(HaveOccurred(), "Delete autoscaling runner set should be successful") - - Eventually( - func() (bool, error) { - updated := new(v1alpha1.AutoscalingRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) - if err == nil { - return false, nil - } - if !errors.IsNotFound(err) { - return false, err - } - - return !controllerutil.ContainsFinalizer(updated, autoscalingRunnerSetFinalizerName), nil - }, - autoscalingRunnerSetTestTimeout, - autoscalingRunnerSetTestInterval, - ).Should(BeEquivalentTo(true), "Finalizer and resource should eventually be deleted") - }) - }) -}) - -var _ = Describe("Test Client optional configuration", func() { - Context("When specifying a proxy", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var configSecret *corev1.Secret - var controller *AutoscalingRunnerSetReconciler - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - controller = &AutoscalingRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ControllerNamespace: autoscalingNS.Name, - DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", - ActionsClient: actions.NewMultiClient("test", logr.Discard()), - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("should be able to make requests to a server using a proxy", func() { - serverSuccessfullyCalled := false - proxy := testserver.New(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - serverSuccessfullyCalled = true - w.WriteHeader(http.StatusOK) - })) - - min := 1 - max := 10 - autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "http://example.com/org/repo", - GitHubConfigSecret: configSecret.Name, - MaxRunners: &max, - MinRunners: &min, - RunnerGroup: "testgroup", - Proxy: &v1alpha1.ProxyConfig{ - HTTP: &v1alpha1.ProxyServerConfig{ - Url: proxy.URL, - }, - }, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err := k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - // wait for server to be called - Eventually( - func() (bool, error) { - return serverSuccessfullyCalled, nil - }, - autoscalingRunnerSetTestTimeout, - 1*time.Nanosecond, - ).Should(BeTrue(), "server was not called") - }) - - It("should be able to make requests to a server using a proxy with user info", func() { - serverSuccessfullyCalled := false - proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - header := r.Header.Get("Proxy-Authorization") - Expect(header).NotTo(BeEmpty()) - - header = strings.TrimPrefix(header, "Basic ") - decoded, err := base64.StdEncoding.DecodeString(header) - Expect(err).NotTo(HaveOccurred()) - Expect(string(decoded)).To(Equal("test:password")) - - serverSuccessfullyCalled = true - w.WriteHeader(http.StatusOK) - })) - GinkgoT().Cleanup(func() { - proxy.Close() - }) - - secretCredentials := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "proxy-credentials", - Namespace: autoscalingNS.Name, - }, - Data: map[string][]byte{ - "username": []byte("test"), - "password": []byte("password"), - }, - } - - err := k8sClient.Create(ctx, secretCredentials) - Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") - - min := 1 - max := 10 - autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "http://example.com/org/repo", - GitHubConfigSecret: configSecret.Name, - MaxRunners: &max, - MinRunners: &min, - RunnerGroup: "testgroup", - Proxy: &v1alpha1.ProxyConfig{ - HTTP: &v1alpha1.ProxyServerConfig{ - Url: proxy.URL, - CredentialSecretRef: "proxy-credentials", - }, - }, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - // wait for server to be called - Eventually( - func() (bool, error) { - return serverSuccessfullyCalled, nil - }, - autoscalingRunnerSetTestTimeout, - 1*time.Nanosecond, - ).Should(BeTrue(), "server was not called") - }) - }) - - Context("When specifying a configmap for root CAs", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var configSecret *corev1.Secret - var rootCAConfigMap *corev1.ConfigMap - var controller *AutoscalingRunnerSetReconciler - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - cert, err := os.ReadFile(filepath.Join( - "../../", - "github", - "actions", - "testdata", - "rootCA.crt", - )) - Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") - rootCAConfigMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "root-ca-configmap", - Namespace: autoscalingNS.Name, - }, - Data: map[string]string{ - "rootCA.crt": string(cert), - }, - } - err = k8sClient.Create(ctx, rootCAConfigMap) - Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") - - controller = &AutoscalingRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ControllerNamespace: autoscalingNS.Name, - DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", - ActionsClient: fake.NewMultiClient(), - } - err = controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("should be able to make requests to a server using root CAs", func() { - controller.ActionsClient = actions.NewMultiClient("test", logr.Discard()) - - certsFolder := filepath.Join( - "../../", - "github", - "actions", - "testdata", - ) - certPath := filepath.Join(certsFolder, "server.crt") - keyPath := filepath.Join(certsFolder, "server.key") - - serverSuccessfullyCalled := false - server := testserver.NewUnstarted(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - serverSuccessfullyCalled = true - w.WriteHeader(http.StatusOK) - })) - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - Expect(err).NotTo(HaveOccurred(), "failed to load server cert") - - server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} - server.StartTLS() - - min := 1 - max := 10 - autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: server.ConfigURLForOrg("my-org"), - GitHubConfigSecret: configSecret.Name, - GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{ - CertificateFrom: &v1alpha1.TLSCertificateSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAConfigMap.Name, - }, - Key: "rootCA.crt", - }, - }, - }, - MaxRunners: &max, - MinRunners: &min, - RunnerGroup: "testgroup", - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - // wait for server to be called - Eventually( - func() (bool, error) { - return serverSuccessfullyCalled, nil - }, - autoscalingRunnerSetTestTimeout, - 1*time.Nanosecond, - ).Should(BeTrue(), "server was not called") - }) - - It("it creates a listener referencing the right configmap for TLS", func() { - min := 1 - max := 10 - autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{ - CertificateFrom: &v1alpha1.TLSCertificateSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAConfigMap.Name, - }, - Key: "rootCA.crt", - }, - }, - }, - MaxRunners: &max, - MinRunners: &min, - RunnerGroup: "testgroup", - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err := k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - Eventually( - func(g Gomega) { - listener := new(v1alpha1.AutoscalingListener) - err := k8sClient.Get( - ctx, - client.ObjectKey{ - Name: scaleSetListenerName(autoscalingRunnerSet), - Namespace: autoscalingRunnerSet.Namespace, - }, - listener, - ) - g.Expect(err).NotTo(HaveOccurred(), "failed to get listener") - - g.Expect(listener.Spec.GitHubServerTLS).NotTo(BeNil(), "listener does not have TLS config") - g.Expect(listener.Spec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "listener does not have TLS config") - }, - autoscalingRunnerSetTestTimeout, - autoscalingListenerTestInterval, - ).Should(Succeed(), "tls config is incorrect") - }) - - It("it creates an ephemeral runner set referencing the right configmap for TLS", func() { - min := 1 - max := 10 - autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: v1alpha1.AutoscalingRunnerSetSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{ - CertificateFrom: &v1alpha1.TLSCertificateSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAConfigMap.Name, - }, - Key: "rootCA.crt", - }, - }, - }, - MaxRunners: &max, - MinRunners: &min, - RunnerGroup: "testgroup", - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - } - - err := k8sClient.Create(ctx, autoscalingRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") - - Eventually( - func(g Gomega) { - runnerSetList := new(v1alpha1.EphemeralRunnerSetList) - err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) - g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") - g.Expect(runnerSetList.Items).To(HaveLen(1), "expected 1 EphemeralRunnerSet to be created") - - runnerSet := &runnerSetList.Items[0] - - g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).NotTo(BeNil(), "expected EphemeralRunnerSpec.GitHubServerTLS to be set") - g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "EphemeralRunnerSpec does not have TLS config") - }, - autoscalingRunnerSetTestTimeout, - autoscalingListenerTestInterval, - ).Should(Succeed()) - }) - }) -}) +// var _ = Describe("Test AutoScalingRunnerSet controller", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet +// var configSecret *corev1.Secret +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller := &AutoscalingRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ControllerNamespace: autoscalingNS.Name, +// DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", +// ActionsClient: fake.NewMultiClient(), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// min := 1 +// max := 10 +// autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// MaxRunners: &max, +// MinRunners: &min, +// RunnerGroup: "testgroup", +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// Context("When creating a new AutoScalingRunnerSet", func() { +// It("It should create/add all required resources for a new AutoScalingRunnerSet (finalizer, runnerscaleset, ephemeralrunnerset, listener)", func() { +// // Check if finalizer is added +// created := new(v1alpha1.AutoscalingRunnerSet) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created) +// if err != nil { +// return "", err +// } +// if len(created.Finalizers) == 0 { +// return "", nil +// } +// return created.Finalizers[0], nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer") +// +// // Check if runner scale set is created on service +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, created) +// if err != nil { +// return "", err +// } +// +// if _, ok := created.Annotations[runnerScaleSetIdKey]; !ok { +// return "", nil +// } +// +// if _, ok := created.Annotations[runnerScaleSetRunnerGroupNameKey]; !ok { +// return "", nil +// } +// +// return fmt.Sprintf("%s_%s", created.Annotations[runnerScaleSetIdKey], created.Annotations[runnerScaleSetRunnerGroupNameKey]), nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(BeEquivalentTo("1_testgroup"), "RunnerScaleSet should be created/fetched and update the AutoScalingRunnerSet's annotation") +// +// // Check if ephemeral runner set is created +// Eventually( +// func() (int, error) { +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// if err != nil { +// return 0, err +// } +// +// return len(runnerSetList.Items), nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created") +// +// // Check if listener is created +// Eventually( +// func() error { +// return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") +// +// // Check if status is updated +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") +// Expect(len(runnerSetList.Items)).To(BeEquivalentTo(1), "Only one EphemeralRunnerSet should be created") +// }) +// }) +// +// Context("When deleting a new AutoScalingRunnerSet", func() { +// It("It should cleanup all resources for a deleting AutoScalingRunnerSet before removing it", func() { +// // Wait till the listener is created +// Eventually( +// func() error { +// return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") +// +// // Delete the AutoScalingRunnerSet +// err := k8sClient.Delete(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to delete AutoScalingRunnerSet") +// +// // Check if the listener is deleted +// Eventually( +// func() error { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) +// if err != nil && errors.IsNotFound(err) { +// return nil +// } +// +// return fmt.Errorf("listener is not deleted") +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be deleted") +// +// // Check if all the EphemeralRunnerSet is deleted +// Eventually( +// func() error { +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// if err != nil { +// return err +// } +// +// if len(runnerSetList.Items) != 0 { +// return fmt.Errorf("EphemeralRunnerSet is not deleted, count=%v", len(runnerSetList.Items)) +// } +// +// return nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(Succeed(), "All EphemeralRunnerSet should be deleted") +// +// // Check if the AutoScalingRunnerSet is deleted +// Eventually( +// func() error { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingRunnerSet)) +// if err != nil && errors.IsNotFound(err) { +// return nil +// } +// +// return fmt.Errorf("AutoScalingRunnerSet is not deleted") +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(Succeed(), "AutoScalingRunnerSet should be deleted") +// }) +// }) +// +// Context("When updating a new AutoScalingRunnerSet", func() { +// It("It should re-create EphemeralRunnerSet and Listener as needed when updating AutoScalingRunnerSet", func() { +// // Wait till the listener is created +// listener := new(v1alpha1.AutoscalingListener) +// Eventually( +// func() error { +// return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") +// +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") +// Expect(len(runnerSetList.Items)).To(Equal(1), "There should be 1 EphemeralRunnerSet") +// runnerSet := runnerSetList.Items[0] +// +// // Update the AutoScalingRunnerSet.Spec.Template +// // This should trigger re-creation of EphemeralRunnerSet and Listener +// patched := autoscalingRunnerSet.DeepCopy() +// patched.Spec.Template.Spec.PriorityClassName = "test-priority-class" +// err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) +// Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") +// autoscalingRunnerSet = patched.DeepCopy() +// +// // We should create a new EphemeralRunnerSet and delete the old one, eventually, we will have only one EphemeralRunnerSet +// Eventually( +// func() (string, error) { +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// if err != nil { +// return "", err +// } +// +// if len(runnerSetList.Items) != 1 { +// return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) +// } +// +// return runnerSetList.Items[0].Labels[LabelKeyRunnerSpecHash], nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(runnerSet.Labels[LabelKeyRunnerSpecHash]), "New EphemeralRunnerSet should be created") +// +// // We should create a new listener +// Eventually( +// func() (string, error) { +// listener := new(v1alpha1.AutoscalingListener) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) +// if err != nil { +// return "", err +// } +// +// return listener.Spec.EphemeralRunnerSetName, nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(runnerSet.Name), "New Listener should be created") +// +// // Only update the Spec for the AutoScalingListener +// // This should trigger re-creation of the Listener only +// runnerSetList = new(v1alpha1.EphemeralRunnerSetList) +// err = k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") +// Expect(len(runnerSetList.Items)).To(Equal(1), "There should be 1 EphemeralRunnerSet") +// runnerSet = runnerSetList.Items[0] +// +// listener = new(v1alpha1.AutoscalingListener) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) +// Expect(err).NotTo(HaveOccurred(), "failed to get Listener") +// +// patched = autoscalingRunnerSet.DeepCopy() +// min := 10 +// patched.Spec.MinRunners = &min +// err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) +// Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") +// +// // We should not re-create a new EphemeralRunnerSet +// Consistently( +// func() (string, error) { +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// if err != nil { +// return "", err +// } +// +// if len(runnerSetList.Items) != 1 { +// return "", fmt.Errorf("We should have only 1 EphemeralRunnerSet, but got %v", len(runnerSetList.Items)) +// } +// +// return string(runnerSetList.Items[0].UID), nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(BeEquivalentTo(string(runnerSet.UID)), "New EphemeralRunnerSet should not be created") +// +// // We should only re-create a new listener +// Eventually( +// func() (string, error) { +// listener := new(v1alpha1.AutoscalingListener) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, listener) +// if err != nil { +// return "", err +// } +// +// return string(listener.UID), nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).ShouldNot(BeEquivalentTo(string(listener.UID)), "New Listener should be created") +// }) +// +// It("It should update RunnerScaleSet's runner group on service when it changes", func() { +// updated := new(v1alpha1.AutoscalingRunnerSet) +// // Wait till the listener is created +// Eventually( +// func() error { +// return k8sClient.Get(ctx, client.ObjectKey{Name: scaleSetListenerName(autoscalingRunnerSet), Namespace: autoscalingRunnerSet.Namespace}, new(v1alpha1.AutoscalingListener)) +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(Succeed(), "Listener should be created") +// +// patched := autoscalingRunnerSet.DeepCopy() +// patched.Spec.RunnerGroup = "testgroup2" +// err := k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) +// Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") +// +// // Check if AutoScalingRunnerSet has the new runner group in its annotation +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) +// if err != nil { +// return "", err +// } +// +// if _, ok := updated.Annotations[runnerScaleSetRunnerGroupNameKey]; !ok { +// return "", nil +// } +// +// return updated.Annotations[runnerScaleSetRunnerGroupNameKey], nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the new runner group in its annotation") +// +// // delete the annotation and it should be re-added +// patched = autoscalingRunnerSet.DeepCopy() +// delete(patched.Annotations, runnerScaleSetRunnerGroupNameKey) +// err = k8sClient.Patch(ctx, patched, client.MergeFrom(autoscalingRunnerSet)) +// Expect(err).NotTo(HaveOccurred(), "failed to patch AutoScalingRunnerSet") +// +// // Check if AutoScalingRunnerSet still has the runner group in its annotation +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) +// if err != nil { +// return "", err +// } +// +// if _, ok := updated.Annotations[runnerScaleSetRunnerGroupNameKey]; !ok { +// return "", nil +// } +// +// return updated.Annotations[runnerScaleSetRunnerGroupNameKey], nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo("testgroup2"), "AutoScalingRunnerSet should have the runner group in its annotation") +// }) +// }) +// +// It("Should update Status on EphemeralRunnerSet status Update", func() { +// ars := new(v1alpha1.AutoscalingRunnerSet) +// Eventually( +// func() (bool, error) { +// err := k8sClient.Get( +// ctx, +// client.ObjectKey{ +// Name: autoscalingRunnerSet.Name, +// Namespace: autoscalingRunnerSet.Namespace, +// }, +// ars, +// ) +// if err != nil { +// return false, err +// } +// return true, nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeTrue(), "AutoscalingRunnerSet should be created") +// +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// Eventually(func() (int, error) { +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(ars.Namespace)) +// if err != nil { +// return 0, err +// } +// return len(runnerSetList.Items), nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo(1), "Failed to fetch runner set list") +// +// runnerSet := runnerSetList.Items[0] +// statusUpdate := runnerSet.DeepCopy() +// statusUpdate.Status.CurrentReplicas = 6 +// statusUpdate.Status.FailedEphemeralRunners = 1 +// statusUpdate.Status.RunningEphemeralRunners = 2 +// statusUpdate.Status.PendingEphemeralRunners = 3 +// +// desiredStatus := v1alpha1.AutoscalingRunnerSetStatus{ +// CurrentRunners: statusUpdate.Status.CurrentReplicas, +// State: "", +// PendingEphemeralRunners: statusUpdate.Status.PendingEphemeralRunners, +// RunningEphemeralRunners: statusUpdate.Status.RunningEphemeralRunners, +// FailedEphemeralRunners: statusUpdate.Status.FailedEphemeralRunners, +// } +// +// err := k8sClient.Status().Patch(ctx, statusUpdate, client.MergeFrom(&runnerSet)) +// Expect(err).NotTo(HaveOccurred(), "Failed to patch runner set status") +// +// Eventually( +// func() (v1alpha1.AutoscalingRunnerSetStatus, error) { +// updated := new(v1alpha1.AutoscalingRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) +// if err != nil { +// return v1alpha1.AutoscalingRunnerSetStatus{}, fmt.Errorf("failed to get AutoScalingRunnerSet: %w", err) +// } +// return updated.Status, nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo(desiredStatus), "AutoScalingRunnerSet status should be updated") +// }) +// }) +// +// var _ = Describe("Test AutoScalingController updates", func() { +// Context("Creating autoscaling runner set with RunnerScaleSetName set", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet +// var configSecret *corev1.Secret +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller := &AutoscalingRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ControllerNamespace: autoscalingNS.Name, +// DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", +// ActionsClient: fake.NewMultiClient( +// fake.WithDefaultClient( +// fake.NewFakeClient( +// fake.WithUpdateRunnerScaleSet( +// &actions.RunnerScaleSet{ +// Id: 1, +// Name: "testset_update", +// RunnerGroupId: 1, +// RunnerGroupName: "testgroup", +// Labels: []actions.Label{{Type: "test", Name: "test"}}, +// RunnerSetting: actions.RunnerSetting{}, +// CreatedOn: time.Now(), +// RunnerJitConfigUrl: "test.test.test", +// Statistics: nil, +// }, +// nil, +// ), +// ), +// nil, +// ), +// ), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("It should be create AutoScalingRunnerSet and has annotation for the RunnerScaleSetName", func() { +// min := 1 +// max := 10 +// autoscalingRunnerSet = &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// MaxRunners: &max, +// MinRunners: &min, +// RunnerScaleSetName: "testset", +// RunnerGroup: "testgroup", +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err := k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// // Wait for the AutoScalingRunnerSet to be created with right annotation +// ars := new(v1alpha1.AutoscalingRunnerSet) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) +// if err != nil { +// return "", err +// } +// +// if val, ok := ars.Annotations[runnerScaleSetNameKey]; ok { +// return val, nil +// } +// +// return "", nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo(autoscalingRunnerSet.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have annotation for the RunnerScaleSetName") +// +// update := autoscalingRunnerSet.DeepCopy() +// update.Spec.RunnerScaleSetName = "testset_update" +// err = k8sClient.Patch(ctx, update, client.MergeFrom(autoscalingRunnerSet)) +// Expect(err).NotTo(HaveOccurred(), "failed to update AutoScalingRunnerSet") +// +// // Wait for the AutoScalingRunnerSet to be updated with right annotation +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) +// if err != nil { +// return "", err +// } +// +// if val, ok := ars.Annotations[runnerScaleSetNameKey]; ok { +// return val, nil +// } +// +// return "", nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo(update.Spec.RunnerScaleSetName), "AutoScalingRunnerSet should have a updated annotation for the RunnerScaleSetName") +// }) +// }) +// }) +// +// var _ = Describe("Test AutoscalingController creation failures", func() { +// Context("When autoscaling runner set creation fails on the client", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// +// controller := &AutoscalingRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ControllerNamespace: autoscalingNS.Name, +// DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", +// ActionsClient: fake.NewMultiClient(), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("It should be able to clean up if annotation related to scale set id does not exist", func() { +// min := 1 +// max := 10 +// autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// MaxRunners: &max, +// MinRunners: &min, +// RunnerGroup: "testgroup", +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err := k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// // wait for the finalizer to be added +// ars := new(v1alpha1.AutoscalingRunnerSet) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) +// if err != nil { +// return "", err +// } +// if len(ars.Finalizers) == 0 { +// return "", nil +// } +// return ars.Finalizers[0], nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo(autoscalingRunnerSetFinalizerName), "AutoScalingRunnerSet should have a finalizer") +// +// ars.ObjectMeta.Annotations = make(map[string]string) +// err = k8sClient.Update(ctx, ars) +// Expect(err).NotTo(HaveOccurred(), "Update autoscaling runner set without annotation should be successful") +// +// Eventually( +// func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, ars) +// if err != nil { +// return false, err +// } +// return len(ars.ObjectMeta.Annotations) == 0, nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo(true), "Autoscaling runner set should be updated with empty annotations") +// +// err = k8sClient.Delete(ctx, ars) +// Expect(err).NotTo(HaveOccurred(), "Delete autoscaling runner set should be successful") +// +// Eventually( +// func() (bool, error) { +// updated := new(v1alpha1.AutoscalingRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: autoscalingRunnerSet.Name, Namespace: autoscalingRunnerSet.Namespace}, updated) +// if err == nil { +// return false, nil +// } +// if !errors.IsNotFound(err) { +// return false, err +// } +// +// return !controllerutil.ContainsFinalizer(updated, autoscalingRunnerSetFinalizerName), nil +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingRunnerSetTestInterval, +// ).Should(BeEquivalentTo(true), "Finalizer and resource should eventually be deleted") +// }) +// }) +// }) +// +// var _ = Describe("Test Client optional configuration", func() { +// Context("When specifying a proxy", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var configSecret *corev1.Secret +// var controller *AutoscalingRunnerSetReconciler +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller = &AutoscalingRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ControllerNamespace: autoscalingNS.Name, +// DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", +// ActionsClient: actions.NewMultiClient("test", logr.Discard()), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("should be able to make requests to a server using a proxy", func() { +// serverSuccessfullyCalled := false +// proxy := testserver.New(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// serverSuccessfullyCalled = true +// w.WriteHeader(http.StatusOK) +// })) +// +// min := 1 +// max := 10 +// autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "http://example.com/org/repo", +// GitHubConfigSecret: configSecret.Name, +// MaxRunners: &max, +// MinRunners: &min, +// RunnerGroup: "testgroup", +// Proxy: &v1alpha1.ProxyConfig{ +// HTTP: &v1alpha1.ProxyServerConfig{ +// Url: proxy.URL, +// }, +// }, +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err := k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// // wait for server to be called +// Eventually( +// func() (bool, error) { +// return serverSuccessfullyCalled, nil +// }, +// autoscalingRunnerSetTestTimeout, +// 1*time.Nanosecond, +// ).Should(BeTrue(), "server was not called") +// }) +// +// It("should be able to make requests to a server using a proxy with user info", func() { +// serverSuccessfullyCalled := false +// proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// header := r.Header.Get("Proxy-Authorization") +// Expect(header).NotTo(BeEmpty()) +// +// header = strings.TrimPrefix(header, "Basic ") +// decoded, err := base64.StdEncoding.DecodeString(header) +// Expect(err).NotTo(HaveOccurred()) +// Expect(string(decoded)).To(Equal("test:password")) +// +// serverSuccessfullyCalled = true +// w.WriteHeader(http.StatusOK) +// })) +// GinkgoT().Cleanup(func() { +// proxy.Close() +// }) +// +// secretCredentials := &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "proxy-credentials", +// Namespace: autoscalingNS.Name, +// }, +// Data: map[string][]byte{ +// "username": []byte("test"), +// "password": []byte("password"), +// }, +// } +// +// err := k8sClient.Create(ctx, secretCredentials) +// Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") +// +// min := 1 +// max := 10 +// autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "http://example.com/org/repo", +// GitHubConfigSecret: configSecret.Name, +// MaxRunners: &max, +// MinRunners: &min, +// RunnerGroup: "testgroup", +// Proxy: &v1alpha1.ProxyConfig{ +// HTTP: &v1alpha1.ProxyServerConfig{ +// Url: proxy.URL, +// CredentialSecretRef: "proxy-credentials", +// }, +// }, +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// // wait for server to be called +// Eventually( +// func() (bool, error) { +// return serverSuccessfullyCalled, nil +// }, +// autoscalingRunnerSetTestTimeout, +// 1*time.Nanosecond, +// ).Should(BeTrue(), "server was not called") +// }) +// }) +// +// Context("When specifying a configmap for root CAs", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var configSecret *corev1.Secret +// var rootCAConfigMap *corev1.ConfigMap +// var controller *AutoscalingRunnerSetReconciler +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// cert, err := os.ReadFile(filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// "rootCA.crt", +// )) +// Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") +// rootCAConfigMap = &corev1.ConfigMap{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "root-ca-configmap", +// Namespace: autoscalingNS.Name, +// }, +// Data: map[string]string{ +// "rootCA.crt": string(cert), +// }, +// } +// err = k8sClient.Create(ctx, rootCAConfigMap) +// Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") +// +// controller = &AutoscalingRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ControllerNamespace: autoscalingNS.Name, +// DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc", +// ActionsClient: fake.NewMultiClient(), +// } +// err = controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("should be able to make requests to a server using root CAs", func() { +// controller.ActionsClient = actions.NewMultiClient("test", logr.Discard()) +// +// certsFolder := filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// ) +// certPath := filepath.Join(certsFolder, "server.crt") +// keyPath := filepath.Join(certsFolder, "server.key") +// +// serverSuccessfullyCalled := false +// server := testserver.NewUnstarted(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// serverSuccessfullyCalled = true +// w.WriteHeader(http.StatusOK) +// })) +// cert, err := tls.LoadX509KeyPair(certPath, keyPath) +// Expect(err).NotTo(HaveOccurred(), "failed to load server cert") +// +// server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} +// server.StartTLS() +// +// min := 1 +// max := 10 +// autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: server.ConfigURLForOrg("my-org"), +// GitHubConfigSecret: configSecret.Name, +// GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{ +// CertificateFrom: &v1alpha1.TLSCertificateSource{ +// ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: rootCAConfigMap.Name, +// }, +// Key: "rootCA.crt", +// }, +// }, +// }, +// MaxRunners: &max, +// MinRunners: &min, +// RunnerGroup: "testgroup", +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// // wait for server to be called +// Eventually( +// func() (bool, error) { +// return serverSuccessfullyCalled, nil +// }, +// autoscalingRunnerSetTestTimeout, +// 1*time.Nanosecond, +// ).Should(BeTrue(), "server was not called") +// }) +// +// It("it creates a listener referencing the right configmap for TLS", func() { +// min := 1 +// max := 10 +// autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{ +// CertificateFrom: &v1alpha1.TLSCertificateSource{ +// ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: rootCAConfigMap.Name, +// }, +// Key: "rootCA.crt", +// }, +// }, +// }, +// MaxRunners: &max, +// MinRunners: &min, +// RunnerGroup: "testgroup", +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err := k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// Eventually( +// func(g Gomega) { +// listener := new(v1alpha1.AutoscalingListener) +// err := k8sClient.Get( +// ctx, +// client.ObjectKey{ +// Name: scaleSetListenerName(autoscalingRunnerSet), +// Namespace: autoscalingRunnerSet.Namespace, +// }, +// listener, +// ) +// g.Expect(err).NotTo(HaveOccurred(), "failed to get listener") +// +// g.Expect(listener.Spec.GitHubServerTLS).NotTo(BeNil(), "listener does not have TLS config") +// g.Expect(listener.Spec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "listener does not have TLS config") +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingListenerTestInterval, +// ).Should(Succeed(), "tls config is incorrect") +// }) +// +// It("it creates an ephemeral runner set referencing the right configmap for TLS", func() { +// min := 1 +// max := 10 +// autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: v1alpha1.AutoscalingRunnerSetSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// GitHubServerTLS: &v1alpha1.GitHubServerTLSConfig{ +// CertificateFrom: &v1alpha1.TLSCertificateSource{ +// ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: rootCAConfigMap.Name, +// }, +// Key: "rootCA.crt", +// }, +// }, +// }, +// MaxRunners: &max, +// MinRunners: &min, +// RunnerGroup: "testgroup", +// Template: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// } +// +// err := k8sClient.Create(ctx, autoscalingRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet") +// +// Eventually( +// func(g Gomega) { +// runnerSetList := new(v1alpha1.EphemeralRunnerSetList) +// err := k8sClient.List(ctx, runnerSetList, client.InNamespace(autoscalingRunnerSet.Namespace)) +// g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunnerSet") +// g.Expect(runnerSetList.Items).To(HaveLen(1), "expected 1 EphemeralRunnerSet to be created") +// +// runnerSet := &runnerSetList.Items[0] +// +// g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).NotTo(BeNil(), "expected EphemeralRunnerSpec.GitHubServerTLS to be set") +// g.Expect(runnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS).To(BeEquivalentTo(autoscalingRunnerSet.Spec.GitHubServerTLS), "EphemeralRunnerSpec does not have TLS config") +// }, +// autoscalingRunnerSetTestTimeout, +// autoscalingListenerTestInterval, +// ).Should(Succeed()) +// }) +// }) +// }) diff --git a/controllers/actions.github.com/ephemeralrunner_controller_test.go b/controllers/actions.github.com/ephemeralrunner_controller_test.go index bd3ca0ae..3efcac56 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller_test.go +++ b/controllers/actions.github.com/ephemeralrunner_controller_test.go @@ -1,944 +1,944 @@ package actionsgithubcom -import ( - "context" - "crypto/tls" - "encoding/base64" - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "time" - - "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" - "github.com/actions/actions-runner-controller/github/actions" - "github.com/go-logr/logr" - - "github.com/actions/actions-runner-controller/github/actions/fake" - "github.com/actions/actions-runner-controller/github/actions/testserver" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -const ( - timeout = time.Second * 10 - interval = time.Millisecond * 250 - runnerImage = "ghcr.io/actions/actions-runner:latest" -) - -func newExampleRunner(name, namespace, configSecretName string) *v1alpha1.EphemeralRunner { - return &v1alpha1.EphemeralRunner{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: v1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecretName, - RunnerScaleSetId: 1, - PodTemplateSpec: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: EphemeralRunnerContainerName, - Image: runnerImage, - Command: []string{"/runner/run.sh"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "runner", - MountPath: "/runner", - }, - }, - }, - }, - InitContainers: []corev1.Container{ - { - Name: "setup", - Image: runnerImage, - Command: []string{"sh", "-c", "cp -r /home/runner/* /runner/"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "runner", - MountPath: "/runner", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "runner", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - }, - }, - }, - } -} - -var _ = Describe("EphemeralRunner", func() { - Describe("Resource manipulation", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var configSecret *corev1.Secret - var controller *EphemeralRunnerReconciler - var ephemeralRunner *v1alpha1.EphemeralRunner - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - controller = &EphemeralRunnerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ActionsClient: fake.NewMultiClient(), - } - - err := controller.SetupWithManager(mgr) - Expect(err).To(BeNil(), "failed to setup controller") - - ephemeralRunner = newExampleRunner("test-runner", autoscalingNS.Name, configSecret.Name) - err = k8sClient.Create(ctx, ephemeralRunner) - Expect(err).To(BeNil(), "failed to create ephemeral runner") - - startManagers(GinkgoT(), mgr) - }) - - It("It should create/add all required resources for EphemeralRunner (finalizer, jit secret)", func() { - created := new(v1alpha1.EphemeralRunner) - // Check if finalizer is added - Eventually( - func() ([]string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, created) - if err != nil { - return nil, err - } - if len(created.Finalizers) == 0 { - return nil, nil - } - - n := len(created.Finalizers) // avoid capacity mismatch - return created.Finalizers[:n:n], nil - }, - timeout, - interval, - ).Should(BeEquivalentTo([]string{ephemeralRunnerActionsFinalizerName, ephemeralRunnerFinalizerName})) - - Eventually( - func() (bool, error) { - secret := new(corev1.Secret) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, secret); err != nil { - return false, err - } - - _, ok := secret.Data[jitTokenKey] - return ok, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - Eventually( - func() (string, error) { - pod := new(corev1.Pod) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return "", err - } - - return pod.Name, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(ephemeralRunner.Name)) - }) - - It("It should re-create pod on failure", func() { - pod := new(corev1.Pod) - Eventually(func() (bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }).Should(BeEquivalentTo(true)) - - err := k8sClient.Delete(ctx, pod) - Expect(err).To(BeNil(), "failed to delete pod") - - pod = new(corev1.Pod) - Eventually(func() (bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - }) - - It("It should clean up resources when deleted", func() { - // wait for pod to be created - pod := new(corev1.Pod) - Eventually(func() (bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }).Should(BeEquivalentTo(true)) - - // create runner-linked pod - runnerLinkedPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-runner-linked-pod", - Namespace: ephemeralRunner.Namespace, - Labels: map[string]string{ - "runner-pod": ephemeralRunner.Name, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner-linked-container", - Image: "ubuntu:latest", - }, - }, - }, - } - - err := k8sClient.Create(ctx, runnerLinkedPod) - Expect(err).To(BeNil(), "failed to create runner linked pod") - Eventually( - func() (bool, error) { - pod := new(corev1.Pod) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedPod.Name, Namespace: runnerLinkedPod.Namespace}, pod); err != nil { - return false, nil - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - // create runner linked secret - runnerLinkedSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-runner-linked-secret", - Namespace: ephemeralRunner.Namespace, - Labels: map[string]string{ - "runner-pod": ephemeralRunner.Name, - }, - }, - Data: map[string][]byte{"test": []byte("test")}, - } - - err = k8sClient.Create(ctx, runnerLinkedSecret) - Expect(err).To(BeNil(), "failed to create runner linked secret") - Eventually( - func() (bool, error) { - secret := new(corev1.Secret) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedSecret.Name, Namespace: runnerLinkedSecret.Namespace}, secret); err != nil { - return false, nil - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - err = k8sClient.Delete(ctx, ephemeralRunner) - Expect(err).To(BeNil(), "failed to delete ephemeral runner") - - Eventually( - func() (bool, error) { - pod := new(corev1.Pod) - err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - if err == nil { - return false, nil - } - return kerrors.IsNotFound(err), nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - Eventually( - func() (bool, error) { - secret := new(corev1.Secret) - err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, secret) - if err == nil { - return false, nil - } - return kerrors.IsNotFound(err), nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - Eventually( - func() (bool, error) { - pod := new(corev1.Pod) - err = k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedPod.Name, Namespace: runnerLinkedPod.Namespace}, pod) - if err == nil { - return false, nil - } - return kerrors.IsNotFound(err), nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - Eventually( - func() (bool, error) { - secret := new(corev1.Secret) - err = k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedSecret.Name, Namespace: runnerLinkedSecret.Namespace}, secret) - if err == nil { - return false, nil - } - return kerrors.IsNotFound(err), nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - Eventually( - func() (bool, error) { - updated := new(v1alpha1.EphemeralRunner) - err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) - if err == nil { - return false, nil - } - return kerrors.IsNotFound(err), nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - }) - - It("It should eventually have runner id set", func() { - Eventually( - func() (int, error) { - updatedEphemeralRunner := new(v1alpha1.EphemeralRunner) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updatedEphemeralRunner) - if err != nil { - return 0, err - } - return updatedEphemeralRunner.Status.RunnerId, nil - }, - timeout, - interval, - ).Should(BeNumerically(">", 0)) - }) - - It("It should patch the ephemeral runner non terminating status", func() { - pod := new(corev1.Pod) - Eventually( - func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - if err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - for _, phase := range []corev1.PodPhase{corev1.PodRunning, corev1.PodPending} { - podCopy := pod.DeepCopy() - pod.Status.Phase = phase - // set container state to force status update - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ - Name: EphemeralRunnerContainerName, - State: corev1.ContainerState{}, - }) - err := k8sClient.Status().Patch(ctx, pod, client.MergeFrom(podCopy)) - Expect(err).To(BeNil(), "failed to patch pod status") - - Eventually( - func() (corev1.PodPhase, error) { - updated := new(v1alpha1.EphemeralRunner) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) - if err != nil { - return "", err - } - return updated.Status.Phase, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(phase)) - } - }) - - It("It should not update phase if container state does not exist", func() { - pod := new(corev1.Pod) - Eventually( - func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - if err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - pod.Status.Phase = corev1.PodRunning - err := k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil(), "failed to patch pod status") - - Consistently( - func() (corev1.PodPhase, error) { - updated := new(v1alpha1.EphemeralRunner) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil { - return corev1.PodUnknown, err - } - return updated.Status.Phase, nil - }, - timeout, - ).Should(BeEquivalentTo("")) - }) - - It("It should not re-create pod indefinitely", func() { - updated := new(v1alpha1.EphemeralRunner) - pod := new(corev1.Pod) - Eventually( - func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) - if err != nil { - return false, err - } - - err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - if err != nil { - if kerrors.IsNotFound(err) && len(updated.Status.Failures) > 5 { - return true, nil - } - - return false, err - } - - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ - Name: EphemeralRunnerContainerName, - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - ExitCode: 1, - }, - }, - }) - err = k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil(), "Failed to update pod status") - return false, fmt.Errorf("pod haven't failed for 5 times.") - }, - timeout, - interval, - ).Should(BeEquivalentTo(true), "we should stop creating pod after 5 failures") - - // In case we still have pod created due to controller-runtime cache delay, mark the container as exited - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - if err == nil { - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ - Name: EphemeralRunnerContainerName, - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - ExitCode: 1, - }, - }, - }) - err := k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil(), "Failed to update pod status") - } - - // EphemeralRunner should failed with reason TooManyPodFailures - Eventually(func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) - if err != nil { - return "", err - } - return updated.Status.Reason, nil - }, timeout, interval).Should(BeEquivalentTo("TooManyPodFailures"), "Reason should be TooManyPodFailures") - - // EphemeralRunner should not have any pod - Eventually(func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - if err == nil { - return false, nil - } - return kerrors.IsNotFound(err), nil - }, timeout, interval).Should(BeEquivalentTo(true)) - }) - - It("It should re-create pod on eviction", func() { - pod := new(corev1.Pod) - Eventually( - func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - if err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - pod.Status.Phase = corev1.PodFailed - pod.Status.Reason = "Evicted" - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ - Name: EphemeralRunnerContainerName, - State: corev1.ContainerState{}, - }) - err := k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil(), "failed to patch pod status") - - updated := new(v1alpha1.EphemeralRunner) - Eventually(func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) - if err != nil { - return false, err - } - return len(updated.Status.Failures) == 1, nil - }, timeout, interval).Should(BeEquivalentTo(true)) - - // should re-create after failure - Eventually( - func() (bool, error) { - pod := new(corev1.Pod) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - }) - - It("It should re-create pod on exit status 0, but runner exists within the service", func() { - pod := new(corev1.Pod) - Eventually( - func() (bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ - Name: EphemeralRunnerContainerName, - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - ExitCode: 0, - }, - }, - }) - err := k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil(), "failed to update pod status") - - updated := new(v1alpha1.EphemeralRunner) - Eventually(func() (bool, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) - if err != nil { - return false, err - } - return len(updated.Status.Failures) == 1, nil - }, timeout, interval).Should(BeEquivalentTo(true)) - - // should re-create after failure - Eventually( - func() (bool, error) { - pod := new(corev1.Pod) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - }) - - It("It should not set the phase to succeeded without pod termination status", func() { - pod := new(corev1.Pod) - Eventually( - func() (bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(true)) - - // first set phase to running - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ - Name: EphemeralRunnerContainerName, - State: corev1.ContainerState{ - Running: &corev1.ContainerStateRunning{ - StartedAt: metav1.Now(), - }, - }, - }) - pod.Status.Phase = corev1.PodRunning - err := k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil()) - - Eventually( - func() (corev1.PodPhase, error) { - updated := new(v1alpha1.EphemeralRunner) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil { - return "", err - } - return updated.Status.Phase, nil - }, - timeout, - interval, - ).Should(BeEquivalentTo(corev1.PodRunning)) - - // set phase to succeeded - pod.Status.Phase = corev1.PodSucceeded - err = k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil()) - - Consistently( - func() (corev1.PodPhase, error) { - updated := new(v1alpha1.EphemeralRunner) - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil { - return "", err - } - return updated.Status.Phase, nil - }, - timeout, - ).Should(BeEquivalentTo(corev1.PodRunning)) - }) - }) - - Describe("Checking the API", func() { - var ctx context.Context - var autoscalingNS *corev1.Namespace - var configSecret *corev1.Secret - var controller *EphemeralRunnerReconciler - var mgr ctrl.Manager - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - controller = &EphemeralRunnerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ActionsClient: fake.NewMultiClient( - fake.WithDefaultClient( - fake.NewFakeClient( - fake.WithGetRunner( - nil, - &actions.ActionsError{ - StatusCode: http.StatusNotFound, - ExceptionName: "AgentNotFoundException", - }, - ), - ), - nil, - ), - ), - } - err := controller.SetupWithManager(mgr) - Expect(err).To(BeNil(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("It should set the Phase to Succeeded", func() { - ephemeralRunner := newExampleRunner("test-runner", autoscalingNS.Name, configSecret.Name) - - err := k8sClient.Create(ctx, ephemeralRunner) - Expect(err).To(BeNil()) - - pod := new(corev1.Pod) - Eventually(func() (bool, error) { - if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { - return false, err - } - return true, nil - }, timeout, interval).Should(BeEquivalentTo(true)) - - pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ - Name: EphemeralRunnerContainerName, - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - ExitCode: 0, - }, - }, - }) - err = k8sClient.Status().Update(ctx, pod) - Expect(err).To(BeNil(), "failed to update pod status") - - updated := new(v1alpha1.EphemeralRunner) - Eventually(func() (corev1.PodPhase, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) - if err != nil { - return "", nil - } - return updated.Status.Phase, nil - }, timeout, interval).Should(BeEquivalentTo(corev1.PodSucceeded)) - }) - }) - - Describe("Pod proxy config", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoScalingNS *corev1.Namespace - var configSecret *corev1.Secret - var controller *EphemeralRunnerReconciler - - BeforeEach(func() { - ctx = context.Background() - autoScalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoScalingNS.Name) - - controller = &EphemeralRunnerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ActionsClient: fake.NewMultiClient(), - } - err := controller.SetupWithManager(mgr) - Expect(err).To(BeNil(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("uses an actions client with proxy transport", func() { - // Use an actual client - controller.ActionsClient = actions.NewMultiClient("test", logr.Discard()) - - proxySuccessfulllyCalled := false - proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - header := r.Header.Get("Proxy-Authorization") - Expect(header).NotTo(BeEmpty()) - - header = strings.TrimPrefix(header, "Basic ") - decoded, err := base64.StdEncoding.DecodeString(header) - Expect(err).NotTo(HaveOccurred()) - Expect(string(decoded)).To(Equal("test:password")) - - proxySuccessfulllyCalled = true - w.WriteHeader(http.StatusOK) - })) - GinkgoT().Cleanup(func() { - proxy.Close() - }) - - secretCredentials := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "proxy-credentials", - Namespace: autoScalingNS.Name, - }, - Data: map[string][]byte{ - "username": []byte("test"), - "password": []byte("password"), - }, - } - - err := k8sClient.Create(ctx, secretCredentials) - Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") - - ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) - ephemeralRunner.Spec.GitHubConfigUrl = "http://example.com/org/repo" - ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{ - HTTP: &v1alpha1.ProxyServerConfig{ - Url: proxy.URL, - CredentialSecretRef: "proxy-credentials", - }, - } - - err = k8sClient.Create(ctx, ephemeralRunner) - Expect(err).To(BeNil(), "failed to create ephemeral runner") - - Eventually( - func() bool { - return proxySuccessfulllyCalled - }, - 2*time.Second, - interval, - ).Should(BeEquivalentTo(true)) - }) - - It("It should create EphemeralRunner with proxy environment variables using ProxySecretRef", func() { - ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) - ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{ - HTTP: &v1alpha1.ProxyServerConfig{ - Url: "http://proxy.example.com:8080", - }, - HTTPS: &v1alpha1.ProxyServerConfig{ - Url: "http://proxy.example.com:8080", - }, - NoProxy: []string{"example.com"}, - } - ephemeralRunner.Spec.ProxySecretRef = "proxy-secret" - err := k8sClient.Create(ctx, ephemeralRunner) - Expect(err).To(BeNil(), "failed to create ephemeral runner") - - pod := new(corev1.Pod) - Eventually( - func(g Gomega) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) - g.Expect(err).To(BeNil(), "failed to get ephemeral runner pod") - }, - timeout, - interval, - ).Should(Succeed(), "failed to get ephemeral runner pod") - - Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ - Name: EnvVarHTTPProxy, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: ephemeralRunner.Spec.ProxySecretRef, - }, - Key: "http_proxy", - }, - }, - })) - - Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ - Name: EnvVarHTTPSProxy, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: ephemeralRunner.Spec.ProxySecretRef, - }, - Key: "https_proxy", - }, - }, - })) - - Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ - Name: EnvVarNoProxy, - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: ephemeralRunner.Spec.ProxySecretRef, - }, - Key: "no_proxy", - }, - }, - })) - }) - }) - - Describe("TLS config", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoScalingNS *corev1.Namespace - var configSecret *corev1.Secret - var controller *EphemeralRunnerReconciler - var rootCAConfigMap *corev1.ConfigMap - - BeforeEach(func() { - ctx = context.Background() - autoScalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoScalingNS.Name) - - cert, err := os.ReadFile(filepath.Join( - "../../", - "github", - "actions", - "testdata", - "rootCA.crt", - )) - Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") - rootCAConfigMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "root-ca-configmap", - Namespace: autoScalingNS.Name, - }, - Data: map[string]string{ - "rootCA.crt": string(cert), - }, - } - err = k8sClient.Create(ctx, rootCAConfigMap) - Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") - - controller = &EphemeralRunnerReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ActionsClient: fake.NewMultiClient(), - } - - err = controller.SetupWithManager(mgr) - Expect(err).To(BeNil(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("should be able to make requests to a server using root CAs", func() { - certsFolder := filepath.Join( - "../../", - "github", - "actions", - "testdata", - ) - certPath := filepath.Join(certsFolder, "server.crt") - keyPath := filepath.Join(certsFolder, "server.key") - - serverSuccessfullyCalled := false - server := testserver.NewUnstarted(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - serverSuccessfullyCalled = true - w.WriteHeader(http.StatusOK) - })) - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - Expect(err).NotTo(HaveOccurred(), "failed to load server cert") - - server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} - server.StartTLS() - - // Use an actual client - controller.ActionsClient = actions.NewMultiClient("test", logr.Discard()) - - ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) - ephemeralRunner.Spec.GitHubConfigUrl = server.ConfigURLForOrg("my-org") - ephemeralRunner.Spec.GitHubServerTLS = &v1alpha1.GitHubServerTLSConfig{ - CertificateFrom: &v1alpha1.TLSCertificateSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAConfigMap.Name, - }, - Key: "rootCA.crt", - }, - }, - } - - err = k8sClient.Create(ctx, ephemeralRunner) - Expect(err).To(BeNil(), "failed to create ephemeral runner") - - Eventually( - func() bool { - return serverSuccessfullyCalled - }, - 2*time.Second, - interval, - ).Should(BeTrue(), "failed to contact server") - }) - }) -}) +// import ( +// "context" +// "crypto/tls" +// "encoding/base64" +// "fmt" +// "net/http" +// "net/http/httptest" +// "os" +// "path/filepath" +// "strings" +// "time" +// +// "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" +// "github.com/actions/actions-runner-controller/github/actions" +// "github.com/go-logr/logr" +// +// "github.com/actions/actions-runner-controller/github/actions/fake" +// "github.com/actions/actions-runner-controller/github/actions/testserver" +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// corev1 "k8s.io/api/core/v1" +// kerrors "k8s.io/apimachinery/pkg/api/errors" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// ctrl "sigs.k8s.io/controller-runtime" +// "sigs.k8s.io/controller-runtime/pkg/client" +// logf "sigs.k8s.io/controller-runtime/pkg/log" +// ) +// +// const ( +// timeout = time.Second * 10 +// interval = time.Millisecond * 250 +// runnerImage = "ghcr.io/actions/actions-runner:latest" +// ) +// +// func newExampleRunner(name, namespace, configSecretName string) *v1alpha1.EphemeralRunner { +// return &v1alpha1.EphemeralRunner{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: name, +// Namespace: namespace, +// }, +// Spec: v1alpha1.EphemeralRunnerSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecretName, +// RunnerScaleSetId: 1, +// PodTemplateSpec: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: EphemeralRunnerContainerName, +// Image: runnerImage, +// Command: []string{"/runner/run.sh"}, +// VolumeMounts: []corev1.VolumeMount{ +// { +// Name: "runner", +// MountPath: "/runner", +// }, +// }, +// }, +// }, +// InitContainers: []corev1.Container{ +// { +// Name: "setup", +// Image: runnerImage, +// Command: []string{"sh", "-c", "cp -r /home/runner/* /runner/"}, +// VolumeMounts: []corev1.VolumeMount{ +// { +// Name: "runner", +// MountPath: "/runner", +// }, +// }, +// }, +// }, +// Volumes: []corev1.Volume{ +// { +// Name: "runner", +// VolumeSource: corev1.VolumeSource{ +// EmptyDir: &corev1.EmptyDirVolumeSource{}, +// }, +// }, +// }, +// }, +// }, +// }, +// } +// } +// +// var _ = Describe("EphemeralRunner", func() { +// Describe("Resource manipulation", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var configSecret *corev1.Secret +// var controller *EphemeralRunnerReconciler +// var ephemeralRunner *v1alpha1.EphemeralRunner +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller = &EphemeralRunnerReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ActionsClient: fake.NewMultiClient(), +// } +// +// err := controller.SetupWithManager(mgr) +// Expect(err).To(BeNil(), "failed to setup controller") +// +// ephemeralRunner = newExampleRunner("test-runner", autoscalingNS.Name, configSecret.Name) +// err = k8sClient.Create(ctx, ephemeralRunner) +// Expect(err).To(BeNil(), "failed to create ephemeral runner") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("It should create/add all required resources for EphemeralRunner (finalizer, jit secret)", func() { +// created := new(v1alpha1.EphemeralRunner) +// // Check if finalizer is added +// Eventually( +// func() ([]string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, created) +// if err != nil { +// return nil, err +// } +// if len(created.Finalizers) == 0 { +// return nil, nil +// } +// +// n := len(created.Finalizers) // avoid capacity mismatch +// return created.Finalizers[:n:n], nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo([]string{ephemeralRunnerActionsFinalizerName, ephemeralRunnerFinalizerName})) +// +// Eventually( +// func() (bool, error) { +// secret := new(corev1.Secret) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, secret); err != nil { +// return false, err +// } +// +// _, ok := secret.Data[jitTokenKey] +// return ok, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// Eventually( +// func() (string, error) { +// pod := new(corev1.Pod) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return "", err +// } +// +// return pod.Name, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(ephemeralRunner.Name)) +// }) +// +// It("It should re-create pod on failure", func() { +// pod := new(corev1.Pod) +// Eventually(func() (bool, error) { +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }).Should(BeEquivalentTo(true)) +// +// err := k8sClient.Delete(ctx, pod) +// Expect(err).To(BeNil(), "failed to delete pod") +// +// pod = new(corev1.Pod) +// Eventually(func() (bool, error) { +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// }) +// +// It("It should clean up resources when deleted", func() { +// // wait for pod to be created +// pod := new(corev1.Pod) +// Eventually(func() (bool, error) { +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }).Should(BeEquivalentTo(true)) +// +// // create runner-linked pod +// runnerLinkedPod := &corev1.Pod{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-runner-linked-pod", +// Namespace: ephemeralRunner.Namespace, +// Labels: map[string]string{ +// "runner-pod": ephemeralRunner.Name, +// }, +// }, +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner-linked-container", +// Image: "ubuntu:latest", +// }, +// }, +// }, +// } +// +// err := k8sClient.Create(ctx, runnerLinkedPod) +// Expect(err).To(BeNil(), "failed to create runner linked pod") +// Eventually( +// func() (bool, error) { +// pod := new(corev1.Pod) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedPod.Name, Namespace: runnerLinkedPod.Namespace}, pod); err != nil { +// return false, nil +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// // create runner linked secret +// runnerLinkedSecret := &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-runner-linked-secret", +// Namespace: ephemeralRunner.Namespace, +// Labels: map[string]string{ +// "runner-pod": ephemeralRunner.Name, +// }, +// }, +// Data: map[string][]byte{"test": []byte("test")}, +// } +// +// err = k8sClient.Create(ctx, runnerLinkedSecret) +// Expect(err).To(BeNil(), "failed to create runner linked secret") +// Eventually( +// func() (bool, error) { +// secret := new(corev1.Secret) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedSecret.Name, Namespace: runnerLinkedSecret.Namespace}, secret); err != nil { +// return false, nil +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// err = k8sClient.Delete(ctx, ephemeralRunner) +// Expect(err).To(BeNil(), "failed to delete ephemeral runner") +// +// Eventually( +// func() (bool, error) { +// pod := new(corev1.Pod) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// if err == nil { +// return false, nil +// } +// return kerrors.IsNotFound(err), nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// Eventually( +// func() (bool, error) { +// secret := new(corev1.Secret) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, secret) +// if err == nil { +// return false, nil +// } +// return kerrors.IsNotFound(err), nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// Eventually( +// func() (bool, error) { +// pod := new(corev1.Pod) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedPod.Name, Namespace: runnerLinkedPod.Namespace}, pod) +// if err == nil { +// return false, nil +// } +// return kerrors.IsNotFound(err), nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// Eventually( +// func() (bool, error) { +// secret := new(corev1.Secret) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: runnerLinkedSecret.Name, Namespace: runnerLinkedSecret.Namespace}, secret) +// if err == nil { +// return false, nil +// } +// return kerrors.IsNotFound(err), nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// Eventually( +// func() (bool, error) { +// updated := new(v1alpha1.EphemeralRunner) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) +// if err == nil { +// return false, nil +// } +// return kerrors.IsNotFound(err), nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// }) +// +// It("It should eventually have runner id set", func() { +// Eventually( +// func() (int, error) { +// updatedEphemeralRunner := new(v1alpha1.EphemeralRunner) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updatedEphemeralRunner) +// if err != nil { +// return 0, err +// } +// return updatedEphemeralRunner.Status.RunnerId, nil +// }, +// timeout, +// interval, +// ).Should(BeNumerically(">", 0)) +// }) +// +// It("It should patch the ephemeral runner non terminating status", func() { +// pod := new(corev1.Pod) +// Eventually( +// func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// if err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// for _, phase := range []corev1.PodPhase{corev1.PodRunning, corev1.PodPending} { +// podCopy := pod.DeepCopy() +// pod.Status.Phase = phase +// // set container state to force status update +// pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ +// Name: EphemeralRunnerContainerName, +// State: corev1.ContainerState{}, +// }) +// err := k8sClient.Status().Patch(ctx, pod, client.MergeFrom(podCopy)) +// Expect(err).To(BeNil(), "failed to patch pod status") +// +// Eventually( +// func() (corev1.PodPhase, error) { +// updated := new(v1alpha1.EphemeralRunner) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) +// if err != nil { +// return "", err +// } +// return updated.Status.Phase, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(phase)) +// } +// }) +// +// It("It should not update phase if container state does not exist", func() { +// pod := new(corev1.Pod) +// Eventually( +// func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// if err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// pod.Status.Phase = corev1.PodRunning +// err := k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil(), "failed to patch pod status") +// +// Consistently( +// func() (corev1.PodPhase, error) { +// updated := new(v1alpha1.EphemeralRunner) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil { +// return corev1.PodUnknown, err +// } +// return updated.Status.Phase, nil +// }, +// timeout, +// ).Should(BeEquivalentTo("")) +// }) +// +// It("It should not re-create pod indefinitely", func() { +// updated := new(v1alpha1.EphemeralRunner) +// pod := new(corev1.Pod) +// Eventually( +// func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) +// if err != nil { +// return false, err +// } +// +// err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// if err != nil { +// if kerrors.IsNotFound(err) && len(updated.Status.Failures) > 5 { +// return true, nil +// } +// +// return false, err +// } +// +// pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ +// Name: EphemeralRunnerContainerName, +// State: corev1.ContainerState{ +// Terminated: &corev1.ContainerStateTerminated{ +// ExitCode: 1, +// }, +// }, +// }) +// err = k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil(), "Failed to update pod status") +// return false, fmt.Errorf("pod haven't failed for 5 times.") +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true), "we should stop creating pod after 5 failures") +// +// // In case we still have pod created due to controller-runtime cache delay, mark the container as exited +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// if err == nil { +// pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ +// Name: EphemeralRunnerContainerName, +// State: corev1.ContainerState{ +// Terminated: &corev1.ContainerStateTerminated{ +// ExitCode: 1, +// }, +// }, +// }) +// err := k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil(), "Failed to update pod status") +// } +// +// // EphemeralRunner should failed with reason TooManyPodFailures +// Eventually(func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) +// if err != nil { +// return "", err +// } +// return updated.Status.Reason, nil +// }, timeout, interval).Should(BeEquivalentTo("TooManyPodFailures"), "Reason should be TooManyPodFailures") +// +// // EphemeralRunner should not have any pod +// Eventually(func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// if err == nil { +// return false, nil +// } +// return kerrors.IsNotFound(err), nil +// }, timeout, interval).Should(BeEquivalentTo(true)) +// }) +// +// It("It should re-create pod on eviction", func() { +// pod := new(corev1.Pod) +// Eventually( +// func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// if err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// pod.Status.Phase = corev1.PodFailed +// pod.Status.Reason = "Evicted" +// pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ +// Name: EphemeralRunnerContainerName, +// State: corev1.ContainerState{}, +// }) +// err := k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil(), "failed to patch pod status") +// +// updated := new(v1alpha1.EphemeralRunner) +// Eventually(func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) +// if err != nil { +// return false, err +// } +// return len(updated.Status.Failures) == 1, nil +// }, timeout, interval).Should(BeEquivalentTo(true)) +// +// // should re-create after failure +// Eventually( +// func() (bool, error) { +// pod := new(corev1.Pod) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// }) +// +// It("It should re-create pod on exit status 0, but runner exists within the service", func() { +// pod := new(corev1.Pod) +// Eventually( +// func() (bool, error) { +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ +// Name: EphemeralRunnerContainerName, +// State: corev1.ContainerState{ +// Terminated: &corev1.ContainerStateTerminated{ +// ExitCode: 0, +// }, +// }, +// }) +// err := k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil(), "failed to update pod status") +// +// updated := new(v1alpha1.EphemeralRunner) +// Eventually(func() (bool, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) +// if err != nil { +// return false, err +// } +// return len(updated.Status.Failures) == 1, nil +// }, timeout, interval).Should(BeEquivalentTo(true)) +// +// // should re-create after failure +// Eventually( +// func() (bool, error) { +// pod := new(corev1.Pod) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// }) +// +// It("It should not set the phase to succeeded without pod termination status", func() { +// pod := new(corev1.Pod) +// Eventually( +// func() (bool, error) { +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(true)) +// +// // first set phase to running +// pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ +// Name: EphemeralRunnerContainerName, +// State: corev1.ContainerState{ +// Running: &corev1.ContainerStateRunning{ +// StartedAt: metav1.Now(), +// }, +// }, +// }) +// pod.Status.Phase = corev1.PodRunning +// err := k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil()) +// +// Eventually( +// func() (corev1.PodPhase, error) { +// updated := new(v1alpha1.EphemeralRunner) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil { +// return "", err +// } +// return updated.Status.Phase, nil +// }, +// timeout, +// interval, +// ).Should(BeEquivalentTo(corev1.PodRunning)) +// +// // set phase to succeeded +// pod.Status.Phase = corev1.PodSucceeded +// err = k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil()) +// +// Consistently( +// func() (corev1.PodPhase, error) { +// updated := new(v1alpha1.EphemeralRunner) +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated); err != nil { +// return "", err +// } +// return updated.Status.Phase, nil +// }, +// timeout, +// ).Should(BeEquivalentTo(corev1.PodRunning)) +// }) +// }) +// +// Describe("Checking the API", func() { +// var ctx context.Context +// var autoscalingNS *corev1.Namespace +// var configSecret *corev1.Secret +// var controller *EphemeralRunnerReconciler +// var mgr ctrl.Manager +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller = &EphemeralRunnerReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ActionsClient: fake.NewMultiClient( +// fake.WithDefaultClient( +// fake.NewFakeClient( +// fake.WithGetRunner( +// nil, +// &actions.ActionsError{ +// StatusCode: http.StatusNotFound, +// ExceptionName: "AgentNotFoundException", +// }, +// ), +// ), +// nil, +// ), +// ), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).To(BeNil(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("It should set the Phase to Succeeded", func() { +// ephemeralRunner := newExampleRunner("test-runner", autoscalingNS.Name, configSecret.Name) +// +// err := k8sClient.Create(ctx, ephemeralRunner) +// Expect(err).To(BeNil()) +// +// pod := new(corev1.Pod) +// Eventually(func() (bool, error) { +// if err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod); err != nil { +// return false, err +// } +// return true, nil +// }, timeout, interval).Should(BeEquivalentTo(true)) +// +// pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, corev1.ContainerStatus{ +// Name: EphemeralRunnerContainerName, +// State: corev1.ContainerState{ +// Terminated: &corev1.ContainerStateTerminated{ +// ExitCode: 0, +// }, +// }, +// }) +// err = k8sClient.Status().Update(ctx, pod) +// Expect(err).To(BeNil(), "failed to update pod status") +// +// updated := new(v1alpha1.EphemeralRunner) +// Eventually(func() (corev1.PodPhase, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, updated) +// if err != nil { +// return "", nil +// } +// return updated.Status.Phase, nil +// }, timeout, interval).Should(BeEquivalentTo(corev1.PodSucceeded)) +// }) +// }) +// +// Describe("Pod proxy config", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoScalingNS *corev1.Namespace +// var configSecret *corev1.Secret +// var controller *EphemeralRunnerReconciler +// +// BeforeEach(func() { +// ctx = context.Background() +// autoScalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoScalingNS.Name) +// +// controller = &EphemeralRunnerReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ActionsClient: fake.NewMultiClient(), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).To(BeNil(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("uses an actions client with proxy transport", func() { +// // Use an actual client +// controller.ActionsClient = actions.NewMultiClient("test", logr.Discard()) +// +// proxySuccessfulllyCalled := false +// proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// header := r.Header.Get("Proxy-Authorization") +// Expect(header).NotTo(BeEmpty()) +// +// header = strings.TrimPrefix(header, "Basic ") +// decoded, err := base64.StdEncoding.DecodeString(header) +// Expect(err).NotTo(HaveOccurred()) +// Expect(string(decoded)).To(Equal("test:password")) +// +// proxySuccessfulllyCalled = true +// w.WriteHeader(http.StatusOK) +// })) +// GinkgoT().Cleanup(func() { +// proxy.Close() +// }) +// +// secretCredentials := &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "proxy-credentials", +// Namespace: autoScalingNS.Name, +// }, +// Data: map[string][]byte{ +// "username": []byte("test"), +// "password": []byte("password"), +// }, +// } +// +// err := k8sClient.Create(ctx, secretCredentials) +// Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") +// +// ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) +// ephemeralRunner.Spec.GitHubConfigUrl = "http://example.com/org/repo" +// ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{ +// HTTP: &v1alpha1.ProxyServerConfig{ +// Url: proxy.URL, +// CredentialSecretRef: "proxy-credentials", +// }, +// } +// +// err = k8sClient.Create(ctx, ephemeralRunner) +// Expect(err).To(BeNil(), "failed to create ephemeral runner") +// +// Eventually( +// func() bool { +// return proxySuccessfulllyCalled +// }, +// 2*time.Second, +// interval, +// ).Should(BeEquivalentTo(true)) +// }) +// +// It("It should create EphemeralRunner with proxy environment variables using ProxySecretRef", func() { +// ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) +// ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{ +// HTTP: &v1alpha1.ProxyServerConfig{ +// Url: "http://proxy.example.com:8080", +// }, +// HTTPS: &v1alpha1.ProxyServerConfig{ +// Url: "http://proxy.example.com:8080", +// }, +// NoProxy: []string{"example.com"}, +// } +// ephemeralRunner.Spec.ProxySecretRef = "proxy-secret" +// err := k8sClient.Create(ctx, ephemeralRunner) +// Expect(err).To(BeNil(), "failed to create ephemeral runner") +// +// pod := new(corev1.Pod) +// Eventually( +// func(g Gomega) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod) +// g.Expect(err).To(BeNil(), "failed to get ephemeral runner pod") +// }, +// timeout, +// interval, +// ).Should(Succeed(), "failed to get ephemeral runner pod") +// +// Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ +// Name: EnvVarHTTPProxy, +// ValueFrom: &corev1.EnvVarSource{ +// SecretKeyRef: &corev1.SecretKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: ephemeralRunner.Spec.ProxySecretRef, +// }, +// Key: "http_proxy", +// }, +// }, +// })) +// +// Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ +// Name: EnvVarHTTPSProxy, +// ValueFrom: &corev1.EnvVarSource{ +// SecretKeyRef: &corev1.SecretKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: ephemeralRunner.Spec.ProxySecretRef, +// }, +// Key: "https_proxy", +// }, +// }, +// })) +// +// Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ +// Name: EnvVarNoProxy, +// ValueFrom: &corev1.EnvVarSource{ +// SecretKeyRef: &corev1.SecretKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: ephemeralRunner.Spec.ProxySecretRef, +// }, +// Key: "no_proxy", +// }, +// }, +// })) +// }) +// }) +// +// Describe("TLS config", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoScalingNS *corev1.Namespace +// var configSecret *corev1.Secret +// var controller *EphemeralRunnerReconciler +// var rootCAConfigMap *corev1.ConfigMap +// +// BeforeEach(func() { +// ctx = context.Background() +// autoScalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoScalingNS.Name) +// +// cert, err := os.ReadFile(filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// "rootCA.crt", +// )) +// Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") +// rootCAConfigMap = &corev1.ConfigMap{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "root-ca-configmap", +// Namespace: autoScalingNS.Name, +// }, +// Data: map[string]string{ +// "rootCA.crt": string(cert), +// }, +// } +// err = k8sClient.Create(ctx, rootCAConfigMap) +// Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") +// +// controller = &EphemeralRunnerReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ActionsClient: fake.NewMultiClient(), +// } +// +// err = controller.SetupWithManager(mgr) +// Expect(err).To(BeNil(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("should be able to make requests to a server using root CAs", func() { +// certsFolder := filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// ) +// certPath := filepath.Join(certsFolder, "server.crt") +// keyPath := filepath.Join(certsFolder, "server.key") +// +// serverSuccessfullyCalled := false +// server := testserver.NewUnstarted(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// serverSuccessfullyCalled = true +// w.WriteHeader(http.StatusOK) +// })) +// cert, err := tls.LoadX509KeyPair(certPath, keyPath) +// Expect(err).NotTo(HaveOccurred(), "failed to load server cert") +// +// server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} +// server.StartTLS() +// +// // Use an actual client +// controller.ActionsClient = actions.NewMultiClient("test", logr.Discard()) +// +// ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name) +// ephemeralRunner.Spec.GitHubConfigUrl = server.ConfigURLForOrg("my-org") +// ephemeralRunner.Spec.GitHubServerTLS = &v1alpha1.GitHubServerTLSConfig{ +// CertificateFrom: &v1alpha1.TLSCertificateSource{ +// ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: rootCAConfigMap.Name, +// }, +// Key: "rootCA.crt", +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, ephemeralRunner) +// Expect(err).To(BeNil(), "failed to create ephemeral runner") +// +// Eventually( +// func() bool { +// return serverSuccessfullyCalled +// }, +// 2*time.Second, +// interval, +// ).Should(BeTrue(), "failed to contact server") +// }) +// }) +// }) diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go index fecf1b00..33437103 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller_test.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller_test.go @@ -1,1162 +1,1162 @@ package actionsgithubcom -import ( - "context" - "crypto/tls" - "encoding/base64" - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "time" - - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" - v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" - "github.com/actions/actions-runner-controller/github/actions" - "github.com/actions/actions-runner-controller/github/actions/fake" - "github.com/actions/actions-runner-controller/github/actions/testserver" -) - -const ( - ephemeralRunnerSetTestTimeout = time.Second * 10 - ephemeralRunnerSetTestInterval = time.Millisecond * 250 - ephemeralRunnerSetTestGitHubToken = "gh_token" -) - -var _ = Describe("Test EphemeralRunnerSet controller", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var ephemeralRunnerSet *actionsv1alpha1.EphemeralRunnerSet - var configSecret *corev1.Secret - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - controller := &EphemeralRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ActionsClient: fake.NewMultiClient(), - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ - EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "https://github.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 100, - PodTemplateSpec: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, ephemeralRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") - - startManagers(GinkgoT(), mgr) - }) - - Context("When creating a new EphemeralRunnerSet", func() { - It("It should create/add all required resources for a new EphemeralRunnerSet (finalizer)", func() { - // Check if finalizer is added - created := new(actionsv1alpha1.EphemeralRunnerSet) - Eventually( - func() (string, error) { - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) - if err != nil { - return "", err - } - if len(created.Finalizers) == 0 { - return "", nil - } - return created.Finalizers[0], nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(ephemeralRunnerSetFinalizerName), "EphemeralRunnerSet should have a finalizer") - - // Check if the number of ephemeral runners are stay 0 - Consistently( - func() (int, error) { - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "No EphemeralRunner should be created") - - // Check if the status stay 0 - Consistently( - func() (int, error) { - runnerSet := new(actionsv1alpha1.EphemeralRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, runnerSet) - if err != nil { - return -1, err - } - - return int(runnerSet.Status.CurrentReplicas), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "EphemeralRunnerSet status should be 0") - - // Scaling up the EphemeralRunnerSet - updated := created.DeepCopy() - updated.Spec.Replicas = 5 - err := k8sClient.Update(ctx, updated) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - // Check if the number of ephemeral runners are created - Eventually( - func() (int, error) { - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") - - // Check if the status is updated - Eventually( - func() (int, error) { - runnerSet := new(actionsv1alpha1.EphemeralRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, runnerSet) - if err != nil { - return -1, err - } - - return int(runnerSet.Status.CurrentReplicas), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "EphemeralRunnerSet status should be 5") - }) - }) - - Context("When deleting a new EphemeralRunnerSet", func() { - It("It should cleanup all resources for a deleting EphemeralRunnerSet before removing it", func() { - created := new(actionsv1alpha1.EphemeralRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) - Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") - - // Scale up the EphemeralRunnerSet - updated := created.DeepCopy() - updated.Spec.Replicas = 5 - err = k8sClient.Update(ctx, updated) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - // Wait for the EphemeralRunnerSet to be scaled up - Eventually( - func() (int, error) { - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") - - // Delete the EphemeralRunnerSet - err = k8sClient.Delete(ctx, created) - Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunnerSet") - - // Check if all ephemeral runners are deleted - Eventually( - func() (int, error) { - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "All EphemeralRunner should be deleted") - - // Check if the EphemeralRunnerSet is deleted - Eventually( - func() error { - deleted := new(actionsv1alpha1.EphemeralRunnerSet) - err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, deleted) - if err != nil { - if kerrors.IsNotFound(err) { - return nil - } - - return err - } - - return fmt.Errorf("EphemeralRunnerSet is not deleted") - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(Succeed(), "EphemeralRunnerSet should be deleted") - }) - }) - - Context("When a new EphemeralRunnerSet scale up and down", func() { - It("It should delete finished EphemeralRunner and create new EphemeralRunner", func() { - created := new(actionsv1alpha1.EphemeralRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) - Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") - - // Scale up the EphemeralRunnerSet - updated := created.DeepCopy() - updated.Spec.Replicas = 5 - err = k8sClient.Update(ctx, updated) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - // Wait for the EphemeralRunnerSet to be scaled up - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - Eventually( - func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") - - // Mark one of the EphemeralRunner as finished - finishedRunner := runnerList.Items[4].DeepCopy() - finishedRunner.Status.Phase = corev1.PodSucceeded - err = k8sClient.Status().Patch(ctx, finishedRunner, client.MergeFrom(&runnerList.Items[4])) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - - // Wait for the finished EphemeralRunner to be deleted - Eventually( - func() error { - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return err - } - - for _, runner := range runnerList.Items { - if runner.Name == finishedRunner.Name { - return fmt.Errorf("EphemeralRunner is not deleted") - } - } - - return nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(Succeed(), "Finished EphemeralRunner should be deleted") - - // We should still have the EphemeralRunnerSet scale up - runnerList = new(actionsv1alpha1.EphemeralRunnerList) - Eventually( - func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") - - // Scale down the EphemeralRunnerSet - updated = created.DeepCopy() - updated.Spec.Replicas = 3 - err = k8sClient.Patch(ctx, updated, client.MergeFrom(created)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - // Wait for the EphemeralRunnerSet to be scaled down - runnerList = new(actionsv1alpha1.EphemeralRunnerList) - Eventually( - func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(3), "3 EphemeralRunner should be created") - - // We will not scale down runner that is running jobs - runningRunner := runnerList.Items[0].DeepCopy() - runningRunner.Status.JobRequestId = 1000 - err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[0])) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - - runningRunner = runnerList.Items[1].DeepCopy() - runningRunner.Status.JobRequestId = 1001 - err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[1])) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - - // Scale down to 1 - updated = created.DeepCopy() - updated.Spec.Replicas = 1 - err = k8sClient.Patch(ctx, updated, client.MergeFrom(created)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - // Wait for the EphemeralRunnerSet to be scaled down to 2 since we still have 2 runner running jobs - runnerList = new(actionsv1alpha1.EphemeralRunnerList) - Eventually( - func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created") - - // We will not scale down failed runner - failedRunner := runnerList.Items[0].DeepCopy() - failedRunner.Status.Phase = corev1.PodFailed - err = k8sClient.Status().Patch(ctx, failedRunner, client.MergeFrom(&runnerList.Items[0])) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - - // Scale down to 0 - updated = created.DeepCopy() - updated.Spec.Replicas = 0 - err = k8sClient.Patch(ctx, updated, client.MergeFrom(created)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - // We should not scale down the EphemeralRunnerSet since we still have 1 runner running job and 1 failed runner - runnerList = new(actionsv1alpha1.EphemeralRunnerList) - Consistently( - func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created") - - // We will scale down to 0 when the running job is completed and the failed runner is deleted - runningRunner = runnerList.Items[1].DeepCopy() - runningRunner.Status.Phase = corev1.PodSucceeded - err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[1])) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - - err = k8sClient.Delete(ctx, &runnerList.Items[0]) - Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunner") - - // Wait for the EphemeralRunnerSet to be scaled down to 0 - runnerList = new(actionsv1alpha1.EphemeralRunnerList) - Eventually( - func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodRunning - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "0 EphemeralRunner should be created") - }) - - It("Should update status on Ephemeral Runner state changes", func() { - created := new(actionsv1alpha1.EphemeralRunnerSet) - Eventually( - func() error { - return k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(Succeed(), "EphemeralRunnerSet should be created") - - // Scale up the EphemeralRunnerSet - updated := created.DeepCopy() - updated.Spec.Replicas = 3 - err := k8sClient.Update(ctx, updated) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet replica count") - - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - Eventually( - func() (bool, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return false, err - } - - if len(runnerList.Items) != 3 { - return false, err - } - - var pendingOriginal *v1alpha1.EphemeralRunner - var runningOriginal *v1alpha1.EphemeralRunner - var failedOriginal *v1alpha1.EphemeralRunner - var empty []*v1alpha1.EphemeralRunner - for _, runner := range runnerList.Items { - switch runner.Status.RunnerId { - case 101: - pendingOriginal = runner.DeepCopy() - case 102: - runningOriginal = runner.DeepCopy() - case 103: - failedOriginal = runner.DeepCopy() - default: - empty = append(empty, runner.DeepCopy()) - } - } - - refetch := false - if pendingOriginal == nil { // if NO pending - refetch = true - pendingOriginal = empty[0] - empty = empty[1:] - - pending := pendingOriginal.DeepCopy() - pending.Status.RunnerId = 101 - pending.Status.Phase = corev1.PodPending - - err = k8sClient.Status().Patch(ctx, pending, client.MergeFrom(pendingOriginal)) - if err != nil { - return false, err - } - } - - if runningOriginal == nil { // if NO running - refetch = true - runningOriginal = empty[0] - empty = empty[1:] - running := runningOriginal.DeepCopy() - running.Status.RunnerId = 102 - running.Status.Phase = corev1.PodRunning - - err = k8sClient.Status().Patch(ctx, running, client.MergeFrom(runningOriginal)) - if err != nil { - return false, err - } - } - - if failedOriginal == nil { // if NO failed - refetch = true - failedOriginal = empty[0] - - failed := pendingOriginal.DeepCopy() - failed.Status.RunnerId = 103 - failed.Status.Phase = corev1.PodFailed - - err = k8sClient.Status().Patch(ctx, failed, client.MergeFrom(failedOriginal)) - if err != nil { - return false, err - } - } - - return !refetch, nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(BeTrue(), "Failed to eventually update to one pending, one running and one failed") - - desiredStatus := v1alpha1.EphemeralRunnerSetStatus{ - CurrentReplicas: 3, - PendingEphemeralRunners: 1, - RunningEphemeralRunners: 1, - FailedEphemeralRunners: 1, - } - Eventually( - func() (v1alpha1.EphemeralRunnerSetStatus, error) { - updated := new(v1alpha1.EphemeralRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) - if err != nil { - return v1alpha1.EphemeralRunnerSetStatus{}, err - } - return updated.Status, nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(BeEquivalentTo(desiredStatus), "Status is not eventually updated to the desired one") - - updated = new(v1alpha1.EphemeralRunnerSet) - err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) - Expect(err).NotTo(HaveOccurred(), "Failed to fetch ephemeral runner set") - - updatedOriginal := updated.DeepCopy() - updated.Spec.Replicas = 0 - - err = k8sClient.Patch(ctx, updated, client.MergeFrom(updatedOriginal)) - Expect(err).NotTo(HaveOccurred(), "Failed to patch ephemeral runner set with 0 replicas") - - Eventually( - func() (int, error) { - runnerList = new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(BeEquivalentTo(1), "Failed to eventually scale down") - - desiredStatus = v1alpha1.EphemeralRunnerSetStatus{ - CurrentReplicas: 1, - PendingEphemeralRunners: 0, - RunningEphemeralRunners: 0, - FailedEphemeralRunners: 1, - } - - Eventually( - func() (v1alpha1.EphemeralRunnerSetStatus, error) { - updated := new(v1alpha1.EphemeralRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) - if err != nil { - return v1alpha1.EphemeralRunnerSetStatus{}, err - } - return updated.Status, nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(BeEquivalentTo(desiredStatus), "Status is not eventually updated to the desired one") - - err = k8sClient.Delete(ctx, &runnerList.Items[0]) - Expect(err).To(BeNil(), "Failed to delete failed ephemeral runner") - - desiredStatus = v1alpha1.EphemeralRunnerSetStatus{} // empty - Eventually( - func() (v1alpha1.EphemeralRunnerSetStatus, error) { - updated := new(v1alpha1.EphemeralRunnerSet) - err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) - if err != nil { - return v1alpha1.EphemeralRunnerSetStatus{}, err - } - return updated.Status, nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(BeEquivalentTo(desiredStatus), "Status is not eventually updated to the desired one") - }) - }) -}) - -var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var ephemeralRunnerSet *actionsv1alpha1.EphemeralRunnerSet - var configSecret *corev1.Secret - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - controller := &EphemeralRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ActionsClient: actions.NewMultiClient("test", logr.Discard()), - } - err := controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("should create a proxy secret and delete the proxy secreat after the runner-set is deleted", func() { - secretCredentials := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "proxy-credentials", - Namespace: autoscalingNS.Name, - }, - Data: map[string][]byte{ - "username": []byte("username"), - "password": []byte("password"), - }, - } - - err := k8sClient.Create(ctx, secretCredentials) - Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") - - ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ - Replicas: 1, - EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "http://example.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 100, - Proxy: &v1alpha1.ProxyConfig{ - HTTP: &v1alpha1.ProxyServerConfig{ - Url: "http://proxy.example.com", - CredentialSecretRef: secretCredentials.Name, - }, - HTTPS: &v1alpha1.ProxyServerConfig{ - Url: "https://proxy.example.com", - CredentialSecretRef: secretCredentials.Name, - }, - NoProxy: []string{"example.com", "example.org"}, - }, - PodTemplateSpec: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, ephemeralRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") - - Eventually(func(g Gomega) { - // Compiled / flattened proxy secret should exist at this point - actualProxySecret := &corev1.Secret{} - err = k8sClient.Get(ctx, client.ObjectKey{ - Namespace: autoscalingNS.Name, - Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), - }, actualProxySecret) - g.Expect(err).NotTo(HaveOccurred(), "failed to get compiled / flattened proxy secret") - - secretFetcher := func(name string) (*corev1.Secret, error) { - secret := &corev1.Secret{} - err = k8sClient.Get(ctx, client.ObjectKey{ - Namespace: autoscalingNS.Name, - Name: name, - }, secret) - return secret, err - } - - // Assert that the proxy secret is created with the correct values - expectedData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(secretFetcher) - g.Expect(err).NotTo(HaveOccurred(), "failed to get proxy secret data") - g.Expect(actualProxySecret.Data).To(Equal(expectedData)) - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(Succeed(), "compiled / flattened proxy secret should exist") - - Eventually(func(g Gomega) { - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunners") - - for _, runner := range runnerList.Items { - g.Expect(runner.Spec.ProxySecretRef).To(Equal(proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet))) - } - }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval).Should(Succeed(), "EphemeralRunners should have a reference to the proxy secret") - - // patch ephemeral runner set to have 0 replicas - patch := client.MergeFrom(ephemeralRunnerSet.DeepCopy()) - ephemeralRunnerSet.Spec.Replicas = 0 - err = k8sClient.Patch(ctx, ephemeralRunnerSet, patch) - Expect(err).NotTo(HaveOccurred(), "failed to patch EphemeralRunnerSet") - - // Set pods to PodSucceeded to simulate an actual EphemeralRunner stopping - Eventually( - func(g Gomega) (int, error) { - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - // Set status to simulate a configured EphemeralRunner - refetch := false - for i, runner := range runnerList.Items { - if runner.Status.RunnerId == 0 { - updatedRunner := runner.DeepCopy() - updatedRunner.Status.Phase = corev1.PodSucceeded - updatedRunner.Status.RunnerId = i + 100 - err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") - refetch = true - } - } - - if refetch { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(1), "1 EphemeralRunner should exist") - - // Delete the EphemeralRunnerSet - err = k8sClient.Delete(ctx, ephemeralRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunnerSet") - - // Assert that the proxy secret is deleted - Eventually(func(g Gomega) { - proxySecret := &corev1.Secret{} - err = k8sClient.Get(ctx, client.ObjectKey{ - Namespace: autoscalingNS.Name, - Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), - }, proxySecret) - g.Expect(err).To(HaveOccurred(), "proxy secret should be deleted") - g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "proxy secret should be deleted") - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(Succeed(), "proxy secret should be deleted") - }) - - It("should configure the actions client to use proxy details", func() { - secretCredentials := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "proxy-credentials", - Namespace: autoscalingNS.Name, - }, - Data: map[string][]byte{ - "username": []byte("test"), - "password": []byte("password"), - }, - } - - err := k8sClient.Create(ctx, secretCredentials) - Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") - - proxySuccessfulllyCalled := false - proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - header := r.Header.Get("Proxy-Authorization") - Expect(header).NotTo(BeEmpty()) - - header = strings.TrimPrefix(header, "Basic ") - decoded, err := base64.StdEncoding.DecodeString(header) - Expect(err).NotTo(HaveOccurred()) - Expect(string(decoded)).To(Equal("test:password")) - - proxySuccessfulllyCalled = true - w.WriteHeader(http.StatusOK) - })) - GinkgoT().Cleanup(func() { - proxy.Close() - }) - - ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ - Replicas: 1, - EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: "http://example.com/owner/repo", - GitHubConfigSecret: configSecret.Name, - RunnerScaleSetId: 100, - Proxy: &v1alpha1.ProxyConfig{ - HTTP: &v1alpha1.ProxyServerConfig{ - Url: proxy.URL, - CredentialSecretRef: "proxy-credentials", - }, - }, - PodTemplateSpec: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, ephemeralRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") - - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - Eventually(func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(BeEquivalentTo(1), "failed to create ephemeral runner") - - runner := runnerList.Items[0].DeepCopy() - runner.Status.Phase = corev1.PodRunning - runner.Status.RunnerId = 100 - err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0])) - Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status") - - runnerSet := new(actionsv1alpha1.EphemeralRunnerSet) - err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, runnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") - - updatedRunnerSet := runnerSet.DeepCopy() - updatedRunnerSet.Spec.Replicas = 0 - err = k8sClient.Patch(ctx, updatedRunnerSet, client.MergeFrom(runnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - Eventually( - func() bool { - return proxySuccessfulllyCalled - }, - 2*time.Second, - interval, - ).Should(BeEquivalentTo(true)) - }) -}) - -var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func() { - var ctx context.Context - var mgr ctrl.Manager - var autoscalingNS *corev1.Namespace - var ephemeralRunnerSet *actionsv1alpha1.EphemeralRunnerSet - var configSecret *corev1.Secret - var rootCAConfigMap *corev1.ConfigMap - - BeforeEach(func() { - ctx = context.Background() - autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) - configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) - - cert, err := os.ReadFile(filepath.Join( - "../../", - "github", - "actions", - "testdata", - "rootCA.crt", - )) - Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") - rootCAConfigMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "root-ca-configmap", - Namespace: autoscalingNS.Name, - }, - Data: map[string]string{ - "rootCA.crt": string(cert), - }, - } - err = k8sClient.Create(ctx, rootCAConfigMap) - Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") - - controller := &EphemeralRunnerSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logf.Log, - ActionsClient: actions.NewMultiClient("test", logr.Discard()), - } - err = controller.SetupWithManager(mgr) - Expect(err).NotTo(HaveOccurred(), "failed to setup controller") - - startManagers(GinkgoT(), mgr) - }) - - It("should be able to make requests to a server using root CAs", func() { - certsFolder := filepath.Join( - "../../", - "github", - "actions", - "testdata", - ) - certPath := filepath.Join(certsFolder, "server.crt") - keyPath := filepath.Join(certsFolder, "server.key") - - serverSuccessfullyCalled := false - server := testserver.NewUnstarted(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - serverSuccessfullyCalled = true - w.WriteHeader(http.StatusOK) - })) - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - Expect(err).NotTo(HaveOccurred(), "failed to load server cert") - - server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} - server.StartTLS() - - ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-asrs", - Namespace: autoscalingNS.Name, - }, - Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ - Replicas: 1, - EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ - GitHubConfigUrl: server.ConfigURLForOrg("my-org"), - GitHubConfigSecret: configSecret.Name, - GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{ - CertificateFrom: &v1alpha1.TLSCertificateSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAConfigMap.Name, - }, - Key: "rootCA.crt", - }, - }, - }, - RunnerScaleSetId: 100, - PodTemplateSpec: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "runner", - Image: "ghcr.io/actions/runner", - }, - }, - }, - }, - }, - }, - } - - err = k8sClient.Create(ctx, ephemeralRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") - - runnerList := new(actionsv1alpha1.EphemeralRunnerList) - Eventually(func() (int, error) { - err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) - if err != nil { - return -1, err - } - - return len(runnerList.Items), nil - }, - ephemeralRunnerSetTestTimeout, - ephemeralRunnerSetTestInterval, - ).Should(BeEquivalentTo(1), "failed to create ephemeral runner") - - runner := runnerList.Items[0].DeepCopy() - Expect(runner.Spec.GitHubServerTLS).NotTo(BeNil(), "runner tls config should not be nil") - Expect(runner.Spec.GitHubServerTLS).To(BeEquivalentTo(ephemeralRunnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS), "runner tls config should be correct") - - runner.Status.Phase = corev1.PodRunning - runner.Status.RunnerId = 100 - err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0])) - Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status") - - currentRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet) - err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, currentRunnerSet) - Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") - - updatedRunnerSet := currentRunnerSet.DeepCopy() - updatedRunnerSet.Spec.Replicas = 0 - err = k8sClient.Patch(ctx, updatedRunnerSet, client.MergeFrom(currentRunnerSet)) - Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") - - // wait for server to be called - Eventually( - func() bool { - return serverSuccessfullyCalled - }, - autoscalingRunnerSetTestTimeout, - 1*time.Nanosecond, - ).Should(BeTrue(), "server was not called") - }) -}) +// import ( +// "context" +// "crypto/tls" +// "encoding/base64" +// "fmt" +// "net/http" +// "net/http/httptest" +// "os" +// "path/filepath" +// "strings" +// "time" +// +// corev1 "k8s.io/api/core/v1" +// kerrors "k8s.io/apimachinery/pkg/api/errors" +// ctrl "sigs.k8s.io/controller-runtime" +// "sigs.k8s.io/controller-runtime/pkg/client" +// logf "sigs.k8s.io/controller-runtime/pkg/log" +// +// "github.com/go-logr/logr" +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// +// actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" +// v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" +// "github.com/actions/actions-runner-controller/github/actions" +// "github.com/actions/actions-runner-controller/github/actions/fake" +// "github.com/actions/actions-runner-controller/github/actions/testserver" +// ) +// +// const ( +// ephemeralRunnerSetTestTimeout = time.Second * 10 +// ephemeralRunnerSetTestInterval = time.Millisecond * 250 +// ephemeralRunnerSetTestGitHubToken = "gh_token" +// ) +// +// var _ = Describe("Test EphemeralRunnerSet controller", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var ephemeralRunnerSet *actionsv1alpha1.EphemeralRunnerSet +// var configSecret *corev1.Secret +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller := &EphemeralRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ActionsClient: fake.NewMultiClient(), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ +// EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ +// GitHubConfigUrl: "https://github.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// RunnerScaleSetId: 100, +// PodTemplateSpec: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, ephemeralRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// Context("When creating a new EphemeralRunnerSet", func() { +// It("It should create/add all required resources for a new EphemeralRunnerSet (finalizer)", func() { +// // Check if finalizer is added +// created := new(actionsv1alpha1.EphemeralRunnerSet) +// Eventually( +// func() (string, error) { +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) +// if err != nil { +// return "", err +// } +// if len(created.Finalizers) == 0 { +// return "", nil +// } +// return created.Finalizers[0], nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(ephemeralRunnerSetFinalizerName), "EphemeralRunnerSet should have a finalizer") +// +// // Check if the number of ephemeral runners are stay 0 +// Consistently( +// func() (int, error) { +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "No EphemeralRunner should be created") +// +// // Check if the status stay 0 +// Consistently( +// func() (int, error) { +// runnerSet := new(actionsv1alpha1.EphemeralRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, runnerSet) +// if err != nil { +// return -1, err +// } +// +// return int(runnerSet.Status.CurrentReplicas), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "EphemeralRunnerSet status should be 0") +// +// // Scaling up the EphemeralRunnerSet +// updated := created.DeepCopy() +// updated.Spec.Replicas = 5 +// err := k8sClient.Update(ctx, updated) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// // Check if the number of ephemeral runners are created +// Eventually( +// func() (int, error) { +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") +// +// // Check if the status is updated +// Eventually( +// func() (int, error) { +// runnerSet := new(actionsv1alpha1.EphemeralRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, runnerSet) +// if err != nil { +// return -1, err +// } +// +// return int(runnerSet.Status.CurrentReplicas), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "EphemeralRunnerSet status should be 5") +// }) +// }) +// +// Context("When deleting a new EphemeralRunnerSet", func() { +// It("It should cleanup all resources for a deleting EphemeralRunnerSet before removing it", func() { +// created := new(actionsv1alpha1.EphemeralRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) +// Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") +// +// // Scale up the EphemeralRunnerSet +// updated := created.DeepCopy() +// updated.Spec.Replicas = 5 +// err = k8sClient.Update(ctx, updated) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// // Wait for the EphemeralRunnerSet to be scaled up +// Eventually( +// func() (int, error) { +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") +// +// // Delete the EphemeralRunnerSet +// err = k8sClient.Delete(ctx, created) +// Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunnerSet") +// +// // Check if all ephemeral runners are deleted +// Eventually( +// func() (int, error) { +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "All EphemeralRunner should be deleted") +// +// // Check if the EphemeralRunnerSet is deleted +// Eventually( +// func() error { +// deleted := new(actionsv1alpha1.EphemeralRunnerSet) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, deleted) +// if err != nil { +// if kerrors.IsNotFound(err) { +// return nil +// } +// +// return err +// } +// +// return fmt.Errorf("EphemeralRunnerSet is not deleted") +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(Succeed(), "EphemeralRunnerSet should be deleted") +// }) +// }) +// +// Context("When a new EphemeralRunnerSet scale up and down", func() { +// It("It should delete finished EphemeralRunner and create new EphemeralRunner", func() { +// created := new(actionsv1alpha1.EphemeralRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) +// Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") +// +// // Scale up the EphemeralRunnerSet +// updated := created.DeepCopy() +// updated.Spec.Replicas = 5 +// err = k8sClient.Update(ctx, updated) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// // Wait for the EphemeralRunnerSet to be scaled up +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// Eventually( +// func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") +// +// // Mark one of the EphemeralRunner as finished +// finishedRunner := runnerList.Items[4].DeepCopy() +// finishedRunner.Status.Phase = corev1.PodSucceeded +// err = k8sClient.Status().Patch(ctx, finishedRunner, client.MergeFrom(&runnerList.Items[4])) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// +// // Wait for the finished EphemeralRunner to be deleted +// Eventually( +// func() error { +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return err +// } +// +// for _, runner := range runnerList.Items { +// if runner.Name == finishedRunner.Name { +// return fmt.Errorf("EphemeralRunner is not deleted") +// } +// } +// +// return nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(Succeed(), "Finished EphemeralRunner should be deleted") +// +// // We should still have the EphemeralRunnerSet scale up +// runnerList = new(actionsv1alpha1.EphemeralRunnerList) +// Eventually( +// func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created") +// +// // Scale down the EphemeralRunnerSet +// updated = created.DeepCopy() +// updated.Spec.Replicas = 3 +// err = k8sClient.Patch(ctx, updated, client.MergeFrom(created)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// // Wait for the EphemeralRunnerSet to be scaled down +// runnerList = new(actionsv1alpha1.EphemeralRunnerList) +// Eventually( +// func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(3), "3 EphemeralRunner should be created") +// +// // We will not scale down runner that is running jobs +// runningRunner := runnerList.Items[0].DeepCopy() +// runningRunner.Status.JobRequestId = 1000 +// err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[0])) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// +// runningRunner = runnerList.Items[1].DeepCopy() +// runningRunner.Status.JobRequestId = 1001 +// err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[1])) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// +// // Scale down to 1 +// updated = created.DeepCopy() +// updated.Spec.Replicas = 1 +// err = k8sClient.Patch(ctx, updated, client.MergeFrom(created)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// // Wait for the EphemeralRunnerSet to be scaled down to 2 since we still have 2 runner running jobs +// runnerList = new(actionsv1alpha1.EphemeralRunnerList) +// Eventually( +// func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created") +// +// // We will not scale down failed runner +// failedRunner := runnerList.Items[0].DeepCopy() +// failedRunner.Status.Phase = corev1.PodFailed +// err = k8sClient.Status().Patch(ctx, failedRunner, client.MergeFrom(&runnerList.Items[0])) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// +// // Scale down to 0 +// updated = created.DeepCopy() +// updated.Spec.Replicas = 0 +// err = k8sClient.Patch(ctx, updated, client.MergeFrom(created)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// // We should not scale down the EphemeralRunnerSet since we still have 1 runner running job and 1 failed runner +// runnerList = new(actionsv1alpha1.EphemeralRunnerList) +// Consistently( +// func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created") +// +// // We will scale down to 0 when the running job is completed and the failed runner is deleted +// runningRunner = runnerList.Items[1].DeepCopy() +// runningRunner.Status.Phase = corev1.PodSucceeded +// err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[1])) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// +// err = k8sClient.Delete(ctx, &runnerList.Items[0]) +// Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunner") +// +// // Wait for the EphemeralRunnerSet to be scaled down to 0 +// runnerList = new(actionsv1alpha1.EphemeralRunnerList) +// Eventually( +// func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodRunning +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(0), "0 EphemeralRunner should be created") +// }) +// +// It("Should update status on Ephemeral Runner state changes", func() { +// created := new(actionsv1alpha1.EphemeralRunnerSet) +// Eventually( +// func() error { +// return k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created) +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(Succeed(), "EphemeralRunnerSet should be created") +// +// // Scale up the EphemeralRunnerSet +// updated := created.DeepCopy() +// updated.Spec.Replicas = 3 +// err := k8sClient.Update(ctx, updated) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet replica count") +// +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// Eventually( +// func() (bool, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return false, err +// } +// +// if len(runnerList.Items) != 3 { +// return false, err +// } +// +// var pendingOriginal *v1alpha1.EphemeralRunner +// var runningOriginal *v1alpha1.EphemeralRunner +// var failedOriginal *v1alpha1.EphemeralRunner +// var empty []*v1alpha1.EphemeralRunner +// for _, runner := range runnerList.Items { +// switch runner.Status.RunnerId { +// case 101: +// pendingOriginal = runner.DeepCopy() +// case 102: +// runningOriginal = runner.DeepCopy() +// case 103: +// failedOriginal = runner.DeepCopy() +// default: +// empty = append(empty, runner.DeepCopy()) +// } +// } +// +// refetch := false +// if pendingOriginal == nil { // if NO pending +// refetch = true +// pendingOriginal = empty[0] +// empty = empty[1:] +// +// pending := pendingOriginal.DeepCopy() +// pending.Status.RunnerId = 101 +// pending.Status.Phase = corev1.PodPending +// +// err = k8sClient.Status().Patch(ctx, pending, client.MergeFrom(pendingOriginal)) +// if err != nil { +// return false, err +// } +// } +// +// if runningOriginal == nil { // if NO running +// refetch = true +// runningOriginal = empty[0] +// empty = empty[1:] +// running := runningOriginal.DeepCopy() +// running.Status.RunnerId = 102 +// running.Status.Phase = corev1.PodRunning +// +// err = k8sClient.Status().Patch(ctx, running, client.MergeFrom(runningOriginal)) +// if err != nil { +// return false, err +// } +// } +// +// if failedOriginal == nil { // if NO failed +// refetch = true +// failedOriginal = empty[0] +// +// failed := pendingOriginal.DeepCopy() +// failed.Status.RunnerId = 103 +// failed.Status.Phase = corev1.PodFailed +// +// err = k8sClient.Status().Patch(ctx, failed, client.MergeFrom(failedOriginal)) +// if err != nil { +// return false, err +// } +// } +// +// return !refetch, nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(BeTrue(), "Failed to eventually update to one pending, one running and one failed") +// +// desiredStatus := v1alpha1.EphemeralRunnerSetStatus{ +// CurrentReplicas: 3, +// PendingEphemeralRunners: 1, +// RunningEphemeralRunners: 1, +// FailedEphemeralRunners: 1, +// } +// Eventually( +// func() (v1alpha1.EphemeralRunnerSetStatus, error) { +// updated := new(v1alpha1.EphemeralRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) +// if err != nil { +// return v1alpha1.EphemeralRunnerSetStatus{}, err +// } +// return updated.Status, nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(BeEquivalentTo(desiredStatus), "Status is not eventually updated to the desired one") +// +// updated = new(v1alpha1.EphemeralRunnerSet) +// err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) +// Expect(err).NotTo(HaveOccurred(), "Failed to fetch ephemeral runner set") +// +// updatedOriginal := updated.DeepCopy() +// updated.Spec.Replicas = 0 +// +// err = k8sClient.Patch(ctx, updated, client.MergeFrom(updatedOriginal)) +// Expect(err).NotTo(HaveOccurred(), "Failed to patch ephemeral runner set with 0 replicas") +// +// Eventually( +// func() (int, error) { +// runnerList = new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(BeEquivalentTo(1), "Failed to eventually scale down") +// +// desiredStatus = v1alpha1.EphemeralRunnerSetStatus{ +// CurrentReplicas: 1, +// PendingEphemeralRunners: 0, +// RunningEphemeralRunners: 0, +// FailedEphemeralRunners: 1, +// } +// +// Eventually( +// func() (v1alpha1.EphemeralRunnerSetStatus, error) { +// updated := new(v1alpha1.EphemeralRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) +// if err != nil { +// return v1alpha1.EphemeralRunnerSetStatus{}, err +// } +// return updated.Status, nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(BeEquivalentTo(desiredStatus), "Status is not eventually updated to the desired one") +// +// err = k8sClient.Delete(ctx, &runnerList.Items[0]) +// Expect(err).To(BeNil(), "Failed to delete failed ephemeral runner") +// +// desiredStatus = v1alpha1.EphemeralRunnerSetStatus{} // empty +// Eventually( +// func() (v1alpha1.EphemeralRunnerSetStatus, error) { +// updated := new(v1alpha1.EphemeralRunnerSet) +// err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, updated) +// if err != nil { +// return v1alpha1.EphemeralRunnerSetStatus{}, err +// } +// return updated.Status, nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(BeEquivalentTo(desiredStatus), "Status is not eventually updated to the desired one") +// }) +// }) +// }) +// +// var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var ephemeralRunnerSet *actionsv1alpha1.EphemeralRunnerSet +// var configSecret *corev1.Secret +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// controller := &EphemeralRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ActionsClient: actions.NewMultiClient("test", logr.Discard()), +// } +// err := controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("should create a proxy secret and delete the proxy secreat after the runner-set is deleted", func() { +// secretCredentials := &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "proxy-credentials", +// Namespace: autoscalingNS.Name, +// }, +// Data: map[string][]byte{ +// "username": []byte("username"), +// "password": []byte("password"), +// }, +// } +// +// err := k8sClient.Create(ctx, secretCredentials) +// Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") +// +// ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ +// Replicas: 1, +// EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ +// GitHubConfigUrl: "http://example.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// RunnerScaleSetId: 100, +// Proxy: &v1alpha1.ProxyConfig{ +// HTTP: &v1alpha1.ProxyServerConfig{ +// Url: "http://proxy.example.com", +// CredentialSecretRef: secretCredentials.Name, +// }, +// HTTPS: &v1alpha1.ProxyServerConfig{ +// Url: "https://proxy.example.com", +// CredentialSecretRef: secretCredentials.Name, +// }, +// NoProxy: []string{"example.com", "example.org"}, +// }, +// PodTemplateSpec: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, ephemeralRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") +// +// Eventually(func(g Gomega) { +// // Compiled / flattened proxy secret should exist at this point +// actualProxySecret := &corev1.Secret{} +// err = k8sClient.Get(ctx, client.ObjectKey{ +// Namespace: autoscalingNS.Name, +// Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), +// }, actualProxySecret) +// g.Expect(err).NotTo(HaveOccurred(), "failed to get compiled / flattened proxy secret") +// +// secretFetcher := func(name string) (*corev1.Secret, error) { +// secret := &corev1.Secret{} +// err = k8sClient.Get(ctx, client.ObjectKey{ +// Namespace: autoscalingNS.Name, +// Name: name, +// }, secret) +// return secret, err +// } +// +// // Assert that the proxy secret is created with the correct values +// expectedData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(secretFetcher) +// g.Expect(err).NotTo(HaveOccurred(), "failed to get proxy secret data") +// g.Expect(actualProxySecret.Data).To(Equal(expectedData)) +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(Succeed(), "compiled / flattened proxy secret should exist") +// +// Eventually(func(g Gomega) { +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunners") +// +// for _, runner := range runnerList.Items { +// g.Expect(runner.Spec.ProxySecretRef).To(Equal(proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet))) +// } +// }, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval).Should(Succeed(), "EphemeralRunners should have a reference to the proxy secret") +// +// // patch ephemeral runner set to have 0 replicas +// patch := client.MergeFrom(ephemeralRunnerSet.DeepCopy()) +// ephemeralRunnerSet.Spec.Replicas = 0 +// err = k8sClient.Patch(ctx, ephemeralRunnerSet, patch) +// Expect(err).NotTo(HaveOccurred(), "failed to patch EphemeralRunnerSet") +// +// // Set pods to PodSucceeded to simulate an actual EphemeralRunner stopping +// Eventually( +// func(g Gomega) (int, error) { +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// // Set status to simulate a configured EphemeralRunner +// refetch := false +// for i, runner := range runnerList.Items { +// if runner.Status.RunnerId == 0 { +// updatedRunner := runner.DeepCopy() +// updatedRunner.Status.Phase = corev1.PodSucceeded +// updatedRunner.Status.RunnerId = i + 100 +// err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner") +// refetch = true +// } +// } +// +// if refetch { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(1), "1 EphemeralRunner should exist") +// +// // Delete the EphemeralRunnerSet +// err = k8sClient.Delete(ctx, ephemeralRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunnerSet") +// +// // Assert that the proxy secret is deleted +// Eventually(func(g Gomega) { +// proxySecret := &corev1.Secret{} +// err = k8sClient.Get(ctx, client.ObjectKey{ +// Namespace: autoscalingNS.Name, +// Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet), +// }, proxySecret) +// g.Expect(err).To(HaveOccurred(), "proxy secret should be deleted") +// g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "proxy secret should be deleted") +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(Succeed(), "proxy secret should be deleted") +// }) +// +// It("should configure the actions client to use proxy details", func() { +// secretCredentials := &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "proxy-credentials", +// Namespace: autoscalingNS.Name, +// }, +// Data: map[string][]byte{ +// "username": []byte("test"), +// "password": []byte("password"), +// }, +// } +// +// err := k8sClient.Create(ctx, secretCredentials) +// Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials") +// +// proxySuccessfulllyCalled := false +// proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// header := r.Header.Get("Proxy-Authorization") +// Expect(header).NotTo(BeEmpty()) +// +// header = strings.TrimPrefix(header, "Basic ") +// decoded, err := base64.StdEncoding.DecodeString(header) +// Expect(err).NotTo(HaveOccurred()) +// Expect(string(decoded)).To(Equal("test:password")) +// +// proxySuccessfulllyCalled = true +// w.WriteHeader(http.StatusOK) +// })) +// GinkgoT().Cleanup(func() { +// proxy.Close() +// }) +// +// ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ +// Replicas: 1, +// EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ +// GitHubConfigUrl: "http://example.com/owner/repo", +// GitHubConfigSecret: configSecret.Name, +// RunnerScaleSetId: 100, +// Proxy: &v1alpha1.ProxyConfig{ +// HTTP: &v1alpha1.ProxyServerConfig{ +// Url: proxy.URL, +// CredentialSecretRef: "proxy-credentials", +// }, +// }, +// PodTemplateSpec: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, ephemeralRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") +// +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// Eventually(func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(BeEquivalentTo(1), "failed to create ephemeral runner") +// +// runner := runnerList.Items[0].DeepCopy() +// runner.Status.Phase = corev1.PodRunning +// runner.Status.RunnerId = 100 +// err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0])) +// Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status") +// +// runnerSet := new(actionsv1alpha1.EphemeralRunnerSet) +// err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, runnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") +// +// updatedRunnerSet := runnerSet.DeepCopy() +// updatedRunnerSet.Spec.Replicas = 0 +// err = k8sClient.Patch(ctx, updatedRunnerSet, client.MergeFrom(runnerSet)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// Eventually( +// func() bool { +// return proxySuccessfulllyCalled +// }, +// 2*time.Second, +// interval, +// ).Should(BeEquivalentTo(true)) +// }) +// }) +// +// var _ = Describe("Test EphemeralRunnerSet controller with custom root CA", func() { +// var ctx context.Context +// var mgr ctrl.Manager +// var autoscalingNS *corev1.Namespace +// var ephemeralRunnerSet *actionsv1alpha1.EphemeralRunnerSet +// var configSecret *corev1.Secret +// var rootCAConfigMap *corev1.ConfigMap +// +// BeforeEach(func() { +// ctx = context.Background() +// autoscalingNS, mgr = createNamespace(GinkgoT(), k8sClient) +// configSecret = createDefaultSecret(GinkgoT(), k8sClient, autoscalingNS.Name) +// +// cert, err := os.ReadFile(filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// "rootCA.crt", +// )) +// Expect(err).NotTo(HaveOccurred(), "failed to read root CA cert") +// rootCAConfigMap = &corev1.ConfigMap{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "root-ca-configmap", +// Namespace: autoscalingNS.Name, +// }, +// Data: map[string]string{ +// "rootCA.crt": string(cert), +// }, +// } +// err = k8sClient.Create(ctx, rootCAConfigMap) +// Expect(err).NotTo(HaveOccurred(), "failed to create configmap with root CAs") +// +// controller := &EphemeralRunnerSetReconciler{ +// Client: mgr.GetClient(), +// Scheme: mgr.GetScheme(), +// Log: logf.Log, +// ActionsClient: actions.NewMultiClient("test", logr.Discard()), +// } +// err = controller.SetupWithManager(mgr) +// Expect(err).NotTo(HaveOccurred(), "failed to setup controller") +// +// startManagers(GinkgoT(), mgr) +// }) +// +// It("should be able to make requests to a server using root CAs", func() { +// certsFolder := filepath.Join( +// "../../", +// "github", +// "actions", +// "testdata", +// ) +// certPath := filepath.Join(certsFolder, "server.crt") +// keyPath := filepath.Join(certsFolder, "server.key") +// +// serverSuccessfullyCalled := false +// server := testserver.NewUnstarted(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// serverSuccessfullyCalled = true +// w.WriteHeader(http.StatusOK) +// })) +// cert, err := tls.LoadX509KeyPair(certPath, keyPath) +// Expect(err).NotTo(HaveOccurred(), "failed to load server cert") +// +// server.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} +// server.StartTLS() +// +// ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-asrs", +// Namespace: autoscalingNS.Name, +// }, +// Spec: actionsv1alpha1.EphemeralRunnerSetSpec{ +// Replicas: 1, +// EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{ +// GitHubConfigUrl: server.ConfigURLForOrg("my-org"), +// GitHubConfigSecret: configSecret.Name, +// GitHubServerTLS: &actionsv1alpha1.GitHubServerTLSConfig{ +// CertificateFrom: &v1alpha1.TLSCertificateSource{ +// ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ +// LocalObjectReference: corev1.LocalObjectReference{ +// Name: rootCAConfigMap.Name, +// }, +// Key: "rootCA.crt", +// }, +// }, +// }, +// RunnerScaleSetId: 100, +// PodTemplateSpec: corev1.PodTemplateSpec{ +// Spec: corev1.PodSpec{ +// Containers: []corev1.Container{ +// { +// Name: "runner", +// Image: "ghcr.io/actions/runner", +// }, +// }, +// }, +// }, +// }, +// }, +// } +// +// err = k8sClient.Create(ctx, ephemeralRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet") +// +// runnerList := new(actionsv1alpha1.EphemeralRunnerList) +// Eventually(func() (int, error) { +// err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace)) +// if err != nil { +// return -1, err +// } +// +// return len(runnerList.Items), nil +// }, +// ephemeralRunnerSetTestTimeout, +// ephemeralRunnerSetTestInterval, +// ).Should(BeEquivalentTo(1), "failed to create ephemeral runner") +// +// runner := runnerList.Items[0].DeepCopy() +// Expect(runner.Spec.GitHubServerTLS).NotTo(BeNil(), "runner tls config should not be nil") +// Expect(runner.Spec.GitHubServerTLS).To(BeEquivalentTo(ephemeralRunnerSet.Spec.EphemeralRunnerSpec.GitHubServerTLS), "runner tls config should be correct") +// +// runner.Status.Phase = corev1.PodRunning +// runner.Status.RunnerId = 100 +// err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0])) +// Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status") +// +// currentRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet) +// err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, currentRunnerSet) +// Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet") +// +// updatedRunnerSet := currentRunnerSet.DeepCopy() +// updatedRunnerSet.Spec.Replicas = 0 +// err = k8sClient.Patch(ctx, updatedRunnerSet, client.MergeFrom(currentRunnerSet)) +// Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet") +// +// // wait for server to be called +// Eventually( +// func() bool { +// return serverSuccessfullyCalled +// }, +// autoscalingRunnerSetTestTimeout, +// 1*time.Nanosecond, +// ).Should(BeTrue(), "server was not called") +// }) +// }) diff --git a/controllers/actions.github.com/helpers_test.go b/controllers/actions.github.com/helpers_test.go index 4adbec61..973e724c 100644 --- a/controllers/actions.github.com/helpers_test.go +++ b/controllers/actions.github.com/helpers_test.go @@ -8,6 +8,7 @@ import ( "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -31,16 +32,16 @@ func startManagers(t ginkgo.GinkgoTInterface, first manager.Manager, others ...m } } -func createNamespace(t ginkgo.GinkgoTInterface, client client.Client) (*corev1.Namespace, manager.Manager) { +func createNamespace(t ginkgo.GinkgoTInterface, client client.Client, cfg *rest.Config) (*corev1.Namespace, manager.Manager) { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling" + RandStringRunes(5)}, } - err := k8sClient.Create(context.Background(), ns) + err := client.Create(context.Background(), ns) require.NoError(t, err) t.Cleanup(func() { - err := k8sClient.Delete(context.Background(), ns) + err := client.Delete(context.Background(), ns) require.NoError(t, err) }) @@ -64,7 +65,7 @@ func createDefaultSecret(t ginkgo.GinkgoTInterface, client client.Client, namesp }, } - err := k8sClient.Create(context.Background(), secret) + err := client.Create(context.Background(), secret) require.NoError(t, err) return secret diff --git a/controllers/actions.github.com/suite_test.go b/controllers/actions.github.com/suite_test.go index 80fb4196..27d8c2d4 100644 --- a/controllers/actions.github.com/suite_test.go +++ b/controllers/actions.github.com/suite_test.go @@ -16,73 +16,73 @@ limitations under the License. package actionsgithubcom -import ( - "os" - "path/filepath" - "testing" - - "github.com/onsi/ginkgo/config" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - // +kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var ( - cfg *rest.Config - k8sClient client.Client - testEnv *envtest.Environment -) - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - config.GinkgoConfig.FocusStrings = append(config.GinkgoConfig.FocusStrings, os.Getenv("GINKGO_FOCUS")) - - RunSpecs(t, "Controller Suite") -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("../..", "config", "crd", "bases")}, - } - - // Avoids the following error: - // 2021-03-19T15:14:11.673+0900 ERROR controller-runtime.controller Reconciler error {"controller": "testns-tvjzjrunner", "request": "testns-gdnyx/example-runnerdeploy-zps4z-j5562", "error": "Pod \"example-runnerdeploy-zps4z-j5562\" is invalid: [spec.containers[1].image: Required value, spec.containers[1].securityContext.privileged: Forbidden: disallowed by cluster policy]"} - testEnv.ControlPlane.GetAPIServer().Configure(). - Append("allow-privileged", "true") - - var err error - cfg, err = testEnv.Start() - Expect(err).ToNot(HaveOccurred()) - Expect(cfg).ToNot(BeNil()) - - err = actionsv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - // +kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).ToNot(HaveOccurred()) - Expect(k8sClient).ToNot(BeNil()) -}) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).ToNot(HaveOccurred()) -}) +// import ( +// "os" +// "path/filepath" +// "testing" +// +// "github.com/onsi/ginkgo/config" +// +// . "github.com/onsi/ginkgo/v2" +// . "github.com/onsi/gomega" +// +// actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1" +// "k8s.io/client-go/kubernetes/scheme" +// "k8s.io/client-go/rest" +// "sigs.k8s.io/controller-runtime/pkg/client" +// "sigs.k8s.io/controller-runtime/pkg/envtest" +// logf "sigs.k8s.io/controller-runtime/pkg/log" +// "sigs.k8s.io/controller-runtime/pkg/log/zap" +// // +kubebuilder:scaffold:imports +// ) +// +// // These tests use Ginkgo (BDD-style Go testing framework). Refer to +// // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. +// +// var ( +// cfg *rest.Config +// k8sClient client.Client +// testEnv *envtest.Environment +// ) +// +// func TestAPIs(t *testing.T) { +// RegisterFailHandler(Fail) +// +// config.GinkgoConfig.FocusStrings = append(config.GinkgoConfig.FocusStrings, os.Getenv("GINKGO_FOCUS")) +// +// RunSpecs(t, "Controller Suite") +// } +// +// var _ = BeforeSuite(func() { +// logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter))) +// +// By("bootstrapping test environment") +// testEnv = &envtest.Environment{ +// CRDDirectoryPaths: []string{filepath.Join("../..", "config", "crd", "bases")}, +// } +// +// // Avoids the following error: +// // 2021-03-19T15:14:11.673+0900 ERROR controller-runtime.controller Reconciler error {"controller": "testns-tvjzjrunner", "request": "testns-gdnyx/example-runnerdeploy-zps4z-j5562", "error": "Pod \"example-runnerdeploy-zps4z-j5562\" is invalid: [spec.containers[1].image: Required value, spec.containers[1].securityContext.privileged: Forbidden: disallowed by cluster policy]"} +// testEnv.ControlPlane.GetAPIServer().Configure(). +// Append("allow-privileged", "true") +// +// var err error +// cfg, err = testEnv.Start() +// Expect(err).ToNot(HaveOccurred()) +// Expect(cfg).ToNot(BeNil()) +// +// err = actionsv1alpha1.AddToScheme(scheme.Scheme) +// Expect(err).NotTo(HaveOccurred()) +// +// // +kubebuilder:scaffold:scheme +// +// k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) +// Expect(err).ToNot(HaveOccurred()) +// Expect(k8sClient).ToNot(BeNil()) +// }) +// +// var _ = AfterSuite(func() { +// By("tearing down the test environment") +// err := testEnv.Stop() +// Expect(err).ToNot(HaveOccurred()) +// }) diff --git a/go.mod b/go.mod index 90f6c0e0..9a858b00 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/onsi/gomega v1.27.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 + github.com/rentziass/eventually v0.0.0-20230321190617-a198b1b1faa3 github.com/stretchr/testify v1.8.2 github.com/teambition/rrule-go v1.8.2 go.uber.org/multierr v1.7.0 diff --git a/go.sum b/go.sum index b09cc55a..34cb23a8 100644 --- a/go.sum +++ b/go.sum @@ -333,6 +333,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/rentziass/eventually v0.0.0-20230321190617-a198b1b1faa3 h1:rzJaKZtKN0I3zA9Hq7J93sGaf1K83vik8nmoIHirtQY= +github.com/rentziass/eventually v0.0.0-20230321190617-a198b1b1faa3/go.mod h1:jiSDJFv0sra6DK66duTH5V4kCfqd9OvmFtflPOExNVI= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=