Add support for proxy (#2286)

Co-authored-by: Nikola Jokic <jokicnikola07@gmail.com>
Co-authored-by: Tingluo Huang <tingluohuang@github.com>
Co-authored-by: Ferenc Hammerl <fhammerl@github.com>
This commit is contained in:
Francesco Renzi
2023-02-21 17:33:48 +00:00
committed by GitHub
parent ced88228fc
commit 6b4250ca90
33 changed files with 1795 additions and 98 deletions

View File

@@ -40,6 +40,7 @@ import (
)
const (
autoscalingListenerContainerName = "autoscaler"
autoscalingListenerOwnerKey = ".metadata.controller"
autoscalingListenerFinalizerName = "autoscalinglistener.actions.github.com/finalizer"
)
@@ -202,6 +203,21 @@ func (r *AutoscalingListenerReconciler) Reconcile(ctx context.Context, req ctrl.
return r.createRoleBindingForListener(ctx, autoscalingListener, listenerRole, serviceAccount, log)
}
// Create a secret containing proxy config if specifiec
if autoscalingListener.Spec.Proxy != nil {
proxySecret := new(corev1.Secret)
if err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingListener.Namespace, Name: proxyListenerSecretName(autoscalingListener)}, proxySecret); err != nil {
if !kerrors.IsNotFound(err) {
log.Error(err, "Unable to get listener proxy secret", "namespace", autoscalingListener.Namespace, "name", proxyListenerSecretName(autoscalingListener))
return ctrl.Result{}, err
}
// Create a mirror secret for the listener pod in the Controller namespace for listener pod to use
log.Info("Creating a listener proxy secret for the listener pod")
return r.createProxySecret(ctx, autoscalingListener, log)
}
}
// TODO: make sure the role binding has the up-to-date role and service account
listenerPod := new(corev1.Pod)
@@ -307,6 +323,25 @@ func (r *AutoscalingListenerReconciler) cleanupResources(ctx context.Context, au
}
logger.Info("Listener pod is deleted")
if autoscalingListener.Spec.Proxy != nil {
logger.Info("Cleaning up the listener proxy secret")
proxySecret := new(corev1.Secret)
err = r.Get(ctx, types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingListener.Namespace}, proxySecret)
switch {
case err == nil:
if proxySecret.ObjectMeta.DeletionTimestamp.IsZero() {
logger.Info("Deleting the listener proxy secret")
if err := r.Delete(ctx, proxySecret); err != nil {
return false, fmt.Errorf("failed to delete listener proxy secret: %v", err)
}
}
return false, nil
case err != nil && !kerrors.IsNotFound(err):
return false, fmt.Errorf("failed to get listener proxy secret: %v", err)
}
logger.Info("Listener proxy secret is deleted")
}
logger.Info("Cleaning up the listener service account")
listenerSa := new(corev1.ServiceAccount)
err = r.Get(ctx, types.NamespacedName{Name: scaleSetListenerServiceAccountName(autoscalingListener), Namespace: autoscalingListener.Namespace}, listenerSa)
@@ -345,7 +380,49 @@ func (r *AutoscalingListenerReconciler) createServiceAccountForListener(ctx cont
}
func (r *AutoscalingListenerReconciler) createListenerPod(ctx context.Context, autoscalingRunnerSet *v1alpha1.AutoscalingRunnerSet, autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, secret *corev1.Secret, logger logr.Logger) (ctrl.Result, error) {
newPod := r.resourceBuilder.newScaleSetListenerPod(autoscalingListener, serviceAccount, secret)
var envs []corev1.EnvVar
if autoscalingListener.Spec.Proxy != nil {
httpURL := corev1.EnvVar{
Name: "http_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)},
Key: "http_proxy",
},
},
}
if autoscalingListener.Spec.Proxy.HTTP != nil {
envs = append(envs, httpURL)
}
httpsURL := corev1.EnvVar{
Name: "https_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)},
Key: "https_proxy",
},
},
}
if autoscalingListener.Spec.Proxy.HTTPS != nil {
envs = append(envs, httpsURL)
}
noProxy := corev1.EnvVar{
Name: "no_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)},
Key: "no_proxy",
},
},
}
if len(autoscalingListener.Spec.Proxy.NoProxy) > 0 {
envs = append(envs, noProxy)
}
}
newPod := r.resourceBuilder.newScaleSetListenerPod(autoscalingListener, serviceAccount, secret, envs...)
if err := ctrl.SetControllerReference(autoscalingListener, newPod, r.Scheme); err != nil {
return ctrl.Result{}, err
@@ -378,6 +455,45 @@ func (r *AutoscalingListenerReconciler) createSecretsForListener(ctx context.Con
return ctrl.Result{}, nil
}
func (r *AutoscalingListenerReconciler) createProxySecret(ctx context.Context, autoscalingListener *v1alpha1.AutoscalingListener, logger logr.Logger) (ctrl.Result, error) {
data, err := autoscalingListener.Spec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) {
var secret corev1.Secret
err := r.Get(ctx, types.NamespacedName{Name: s, Namespace: autoscalingListener.Spec.AutoscalingRunnerSetNamespace}, &secret)
if err != nil {
return nil, fmt.Errorf("failed to get secret %s: %w", s, err)
}
return &secret, nil
})
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to convert proxy config to secret data: %w", err)
}
newProxySecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: proxyListenerSecretName(autoscalingListener),
Namespace: autoscalingListener.Namespace,
Labels: map[string]string{
"auto-scaling-runner-set-namespace": autoscalingListener.Spec.AutoscalingRunnerSetNamespace,
"auto-scaling-runner-set-name": autoscalingListener.Spec.AutoscalingRunnerSetName,
},
},
Data: data,
}
if err := ctrl.SetControllerReference(autoscalingListener, newProxySecret, r.Scheme); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to create listener proxy secret: %w", err)
}
logger.Info("Creating listener proxy secret", "namespace", newProxySecret.Namespace, "name", newProxySecret.Name)
if err := r.Create(ctx, newProxySecret); err != nil {
logger.Error(err, "Unable to create listener secret", "namespace", newProxySecret.Namespace, "name", newProxySecret.Name)
return ctrl.Result{}, err
}
logger.Info("Created listener proxy secret", "namespace", newProxySecret.Namespace, "name", newProxySecret.Name)
return ctrl.Result{}, nil
}
func (r *AutoscalingListenerReconciler) updateSecretsForListener(ctx context.Context, secret *corev1.Secret, mirrorSecret *corev1.Secret, logger logr.Logger) (ctrl.Result, error) {
dataHash := hash.ComputeTemplateHash(secret.Data)
updatedMirrorSecret := mirrorSecret.DeepCopy()

View File

@@ -13,7 +13,9 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
)
@@ -222,7 +224,7 @@ var _ = Describe("Test AutoScalingListener controller", func() {
Context("When deleting a new AutoScalingListener", func() {
It("It should cleanup all resources for a deleting AutoScalingListener before removing it", func() {
// Waiting for the pod is created
// Waiting for the pod to be created
pod := new(corev1.Pod)
Eventually(
func() (string, error) {
@@ -391,3 +393,234 @@ var _ = Describe("Test AutoScalingListener controller", func() {
})
})
})
var _ = Describe("Test AutoScalingListener controller with proxy", func() {
var ctx context.Context
var cancel context.CancelFunc
autoscalingNS := new(corev1.Namespace)
autoscalingRunnerSet := new(actionsv1alpha1.AutoscalingRunnerSet)
configSecret := new(corev1.Secret)
autoscalingListener := new(actionsv1alpha1.AutoscalingListener)
createRunnerSetAndListener := func(proxy *actionsv1alpha1.ProxyConfig) {
min := 1
max := 10
autoscalingRunnerSet = &actionsv1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
},
Spec: actionsv1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
Proxy: proxy,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err := k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
autoscalingListener = &actionsv1alpha1.AutoscalingListener{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asl",
Namespace: autoscalingNS.Name,
},
Spec: actionsv1alpha1.AutoscalingListenerSpec{
GitHubConfigUrl: "https://github.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
RunnerScaleSetId: 1,
AutoscalingRunnerSetNamespace: autoscalingRunnerSet.Namespace,
AutoscalingRunnerSetName: autoscalingRunnerSet.Name,
EphemeralRunnerSetName: "test-ers",
MaxRunners: 10,
MinRunners: 1,
Image: "ghcr.io/owner/repo",
Proxy: proxy,
},
}
err = k8sClient.Create(ctx, autoscalingListener)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingListener")
}
BeforeEach(func() {
ctx, cancel = context.WithCancel(context.TODO())
autoscalingNS = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling-listener" + RandStringRunes(5)},
}
err := k8sClient.Create(ctx, autoscalingNS)
Expect(err).NotTo(HaveOccurred(), "failed to create test namespace for AutoScalingRunnerSet")
configSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "github-config-secret",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"github_token": []byte(autoscalingListenerTestGitHubToken),
},
}
err = k8sClient.Create(ctx, configSecret)
Expect(err).NotTo(HaveOccurred(), "failed to create config secret")
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Namespace: autoscalingNS.Name,
MetricsBindAddress: "0",
})
Expect(err).NotTo(HaveOccurred(), "failed to create manager")
controller := &AutoscalingListenerReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
}
err = controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
go func() {
defer GinkgoRecover()
err := mgr.Start(ctx)
Expect(err).NotTo(HaveOccurred(), "failed to start manager")
}()
})
AfterEach(func() {
defer cancel()
err := k8sClient.Delete(ctx, autoscalingNS)
Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace for AutoScalingRunnerSet")
})
It("should create a secret in the listener namespace containing proxy details, use it to populate env vars on the pod and should delete it as part of cleanup", func() {
proxyCredentials := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "proxy-credentials",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"username": []byte("test"),
"password": []byte("password"),
},
}
err := k8sClient.Create(ctx, proxyCredentials)
Expect(err).NotTo(HaveOccurred(), "failed to create proxy credentials secret")
proxy := &actionsv1alpha1.ProxyConfig{
HTTP: &actionsv1alpha1.ProxyServerConfig{
Url: "http://localhost:8080",
CredentialSecretRef: "proxy-credentials",
},
HTTPS: &actionsv1alpha1.ProxyServerConfig{
Url: "https://localhost:8443",
CredentialSecretRef: "proxy-credentials",
},
NoProxy: []string{
"example.com",
"example.org",
},
}
createRunnerSetAndListener(proxy)
var proxySecret corev1.Secret
Eventually(
func(g Gomega) {
err := k8sClient.Get(
ctx,
types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name},
&proxySecret,
)
g.Expect(err).NotTo(HaveOccurred(), "failed to get secret")
expected, err := autoscalingListener.Spec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) {
var secret corev1.Secret
err := k8sClient.Get(ctx, types.NamespacedName{Name: s, Namespace: autoscalingNS.Name}, &secret)
if err != nil {
return nil, err
}
return &secret, nil
})
g.Expect(err).NotTo(HaveOccurred(), "failed to convert proxy config to secret data")
g.Expect(proxySecret.Data).To(Equal(expected))
},
autoscalingRunnerSetTestTimeout,
autoscalingRunnerSetTestInterval,
).Should(Succeed(), "failed to create secret with proxy details")
// wait for listener pod to be created
Eventually(
func(g Gomega) {
pod := new(corev1.Pod)
err := k8sClient.Get(
ctx,
client.ObjectKey{Name: autoscalingListener.Name, Namespace: autoscalingListener.Namespace},
pod,
)
g.Expect(err).NotTo(HaveOccurred(), "failed to get pod")
g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{
Name: "http_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)},
Key: "http_proxy",
},
},
}), "http_proxy environment variable not found")
g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{
Name: "https_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)},
Key: "https_proxy",
},
},
}), "https_proxy environment variable not found")
g.Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{
Name: "no_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: proxyListenerSecretName(autoscalingListener)},
Key: "no_proxy",
},
},
}), "no_proxy environment variable not found")
},
autoscalingListenerTestTimeout,
autoscalingListenerTestInterval).Should(Succeed(), "failed to create listener pod with proxy details")
// Delete the AutoScalingListener
err = k8sClient.Delete(ctx, autoscalingListener)
Expect(err).NotTo(HaveOccurred(), "failed to delete test AutoScalingListener")
Eventually(
func(g Gomega) {
var proxySecret corev1.Secret
err := k8sClient.Get(
ctx,
types.NamespacedName{Name: proxyListenerSecretName(autoscalingListener), Namespace: autoscalingNS.Name},
&proxySecret,
)
g.Expect(kerrors.IsNotFound(err)).To(BeTrue())
},
autoscalingListenerTestTimeout,
autoscalingListenerTestInterval).Should(Succeed(), "failed to delete secret with proxy details")
})
})

View File

@@ -42,7 +42,6 @@ import (
const (
// TODO: Replace with shared image.
name = "autoscaler"
autoscalingRunnerSetOwnerKey = ".metadata.controller"
LabelKeyRunnerSpecHash = "runner-spec-hash"
LabelKeyAutoScaleRunnerSetName = "auto-scale-runner-set-name"
@@ -495,7 +494,31 @@ func (r *AutoscalingRunnerSetReconciler) actionsClientFor(ctx context.Context, a
return nil, fmt.Errorf("failed to find GitHub config secret: %w", err)
}
return r.ActionsClient.GetClientFromSecret(ctx, autoscalingRunnerSet.Spec.GitHubConfigUrl, autoscalingRunnerSet.Namespace, configSecret.Data)
var opts []actions.ClientOption
if autoscalingRunnerSet.Spec.Proxy != nil {
proxyFunc, err := autoscalingRunnerSet.Spec.Proxy.ProxyFunc(func(s string) (*corev1.Secret, error) {
var secret corev1.Secret
err := r.Get(ctx, types.NamespacedName{Namespace: autoscalingRunnerSet.Namespace, Name: s}, &secret)
if err != nil {
return nil, fmt.Errorf("failed to get proxy secret %s: %w", s, err)
}
return &secret, nil
})
if err != nil {
return nil, fmt.Errorf("failed to get proxy func: %w", err)
}
opts = append(opts, actions.WithProxy(proxyFunc))
}
return r.ActionsClient.GetClientFromSecret(
ctx,
autoscalingRunnerSet.Spec.GitHubConfigUrl,
autoscalingRunnerSet.Namespace,
configSecret.Data,
opts...,
)
}
// SetupWithManager sets up the controller with the Manager.

View File

@@ -2,7 +2,11 @@ package actionsgithubcom
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
@@ -11,13 +15,16 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"github.com/go-logr/logr"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/actions/actions-runner-controller/github/actions"
"github.com/actions/actions-runner-controller/github/actions/fake"
"github.com/actions/actions-runner-controller/github/actions/testserver"
)
const (
@@ -570,3 +577,206 @@ var _ = Describe("Test AutoscalingController creation failures", func() {
})
})
})
var _ = Describe("Test Client optional configuration", func() {
Context("When specifying a proxy", func() {
var ctx context.Context
var cancel context.CancelFunc
autoscalingNS := new(corev1.Namespace)
configSecret := new(corev1.Secret)
var mgr ctrl.Manager
BeforeEach(func() {
ctx, cancel = context.WithCancel(context.TODO())
autoscalingNS = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling" + RandStringRunes(5)},
}
err := k8sClient.Create(ctx, autoscalingNS)
Expect(err).NotTo(HaveOccurred(), "failed to create test namespace for AutoScalingRunnerSet")
configSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "github-config-secret",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"github_token": []byte(autoscalingRunnerSetTestGitHubToken),
},
}
err = k8sClient.Create(ctx, configSecret)
Expect(err).NotTo(HaveOccurred(), "failed to create config secret")
mgr, err = ctrl.NewManager(cfg, ctrl.Options{
Namespace: autoscalingNS.Name,
})
Expect(err).NotTo(HaveOccurred(), "failed to create manager")
go func() {
defer GinkgoRecover()
err := mgr.Start(ctx)
Expect(err).NotTo(HaveOccurred(), "failed to start manager")
}()
})
AfterEach(func() {
defer cancel()
err := k8sClient.Delete(ctx, autoscalingNS)
Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace for AutoScalingRunnerSet")
})
It("should be able to make requests to a server using a proxy", func() {
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ActionsClient: actions.NewMultiClient("test", logr.Discard()),
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
serverSuccessfullyCalled := false
proxy := testserver.New(GinkgoT(), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverSuccessfullyCalled = true
w.WriteHeader(http.StatusOK)
}))
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "http://example.com/org/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: proxy.URL,
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for server to be called
Eventually(
func() (bool, error) {
return serverSuccessfullyCalled, nil
},
autoscalingRunnerSetTestTimeout,
1*time.Nanosecond,
).Should(BeTrue(), "server was not called")
})
It("should be able to make requests to a server using a proxy with user info", func() {
controller := &AutoscalingRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ControllerNamespace: autoscalingNS.Name,
DefaultRunnerScaleSetListenerImage: "ghcr.io/actions/arc",
ActionsClient: actions.NewMultiClient("test", logr.Discard()),
}
err := controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
serverSuccessfullyCalled := false
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
header := r.Header.Get("Proxy-Authorization")
Expect(header).NotTo(BeEmpty())
header = strings.TrimPrefix(header, "Basic ")
decoded, err := base64.StdEncoding.DecodeString(header)
Expect(err).NotTo(HaveOccurred())
Expect(string(decoded)).To(Equal("test:password"))
serverSuccessfullyCalled = true
w.WriteHeader(http.StatusOK)
}))
GinkgoT().Cleanup(func() {
proxy.Close()
})
secretCredentials := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "proxy-credentials",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"username": []byte("test"),
"password": []byte("password"),
},
}
err = k8sClient.Create(ctx, secretCredentials)
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
min := 1
max := 10
autoscalingRunnerSet := &v1alpha1.AutoscalingRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
},
Spec: v1alpha1.AutoscalingRunnerSetSpec{
GitHubConfigUrl: "http://example.com/org/repo",
GitHubConfigSecret: configSecret.Name,
MaxRunners: &max,
MinRunners: &min,
RunnerGroup: "testgroup",
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: proxy.URL,
CredentialSecretRef: "proxy-credentials",
},
},
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
}
err = k8sClient.Create(ctx, autoscalingRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create AutoScalingRunnerSet")
// wait for server to be called
Eventually(
func() (bool, error) {
return serverSuccessfullyCalled, nil
},
autoscalingRunnerSetTestTimeout,
1*time.Nanosecond,
).Should(BeTrue(), "server was not called")
})
})
})

View File

@@ -9,3 +9,10 @@ const (
EnvVarRunnerJITConfig = "ACTIONS_RUNNER_INPUT_JITCONFIG"
EnvVarRunnerExtraUserAgent = "GITHUB_ACTIONS_RUNNER_EXTRA_USER_AGENT"
)
// Environment variable names used to set proxy variables for containers
const (
EnvVarHTTPProxy = "http_proxy"
EnvVarHTTPSProxy = "https_proxy"
EnvVarNoProxy = "no_proxy"
)

View File

@@ -557,8 +557,56 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
}
func (r *EphemeralRunnerReconciler) createPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, log logr.Logger) (ctrl.Result, error) {
var envs []corev1.EnvVar
if runner.Spec.ProxySecretRef != "" {
http := corev1.EnvVar{
Name: "http_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: runner.Spec.ProxySecretRef,
},
Key: "http_proxy",
},
},
}
if runner.Spec.Proxy.HTTP != nil {
envs = append(envs, http)
}
https := corev1.EnvVar{
Name: "https_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: runner.Spec.ProxySecretRef,
},
Key: "https_proxy",
},
},
}
if runner.Spec.Proxy.HTTPS != nil {
envs = append(envs, https)
}
noProxy := corev1.EnvVar{
Name: "no_proxy",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: runner.Spec.ProxySecretRef,
},
Key: "no_proxy",
},
},
}
if len(runner.Spec.Proxy.NoProxy) > 0 {
envs = append(envs, noProxy)
}
}
log.Info("Creating new pod for ephemeral runner")
newPod := r.resourceBuilder.newEphemeralRunnerPod(ctx, runner, secret)
newPod := r.resourceBuilder.newEphemeralRunnerPod(ctx, runner, secret, envs...)
if err := ctrl.SetControllerReference(runner, newPod, r.Scheme); err != nil {
log.Error(err, "Failed to set controller reference to a new pod")
@@ -632,7 +680,31 @@ func (r *EphemeralRunnerReconciler) actionsClientFor(ctx context.Context, runner
return nil, fmt.Errorf("failed to get secret: %w", err)
}
return r.ActionsClient.GetClientFromSecret(ctx, runner.Spec.GitHubConfigUrl, runner.Namespace, secret.Data)
var opts []actions.ClientOption
if runner.Spec.Proxy != nil {
proxyFunc, err := runner.Spec.Proxy.ProxyFunc(func(s string) (*corev1.Secret, error) {
var secret corev1.Secret
err := r.Get(ctx, types.NamespacedName{Namespace: runner.Namespace, Name: s}, &secret)
if err != nil {
return nil, fmt.Errorf("failed to get proxy secret %s: %w", s, err)
}
return &secret, nil
})
if err != nil {
return nil, fmt.Errorf("failed to get proxy func: %w", err)
}
opts = append(opts, actions.WithProxy(proxyFunc))
}
return r.ActionsClient.GetClientFromSecret(
ctx,
runner.Spec.GitHubConfigUrl,
runner.Namespace,
secret.Data,
opts...,
)
}
// runnerRegisteredWithService checks if the runner is still registered with the service

View File

@@ -2,12 +2,16 @@ package actionsgithubcom
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"time"
"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/actions/actions-runner-controller/github/actions"
"github.com/go-logr/logr"
"github.com/actions/actions-runner-controller/github/actions/fake"
. "github.com/onsi/ginkgo/v2"
@@ -773,4 +777,185 @@ var _ = Describe("EphemeralRunner", func() {
}, timeout, interval).Should(BeEquivalentTo(corev1.PodSucceeded))
})
})
Describe("Pod proxy config", func() {
var ctx context.Context
var cancel context.CancelFunc
autoScalingNS := new(corev1.Namespace)
configSecret := new(corev1.Secret)
controller := new(EphemeralRunnerReconciler)
BeforeEach(func() {
ctx, cancel = context.WithCancel(context.Background())
autoScalingNS = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "testns-autoscaling-runner" + RandStringRunes(5),
},
}
err := k8sClient.Create(ctx, autoScalingNS)
Expect(err).To(BeNil(), "failed to create test namespace for EphemeralRunner")
configSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "github-config-secret",
Namespace: autoScalingNS.Name,
},
Data: map[string][]byte{
"github_token": []byte(gh_token),
},
}
err = k8sClient.Create(ctx, configSecret)
Expect(err).To(BeNil(), "failed to create config secret")
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Namespace: autoScalingNS.Name,
MetricsBindAddress: "0",
})
Expect(err).To(BeNil(), "failed to create manager")
controller = &EphemeralRunnerReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ActionsClient: fake.NewMultiClient(),
}
err = controller.SetupWithManager(mgr)
Expect(err).To(BeNil(), "failed to setup controller")
go func() {
defer GinkgoRecover()
err := mgr.Start(ctx)
Expect(err).To(BeNil(), "failed to start manager")
}()
})
AfterEach(func() {
defer cancel()
err := k8sClient.Delete(ctx, autoScalingNS)
Expect(err).To(BeNil(), "failed to delete test namespace for EphemeralRunner")
})
It("uses an actions client with proxy transport", func() {
// Use an actual client
controller.ActionsClient = actions.NewMultiClient("test", logr.Discard())
proxySuccessfulllyCalled := false
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
header := r.Header.Get("Proxy-Authorization")
Expect(header).NotTo(BeEmpty())
header = strings.TrimPrefix(header, "Basic ")
decoded, err := base64.StdEncoding.DecodeString(header)
Expect(err).NotTo(HaveOccurred())
Expect(string(decoded)).To(Equal("test:password"))
proxySuccessfulllyCalled = true
w.WriteHeader(http.StatusOK)
}))
GinkgoT().Cleanup(func() {
proxy.Close()
})
secretCredentials := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "proxy-credentials",
Namespace: autoScalingNS.Name,
},
Data: map[string][]byte{
"username": []byte("test"),
"password": []byte("password"),
},
}
err := k8sClient.Create(ctx, secretCredentials)
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name)
ephemeralRunner.Spec.GitHubConfigUrl = "http://example.com/org/repo"
ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: proxy.URL,
CredentialSecretRef: "proxy-credentials",
},
}
err = k8sClient.Create(ctx, ephemeralRunner)
Expect(err).To(BeNil(), "failed to create ephemeral runner")
Eventually(
func() bool {
return proxySuccessfulllyCalled
},
2*time.Second,
interval,
).Should(BeEquivalentTo(true))
})
It("It should create EphemeralRunner with proxy environment variables using ProxySecretRef", func() {
ephemeralRunner := newExampleRunner("test-runner", autoScalingNS.Name, configSecret.Name)
ephemeralRunner.Spec.Proxy = &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: "http://proxy.example.com:8080",
},
HTTPS: &v1alpha1.ProxyServerConfig{
Url: "http://proxy.example.com:8080",
},
NoProxy: []string{"example.com"},
}
ephemeralRunner.Spec.ProxySecretRef = "proxy-secret"
err := k8sClient.Create(ctx, ephemeralRunner)
Expect(err).To(BeNil(), "failed to create ephemeral runner")
pod := new(corev1.Pod)
Eventually(
func(g Gomega) {
err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunner.Name, Namespace: ephemeralRunner.Namespace}, pod)
g.Expect(err).To(BeNil(), "failed to get ephemeral runner pod")
},
timeout,
interval,
).Should(Succeed(), "failed to get ephemeral runner pod")
Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{
Name: EnvVarHTTPProxy,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: ephemeralRunner.Spec.ProxySecretRef,
},
Key: "http_proxy",
},
},
}))
Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{
Name: EnvVarHTTPSProxy,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: ephemeralRunner.Spec.ProxySecretRef,
},
Key: "https_proxy",
},
},
}))
Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{
Name: EnvVarNoProxy,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: ephemeralRunner.Spec.ProxySecretRef,
},
Key: "no_proxy",
},
},
}))
})
})
})

View File

@@ -122,6 +122,24 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, nil
}
// Create proxy secret if not present
if ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy != nil {
proxySecret := new(corev1.Secret)
if err := r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunnerSet.Namespace, Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)}, proxySecret); err != nil {
if !kerrors.IsNotFound(err) {
log.Error(err, "Unable to get ephemeralRunnerSet proxy secret", "namespace", ephemeralRunnerSet.Namespace, "name", proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet))
return ctrl.Result{}, err
}
// Create a compiled secret for the runner pods in the runnerset namespace
log.Info("Creating a ephemeralRunnerSet proxy secret for the runner pods")
if err := r.createProxySecret(ctx, ephemeralRunnerSet, log); err != nil {
log.Error(err, "Unable to create ephemeralRunnerSet proxy secret", "namespace", ephemeralRunnerSet.Namespace, "set-name", ephemeralRunnerSet.Name)
return ctrl.Result{}, err
}
}
}
// Find all EphemeralRunner with matching namespace and own by this EphemeralRunnerSet.
ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList)
err := r.List(
@@ -196,15 +214,39 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
return ctrl.Result{}, nil
}
func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (done bool, err error) {
func (r *EphemeralRunnerSetReconciler) cleanUpProxySecret(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) error {
if ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy == nil {
return nil
}
log.Info("Deleting proxy secret")
proxySecret := new(corev1.Secret)
proxySecret.Namespace = ephemeralRunnerSet.Namespace
proxySecret.Name = proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)
if err := r.Delete(ctx, proxySecret); err != nil && !kerrors.IsNotFound(err) {
return fmt.Errorf("failed to delete proxy secret: %v", err)
}
log.Info("Deleted proxy secret")
return nil
}
func (r *EphemeralRunnerSetReconciler) cleanUpEphemeralRunners(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) (bool, error) {
ephemeralRunnerList := new(v1alpha1.EphemeralRunnerList)
err = r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{ephemeralRunnerSetReconcilerOwnerKey: ephemeralRunnerSet.Name})
err := r.List(ctx, ephemeralRunnerList, client.InNamespace(ephemeralRunnerSet.Namespace), client.MatchingFields{ephemeralRunnerSetReconcilerOwnerKey: ephemeralRunnerSet.Name})
if err != nil {
return false, fmt.Errorf("failed to list child ephemeral runners: %v", err)
}
log.Info("Actual Ephemeral runner counts", "count", len(ephemeralRunnerList.Items))
// only if there are no ephemeral runners left, return true
if len(ephemeralRunnerList.Items) == 0 {
err := r.cleanUpProxySecret(ctx, ephemeralRunnerSet, log)
if err != nil {
return false, err
}
log.Info("All ephemeral runners are deleted")
return true, nil
}
@@ -269,6 +311,9 @@ func (r *EphemeralRunnerSetReconciler) createEphemeralRunners(ctx context.Contex
errs := make([]error, 0)
for i := 0; i < count; i++ {
ephemeralRunner := r.resourceBuilder.newEphemeralRunner(runnerSet)
if runnerSet.Spec.EphemeralRunnerSpec.Proxy != nil {
ephemeralRunner.Spec.ProxySecretRef = proxyEphemeralRunnerSetSecretName(runnerSet)
}
// Make sure that we own the resource we create.
if err := ctrl.SetControllerReference(runnerSet, ephemeralRunner, r.Scheme); err != nil {
@@ -290,6 +335,45 @@ func (r *EphemeralRunnerSetReconciler) createEphemeralRunners(ctx context.Contex
return multierr.Combine(errs...)
}
func (r *EphemeralRunnerSetReconciler) createProxySecret(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, log logr.Logger) error {
proxySecretData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(func(s string) (*corev1.Secret, error) {
secret := new(corev1.Secret)
err := r.Get(ctx, types.NamespacedName{Namespace: ephemeralRunnerSet.Namespace, Name: s}, secret)
return secret, err
})
if err != nil {
return fmt.Errorf("failed to convert proxy config to secret data: %w", err)
}
runnerPodProxySecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet),
Namespace: ephemeralRunnerSet.Namespace,
Labels: map[string]string{
// TODO: figure out autoScalingRunnerSet name and set it as a label for this secret
// "auto-scaling-runner-set-namespace": ephemeralRunnerSet.Namespace,
// "auto-scaling-runner-set-name": ephemeralRunnerSet.Name,
},
},
Data: proxySecretData,
}
// Make sure that we own the resource we create.
if err := ctrl.SetControllerReference(ephemeralRunnerSet, runnerPodProxySecret, r.Scheme); err != nil {
log.Error(err, "failed to set controller reference on proxy secret")
return err
}
log.Info("Creating new proxy secret")
if err := r.Create(ctx, runnerPodProxySecret); err != nil {
log.Error(err, "failed to create proxy secret")
return err
}
log.Info("Created new proxy secret")
return nil
}
// deleteIdleEphemeralRunners try to deletes `count` number of v1alpha1.EphemeralRunner resources in the cluster.
// It will only delete `v1alpha1.EphemeralRunner` that has registered with Actions service
// which has a `v1alpha1.EphemeralRunner.Status.RunnerId` set.
@@ -366,8 +450,31 @@ func (r *EphemeralRunnerSetReconciler) actionsClientFor(ctx context.Context, rs
if err := r.Get(ctx, types.NamespacedName{Namespace: rs.Namespace, Name: rs.Spec.EphemeralRunnerSpec.GitHubConfigSecret}, secret); err != nil {
return nil, fmt.Errorf("failed to get secret: %w", err)
}
var opts []actions.ClientOption
if rs.Spec.EphemeralRunnerSpec.Proxy != nil {
proxyFunc, err := rs.Spec.EphemeralRunnerSpec.Proxy.ProxyFunc(func(s string) (*corev1.Secret, error) {
var secret corev1.Secret
err := r.Get(ctx, types.NamespacedName{Namespace: rs.Namespace, Name: s}, &secret)
if err != nil {
return nil, fmt.Errorf("failed to get secret %s: %w", s, err)
}
return r.ActionsClient.GetClientFromSecret(ctx, rs.Spec.EphemeralRunnerSpec.GitHubConfigUrl, rs.Namespace, secret.Data)
return &secret, nil
})
if err != nil {
return nil, fmt.Errorf("failed to get proxy func: %w", err)
}
opts = append(opts, actions.WithProxy(proxyFunc))
}
return r.ActionsClient.GetClientFromSecret(
ctx,
rs.Spec.EphemeralRunnerSpec.GitHubConfigUrl,
rs.Namespace,
secret.Data,
opts...,
)
}
// SetupWithManager sets up the controller with the Manager.

View File

@@ -2,7 +2,11 @@ package actionsgithubcom
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
@@ -11,11 +15,14 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"github.com/go-logr/logr"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
actionsv1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
"github.com/actions/actions-runner-controller/github/actions"
"github.com/actions/actions-runner-controller/github/actions/fake"
)
@@ -585,3 +592,315 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() {
})
})
})
var _ = Describe("Test EphemeralRunnerSet controller with proxy settings", func() {
var ctx context.Context
var cancel context.CancelFunc
autoscalingNS := new(corev1.Namespace)
ephemeralRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet)
configSecret := new(corev1.Secret)
BeforeEach(func() {
ctx, cancel = context.WithCancel(context.TODO())
autoscalingNS = &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "testns-autoscaling-runnerset" + RandStringRunes(5)},
}
err := k8sClient.Create(ctx, autoscalingNS)
Expect(err).NotTo(HaveOccurred(), "failed to create test namespace for EphemeralRunnerSet")
configSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "github-config-secret",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"github_token": []byte(ephemeralRunnerSetTestGitHubToken),
},
}
err = k8sClient.Create(ctx, configSecret)
Expect(err).NotTo(HaveOccurred(), "failed to create config secret")
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Namespace: autoscalingNS.Name,
MetricsBindAddress: "0",
})
Expect(err).NotTo(HaveOccurred(), "failed to create manager")
controller := &EphemeralRunnerSetReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Log: logf.Log,
ActionsClient: actions.NewMultiClient("test", logr.Discard()),
}
err = controller.SetupWithManager(mgr)
Expect(err).NotTo(HaveOccurred(), "failed to setup controller")
go func() {
defer GinkgoRecover()
err := mgr.Start(ctx)
Expect(err).NotTo(HaveOccurred(), "failed to start manager")
}()
})
AfterEach(func() {
defer cancel()
err := k8sClient.Delete(ctx, autoscalingNS)
Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace for EphemeralRunnerSet")
})
It("should create a proxy secret and delete the proxy secreat after the runner-set is deleted", func() {
secretCredentials := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "proxy-credentials",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"username": []byte("username"),
"password": []byte("password"),
},
}
err := k8sClient.Create(ctx, secretCredentials)
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
},
Spec: actionsv1alpha1.EphemeralRunnerSetSpec{
Replicas: 1,
EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{
GitHubConfigUrl: "http://example.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
RunnerScaleSetId: 100,
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: "http://proxy.example.com",
CredentialSecretRef: secretCredentials.Name,
},
HTTPS: &v1alpha1.ProxyServerConfig{
Url: "https://proxy.example.com",
CredentialSecretRef: secretCredentials.Name,
},
NoProxy: []string{"example.com", "example.org"},
},
PodTemplateSpec: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
},
}
err = k8sClient.Create(ctx, ephemeralRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet")
Eventually(func(g Gomega) {
// Compiled / flattened proxy secret should exist at this point
actualProxySecret := &corev1.Secret{}
err = k8sClient.Get(ctx, client.ObjectKey{
Namespace: autoscalingNS.Name,
Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet),
}, actualProxySecret)
g.Expect(err).NotTo(HaveOccurred(), "failed to get compiled / flattened proxy secret")
secretFetcher := func(name string) (*corev1.Secret, error) {
secret := &corev1.Secret{}
err = k8sClient.Get(ctx, client.ObjectKey{
Namespace: autoscalingNS.Name,
Name: name,
}, secret)
return secret, err
}
// Assert that the proxy secret is created with the correct values
expectedData, err := ephemeralRunnerSet.Spec.EphemeralRunnerSpec.Proxy.ToSecretData(secretFetcher)
g.Expect(err).NotTo(HaveOccurred(), "failed to get proxy secret data")
g.Expect(actualProxySecret.Data).To(Equal(expectedData))
},
ephemeralRunnerSetTestTimeout,
ephemeralRunnerSetTestInterval,
).Should(Succeed(), "compiled / flattened proxy secret should exist")
Eventually(func(g Gomega) {
runnerList := new(actionsv1alpha1.EphemeralRunnerList)
err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
g.Expect(err).NotTo(HaveOccurred(), "failed to list EphemeralRunners")
for _, runner := range runnerList.Items {
g.Expect(runner.Spec.ProxySecretRef).To(Equal(proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet)))
}
}, ephemeralRunnerSetTestTimeout, ephemeralRunnerSetTestInterval).Should(Succeed(), "EphemeralRunners should have a reference to the proxy secret")
// patch ephemeral runner set to have 0 replicas
patch := client.MergeFrom(ephemeralRunnerSet.DeepCopy())
ephemeralRunnerSet.Spec.Replicas = 0
err = k8sClient.Patch(ctx, ephemeralRunnerSet, patch)
Expect(err).NotTo(HaveOccurred(), "failed to patch EphemeralRunnerSet")
// Set pods to PodSucceeded to simulate an actual EphemeralRunner stopping
Eventually(
func(g Gomega) (int, error) {
runnerList := new(actionsv1alpha1.EphemeralRunnerList)
err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
if err != nil {
return -1, err
}
// Set status to simulate a configured EphemeralRunner
refetch := false
for i, runner := range runnerList.Items {
if runner.Status.RunnerId == 0 {
updatedRunner := runner.DeepCopy()
updatedRunner.Status.Phase = corev1.PodSucceeded
updatedRunner.Status.RunnerId = i + 100
err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner))
Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
refetch = true
}
}
if refetch {
err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
if err != nil {
return -1, err
}
}
return len(runnerList.Items), nil
},
ephemeralRunnerSetTestTimeout,
ephemeralRunnerSetTestInterval).Should(BeEquivalentTo(1), "1 EphemeralRunner should exist")
// Delete the EphemeralRunnerSet
err = k8sClient.Delete(ctx, ephemeralRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunnerSet")
// Assert that the proxy secret is deleted
Eventually(func(g Gomega) {
proxySecret := &corev1.Secret{}
err = k8sClient.Get(ctx, client.ObjectKey{
Namespace: autoscalingNS.Name,
Name: proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet),
}, proxySecret)
g.Expect(err).To(HaveOccurred(), "proxy secret should be deleted")
g.Expect(kerrors.IsNotFound(err)).To(BeTrue(), "proxy secret should be deleted")
},
ephemeralRunnerSetTestTimeout,
ephemeralRunnerSetTestInterval,
).Should(Succeed(), "proxy secret should be deleted")
})
It("should configure the actions client to use proxy details", func() {
secretCredentials := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "proxy-credentials",
Namespace: autoscalingNS.Name,
},
Data: map[string][]byte{
"username": []byte("test"),
"password": []byte("password"),
},
}
err := k8sClient.Create(ctx, secretCredentials)
Expect(err).NotTo(HaveOccurred(), "failed to create secret credentials")
proxySuccessfulllyCalled := false
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
header := r.Header.Get("Proxy-Authorization")
Expect(header).NotTo(BeEmpty())
header = strings.TrimPrefix(header, "Basic ")
decoded, err := base64.StdEncoding.DecodeString(header)
Expect(err).NotTo(HaveOccurred())
Expect(string(decoded)).To(Equal("test:password"))
proxySuccessfulllyCalled = true
w.WriteHeader(http.StatusOK)
}))
GinkgoT().Cleanup(func() {
proxy.Close()
})
ephemeralRunnerSet = &actionsv1alpha1.EphemeralRunnerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-asrs",
Namespace: autoscalingNS.Name,
},
Spec: actionsv1alpha1.EphemeralRunnerSetSpec{
Replicas: 1,
EphemeralRunnerSpec: actionsv1alpha1.EphemeralRunnerSpec{
GitHubConfigUrl: "http://example.com/owner/repo",
GitHubConfigSecret: configSecret.Name,
RunnerScaleSetId: 100,
Proxy: &v1alpha1.ProxyConfig{
HTTP: &v1alpha1.ProxyServerConfig{
Url: proxy.URL,
CredentialSecretRef: "proxy-credentials",
},
},
PodTemplateSpec: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "runner",
Image: "ghcr.io/actions/runner",
},
},
},
},
},
},
}
err = k8sClient.Create(ctx, ephemeralRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to create EphemeralRunnerSet")
runnerList := new(actionsv1alpha1.EphemeralRunnerList)
Eventually(func() (int, error) {
err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
if err != nil {
return -1, err
}
return len(runnerList.Items), nil
},
ephemeralRunnerSetTestTimeout,
ephemeralRunnerSetTestInterval,
).Should(BeEquivalentTo(1), "failed to create ephemeral runner")
runner := runnerList.Items[0].DeepCopy()
runner.Status.Phase = corev1.PodRunning
runner.Status.RunnerId = 100
err = k8sClient.Status().Patch(ctx, runner, client.MergeFrom(&runnerList.Items[0]))
Expect(err).NotTo(HaveOccurred(), "failed to update ephemeral runner status")
updatedRunnerSet := new(actionsv1alpha1.EphemeralRunnerSet)
err = k8sClient.Get(ctx, client.ObjectKey{Namespace: ephemeralRunnerSet.Namespace, Name: ephemeralRunnerSet.Name}, updatedRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
updatedRunnerSet.Spec.Replicas = 0
err = k8sClient.Update(ctx, updatedRunnerSet)
Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
Eventually(
func() bool {
return proxySuccessfulllyCalled
},
2*time.Second,
interval,
).Should(BeEquivalentTo(true))
})
})

View File

@@ -18,10 +18,9 @@ const (
jitTokenKey = "jitToken"
)
type resourceBuilder struct {
}
type resourceBuilder struct{}
func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, secret *corev1.Secret) *corev1.Pod {
func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.AutoscalingListener, serviceAccount *corev1.ServiceAccount, secret *corev1.Secret, envs ...corev1.EnvVar) *corev1.Pod {
newLabels := map[string]string{}
newLabels[scaleSetListenerLabel] = fmt.Sprintf("%v-%v", autoscalingListener.Spec.AutoscalingRunnerSetNamespace, autoscalingListener.Spec.AutoscalingRunnerSetName)
@@ -51,6 +50,7 @@ func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A
Value: strconv.Itoa(autoscalingListener.Spec.RunnerScaleSetId),
},
}
listenerEnv = append(listenerEnv, envs...)
if _, ok := secret.Data["github_token"]; ok {
listenerEnv = append(listenerEnv, corev1.EnvVar{
@@ -112,7 +112,7 @@ func (b *resourceBuilder) newScaleSetListenerPod(autoscalingListener *v1alpha1.A
ServiceAccountName: serviceAccount.Name,
Containers: []corev1.Container{
{
Name: name,
Name: autoscalingListenerContainerName,
Image: autoscalingListener.Spec.Image,
Env: listenerEnv,
ImagePullPolicy: corev1.PullIfNotPresent,
@@ -299,6 +299,7 @@ func (b *resourceBuilder) newAutoScalingListener(autoscalingRunnerSet *v1alpha1.
MaxRunners: effectiveMaxRunners,
Image: image,
ImagePullSecrets: imagePullSecrets,
Proxy: autoscalingRunnerSet.Spec.Proxy,
},
}
@@ -316,7 +317,7 @@ func (b *resourceBuilder) newEphemeralRunner(ephemeralRunnerSet *v1alpha1.Epheme
}
}
func (b *resourceBuilder) newEphemeralRunnerPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret) *corev1.Pod {
func (b *resourceBuilder) newEphemeralRunnerPod(ctx context.Context, runner *v1alpha1.EphemeralRunner, secret *corev1.Secret, envs ...corev1.EnvVar) *corev1.Pod {
var newPod corev1.Pod
labels := map[string]string{}
@@ -374,7 +375,9 @@ func (b *resourceBuilder) newEphemeralRunnerPod(ctx context.Context, runner *v1a
corev1.EnvVar{
Name: EnvVarRunnerExtraUserAgent,
Value: fmt.Sprintf("actions-runner-controller/%s", build.Version),
})
},
)
c.Env = append(c.Env, envs...)
}
newPod.Spec.Containers = append(newPod.Spec.Containers, c)
@@ -427,6 +430,22 @@ func scaleSetListenerSecretMirrorName(autoscalingListener *v1alpha1.AutoscalingL
return fmt.Sprintf("%v-%v-listener", autoscalingListener.Spec.AutoscalingRunnerSetName, namespaceHash)
}
func proxyListenerSecretName(autoscalingListener *v1alpha1.AutoscalingListener) string {
namespaceHash := hash.FNVHashString(autoscalingListener.Spec.AutoscalingRunnerSetNamespace)
if len(namespaceHash) > 8 {
namespaceHash = namespaceHash[:8]
}
return fmt.Sprintf("%v-%v-listener-proxy", autoscalingListener.Spec.AutoscalingRunnerSetName, namespaceHash)
}
func proxyEphemeralRunnerSetSecretName(ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet) string {
namespaceHash := hash.FNVHashString(ephemeralRunnerSet.Namespace)
if len(namespaceHash) > 8 {
namespaceHash = namespaceHash[:8]
}
return fmt.Sprintf("%v-%v-runner-proxy", ephemeralRunnerSet.Name, namespaceHash)
}
func rulesForListenerRole(resourceNames []string) []rbacv1.PolicyRule {
return []rbacv1.PolicyRule{
{