Prepare 0.9.1 release (#3448)

Co-authored-by: Francesco Renzi <rentziass@gmail.com>

.github/workflows/gha-e2e-tests.yaml

@@ -16,7 +16,7 @@ env:
   TARGET_ORG: actions-runner-controller
   TARGET_REPO: arc_e2e_test_dummy
   IMAGE_NAME: "arc-test-image"
-  IMAGE_VERSION: "0.9.0"
+  IMAGE_VERSION: "0.9.1"
 
 concurrency:
   # This will make sure we only apply the concurrency limits on pull requests

Makefile

@@ -6,7 +6,7 @@ endif
 DOCKER_USER ?= $(shell echo ${DOCKER_IMAGE_NAME} | cut -d / -f1)
 VERSION ?= dev
 COMMIT_SHA = $(shell git rev-parse HEAD)
-RUNNER_VERSION ?= 2.314.1
+RUNNER_VERSION ?= 2.315.0
 TARGETPLATFORM ?= $(shell arch)
 RUNNER_NAME ?= ${DOCKER_USER}/actions-runner
 RUNNER_TAG  ?= ${VERSION}

charts/gha-runner-scale-set-controller/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.0
+version: 0.9.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.9.0"
+appVersion: "0.9.1"
 
 home: https://github.com/actions/actions-runner-controller
 

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -128,6 +128,10 @@ spec:
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}

charts/gha-runner-scale-set-controller/tests/template_test.go

@@ -345,6 +345,7 @@ func TestTemplate_ControllerDeployment_Defaults(t *testing.T) {
 
 	assert.Len(t, deployment.Spec.Template.Spec.NodeSelector, 0)
 	assert.Nil(t, deployment.Spec.Template.Spec.Affinity)
+	assert.Len(t, deployment.Spec.Template.Spec.TopologySpreadConstraints, 0)
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 0)
 
 	managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev"
@@ -424,6 +425,9 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) {
 			"tolerations[0].key":           "foo",
 			"affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key":      "foo",
 			"affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator": "bar",
+			"topologySpreadConstraints[0].labelSelector.matchLabels.foo":                                                             "bar",
+			"topologySpreadConstraints[0].maxSkew":                                                                                   "1",
+			"topologySpreadConstraints[0].topologyKey":                                                                               "foo",
 			"priorityClassName":         "test-priority-class",
 			"flags.updateStrategy":      "eventual",
 			"flags.logLevel":            "info",
@@ -487,6 +491,11 @@ func TestTemplate_ControllerDeployment_Customize(t *testing.T) {
 	assert.Equal(t, "foo", deployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key)
 	assert.Equal(t, "bar", string(deployment.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Operator))
 
+	assert.Len(t, deployment.Spec.Template.Spec.TopologySpreadConstraints, 1)
+	assert.Equal(t, "bar", deployment.Spec.Template.Spec.TopologySpreadConstraints[0].LabelSelector.MatchLabels["foo"])
+	assert.Equal(t, int32(1), deployment.Spec.Template.Spec.TopologySpreadConstraints[0].MaxSkew)
+	assert.Equal(t, "foo", deployment.Spec.Template.Spec.TopologySpreadConstraints[0].TopologyKey)
+
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 1)
 	assert.Equal(t, "foo", deployment.Spec.Template.Spec.Tolerations[0].Key)
 
@@ -745,6 +754,7 @@ func TestTemplate_ControllerDeployment_WatchSingleNamespace(t *testing.T) {
 
 	assert.Len(t, deployment.Spec.Template.Spec.NodeSelector, 0)
 	assert.Nil(t, deployment.Spec.Template.Spec.Affinity)
+	assert.Len(t, deployment.Spec.Template.Spec.TopologySpreadConstraints, 0)
 	assert.Len(t, deployment.Spec.Template.Spec.Tolerations, 0)
 
 	managerImage := "ghcr.io/actions/gha-runner-scale-set-controller:dev"

charts/gha-runner-scale-set-controller/values.yaml

@@ -72,6 +72,8 @@ tolerations: []
 
 affinity: {}
 
+topologySpreadConstraints: []
+
 # Mount volumes in the container.
 volumes: []
 volumeMounts: []

charts/gha-runner-scale-set/Chart.yaml

@@ -15,13 +15,13 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.0
+version: 0.9.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.9.0"
+appVersion: "0.9.1"
 
 home: https://github.com/actions/actions-runner-controller
 

charts/gha-runner-scale-set/values.yaml

@@ -88,7 +88,7 @@ githubConfigSecret:
 #   kubernetesModeServiceAccount:
 #     annotations:
 
-## template is the PodSpec for each listener Pod
+## listenerTemplate is the PodSpec for each listener Pod
 ## For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
 # listenerTemplate:
 #   spec:

cmd/ghalistener/app/app.go

@@ -117,15 +117,19 @@ func (app *App) Run(ctx context.Context) error {
 	}
 
 	g, ctx := errgroup.WithContext(ctx)
+	metricsCtx, cancelMetrics := context.WithCancelCause(ctx)
+
 	g.Go(func() error {
 		app.logger.Info("Starting listener")
-		return app.listener.Listen(ctx, app.worker)
+		listnerErr := app.listener.Listen(ctx, app.worker)
+		cancelMetrics(fmt.Errorf("Listener exited: %w", listnerErr))
+		return listnerErr
 	})
 
 	if app.metrics != nil {
 		g.Go(func() error {
 			app.logger.Info("Starting metrics server")
-			return app.metrics.ListenAndServe(ctx)
+			return app.metrics.ListenAndServe(metricsCtx)
 		})
 	}
 

cmd/ghalistener/listener/listener.go

@@ -31,7 +31,7 @@ const (
 type Client interface {
 	GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (*actions.AcquirableJobList, error)
 	CreateMessageSession(ctx context.Context, runnerScaleSetId int, owner string) (*actions.RunnerScaleSetSession, error)
-	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*actions.RunnerScaleSetMessage, error)
+	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error)
 	DeleteMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, messageId int64) error
 	AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQueueAccessToken string, requestIds []int64) ([]int64, error)
 	RefreshMessageSession(ctx context.Context, runnerScaleSetId int, sessionId *uuid.UUID) (*actions.RunnerScaleSetSession, error)
@@ -80,6 +80,7 @@ type Listener struct {
 
 	// updated fields
 	lastMessageID int64                          // The ID of the last processed message.
+	maxCapacity   int                            // The maximum number of runners that can be created.
 	session       *actions.RunnerScaleSetSession // The session for managing the runner scale set.
 }
 
@@ -93,6 +94,7 @@ func New(config Config) (*Listener, error) {
 		client:      config.Client,
 		logger:      config.Logger,
 		metrics:     metrics.Discard,
+		maxCapacity: config.MaxRunners,
 	}
 
 	if config.Metrics != nil {
@@ -164,11 +166,16 @@ func (l *Listener) Listen(ctx context.Context, handler Handler) error {
 		}
 
 		if msg == nil {
+			_, err := handler.HandleDesiredRunnerCount(ctx, 0, 0)
+			if err != nil {
+				return fmt.Errorf("handling nil message failed: %w", err)
+			}
+
 			continue
 		}
 
-		// New context is created to avoid cancelation during message handling.
-		if err := l.handleMessage(context.Background(), handler, msg); err != nil {
+		// Remove cancellation from the context to avoid cancelling the message handling.
+		if err := l.handleMessage(context.WithoutCancel(ctx), handler, msg); err != nil {
 			return fmt.Errorf("failed to handle message: %w", err)
 		}
 	}
@@ -262,7 +269,7 @@ func (l *Listener) createSession(ctx context.Context) error {
 
 func (l *Listener) getMessage(ctx context.Context) (*actions.RunnerScaleSetMessage, error) {
 	l.logger.Info("Getting next message", "lastMessageID", l.lastMessageID)
-	msg, err := l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID)
+	msg, err := l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID, l.maxCapacity)
 	if err == nil { // if NO error
 		return msg, nil
 	}
@@ -278,7 +285,7 @@ func (l *Listener) getMessage(ctx context.Context) (*actions.RunnerScaleSetMessa
 
 	l.logger.Info("Getting next message", "lastMessageID", l.lastMessageID)
 
-	msg, err = l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID)
+	msg, err = l.client.GetMessage(ctx, l.session.MessageQueueUrl, l.session.MessageQueueAccessToken, l.lastMessageID, l.maxCapacity)
 	if err != nil { // if NO error
 		return nil, fmt.Errorf("failed to get next message after message session refresh: %w", err)
 	}

cmd/ghalistener/listener/listener_test.go

@@ -123,13 +123,14 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
 		want := &actions.RunnerScaleSetMessage{
 			MessageId: 1,
 		}
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(want, nil).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(want, nil).Once()
 		config.Client = client
 
 		l, err := New(config)
@@ -148,10 +149,11 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, &actions.HttpClientSideError{Code: http.StatusNotFound}).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(nil, &actions.HttpClientSideError{Code: http.StatusNotFound}).Once()
 		config.Client = client
 
 		l, err := New(config)
@@ -170,6 +172,7 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
@@ -185,12 +188,12 @@ func TestListener_getMessage(t *testing.T) {
 		}
 		client.On("RefreshMessageSession", ctx, mock.Anything, mock.Anything).Return(session, nil).Once()
 
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
 
 		want := &actions.RunnerScaleSetMessage{
 			MessageId: 1,
 		}
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(want, nil).Once()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(want, nil).Once()
 
 		config.Client = client
 
@@ -214,6 +217,7 @@ func TestListener_getMessage(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
@@ -229,7 +233,7 @@ func TestListener_getMessage(t *testing.T) {
 		}
 		client.On("RefreshMessageSession", ctx, mock.Anything, mock.Anything).Return(session, nil).Once()
 
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, &actions.MessageQueueTokenExpiredError{}).Twice()
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Twice()
 
 		config.Client = client
 
@@ -450,6 +454,7 @@ func TestListener_Listen(t *testing.T) {
 		config := Config{
 			ScaleSetID: 1,
 			Metrics:    metrics.Discard,
+			MaxRunners: 10,
 		}
 
 		client := listenermocks.NewClient(t)
@@ -470,7 +475,7 @@ func TestListener_Listen(t *testing.T) {
 			MessageType: "RunnerScaleSetJobMessages",
 			Statistics:  &actions.RunnerScaleSetStatistic{},
 		}
-		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything).
+		client.On("GetMessage", ctx, mock.Anything, mock.Anything, mock.Anything, 10).
 			Return(msg, nil).
 			Run(
 				func(mock.Arguments) {
@@ -479,8 +484,8 @@ func TestListener_Listen(t *testing.T) {
 			).
 			Once()
 
-			// Ensure delete message is called with background context
-		client.On("DeleteMessage", context.Background(), mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
+		// Ensure delete message is called without cancel
+		client.On("DeleteMessage", context.WithoutCancel(ctx), mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
 
 		config.Client = client
 

cmd/ghalistener/listener/mocks/client.go

@@ -123,25 +123,25 @@ func (_m *Client) GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (
 	return r0, r1
 }
 
-// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId
-func (_m *Client) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64) (*actions.RunnerScaleSetMessage, error) {
-	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity
+func (_m *Client) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error) {
+	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 
 	var r0 *actions.RunnerScaleSetMessage
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) (*actions.RunnerScaleSetMessage, error)); ok {
-		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) (*actions.RunnerScaleSetMessage, error)); ok {
+		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) *actions.RunnerScaleSetMessage); ok {
-		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) *actions.RunnerScaleSetMessage); ok {
+		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*actions.RunnerScaleSetMessage)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64) error); ok {
-		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int) error); ok {
+		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		r1 = ret.Error(1)
 	}

cmd/ghalistener/metrics/metrics.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/actions/actions-runner-controller/github/actions"
 	"github.com/go-logr/logr"
@@ -338,7 +339,9 @@ func (e *exporter) ListenAndServe(ctx context.Context) error {
 	e.logger.Info("starting metrics server", "addr", e.srv.Addr)
 	go func() {
 		<-ctx.Done()
-		e.logger.Info("stopping metrics server")
+		e.logger.Info("stopping metrics server", "err", ctx.Err())
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
 		e.srv.Shutdown(ctx)
 	}()
 	return e.srv.ListenAndServe()

cmd/ghalistener/worker/worker.go

@@ -177,12 +177,12 @@ func (w *Worker) HandleDesiredRunnerCount(ctx context.Context, count int, jobsCo
 		"jobsCompleted", jobsCompleted,
 	}
 
-	if w.lastPatch == targetRunnerCount && jobsCompleted == 0 {
-		w.logger.Info("Skipping patch", logValues...)
-		return targetRunnerCount, nil
+	if count == 0 && jobsCompleted == 0 {
+		w.lastPatchID = 0
+	} else {
+		w.lastPatchID++
 	}
 
-	w.lastPatchID++
 	w.lastPatch = targetRunnerCount
 
 	original, err := json.Marshal(

cmd/githubrunnerscalesetlistener/autoScalerMessageListener.go

@@ -129,7 +129,7 @@ func (m *AutoScalerClient) Close() error {
 	return m.client.Close()
 }
 
-func (m *AutoScalerClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error) error {
+func (m *AutoScalerClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error, maxCapacity int) error {
 	if m.initialMessage != nil {
 		err := handler(m.initialMessage)
 		if err != nil {
@@ -141,7 +141,7 @@ func (m *AutoScalerClient) GetRunnerScaleSetMessage(ctx context.Context, handler
 	}
 
 	for {
-		message, err := m.client.GetMessage(ctx, m.lastMessageId)
+		message, err := m.client.GetMessage(ctx, m.lastMessageId, maxCapacity)
 		if err != nil {
 			return fmt.Errorf("get message failed from refreshing client. %w", err)
 		}

cmd/githubrunnerscalesetlistener/autoScalerMessageListener_test.go

@@ -317,7 +317,7 @@ func TestGetRunnerScaleSetMessage(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -332,7 +332,7 @@ func TestGetRunnerScaleSetMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(0), asClient.lastMessageId, "Initial message")
@@ -340,7 +340,7 @@ func TestGetRunnerScaleSetMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(1), asClient.lastMessageId, "Last message id should be updated")
@@ -368,7 +368,7 @@ func TestGetRunnerScaleSetMessage_HandleFailed(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -383,14 +383,14 @@ func TestGetRunnerScaleSetMessage_HandleFailed(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return fmt.Errorf("error")
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "handle message failed. error", "Error getting message")
 	assert.Equal(t, int64(0), asClient.lastMessageId, "Last message id should not be updated")
@@ -419,7 +419,7 @@ func TestGetRunnerScaleSetMessage_HandleInitialMessage(t *testing.T) {
 			TotalAssignedJobs:  2,
 		},
 	}
-	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
+	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything, mock.Anything).Return(session, nil)
 	mockActionsClient.On("GetAcquirableJobs", ctx, 1).Return(&actions.AcquirableJobList{
 		Count: 1,
 		Jobs: []actions.AcquirableJob{
@@ -439,7 +439,7 @@ func TestGetRunnerScaleSetMessage_HandleInitialMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Nil(t, asClient.initialMessage, "Initial message should be nil")
@@ -488,7 +488,7 @@ func TestGetRunnerScaleSetMessage_HandleInitialMessageFailed(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return fmt.Errorf("error")
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "fail to process initial message. error", "Error getting message")
 	assert.NotNil(t, asClient.initialMessage, "Initial message should be nil")
@@ -516,8 +516,8 @@ func TestGetRunnerScaleSetMessage_RetryUntilGetMessage(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(nil, nil).Times(3)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(nil, nil).Times(3)
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -532,13 +532,13 @@ func TestGetRunnerScaleSetMessage_RetryUntilGetMessage(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 	assert.NoError(t, err, "Error getting initial message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(1), asClient.lastMessageId, "Last message id should be updated")
@@ -565,7 +565,7 @@ func TestGetRunnerScaleSetMessage_ErrorOnGetMessage(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(nil, fmt.Errorf("error"))
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(nil, fmt.Errorf("error"))
 
 	asClient, err := NewAutoScalerClient(ctx, mockActionsClient, &logger, 1, func(asc *AutoScalerClient) {
 		asc.client = mockSessionClient
@@ -575,12 +575,12 @@ func TestGetRunnerScaleSetMessage_ErrorOnGetMessage(t *testing.T) {
 	// process initial message
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		return nil
-	})
+	}, 10)
 	assert.NoError(t, err, "Error getting initial message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		return fmt.Errorf("Should not be called")
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "get message failed from refreshing client. error", "Error should be returned")
 	assert.Equal(t, int64(0), asClient.lastMessageId, "Last message id should be updated")
@@ -608,7 +608,7 @@ func TestDeleteRunnerScaleSetMessage_Error(t *testing.T) {
 		Statistics: &actions.RunnerScaleSetStatistic{},
 	}
 	mockActionsClient.On("CreateMessageSession", ctx, 1, mock.Anything).Return(session, nil)
-	mockSessionClient.On("GetMessage", ctx, int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockSessionClient.On("GetMessage", ctx, int64(0), mock.Anything).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -623,13 +623,13 @@ func TestDeleteRunnerScaleSetMessage_Error(t *testing.T) {
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 	assert.NoError(t, err, "Error getting initial message")
 
 	err = asClient.GetRunnerScaleSetMessage(ctx, func(msg *actions.RunnerScaleSetMessage) error {
 		logger.Info("Message received", "messageId", msg.MessageId, "messageType", msg.MessageType, "body", msg.Body)
 		return nil
-	})
+	}, 10)
 
 	assert.ErrorContains(t, err, "delete message failed from refreshing client. error", "Error getting message")
 	assert.Equal(t, int64(1), asClient.lastMessageId, "Last message id should be updated")

cmd/githubrunnerscalesetlistener/autoScalerService.go

@@ -89,7 +89,7 @@ func (s *Service) Start() error {
 			s.logger.Info("service is stopped.")
 			return nil
 		default:
-			err := s.rsClient.GetRunnerScaleSetMessage(s.ctx, s.processMessage)
+			err := s.rsClient.GetRunnerScaleSetMessage(s.ctx, s.processMessage, s.settings.MaxRunners)
 			if err != nil {
 				return fmt.Errorf("could not get and process message. %w", err)
 			}

cmd/githubrunnerscalesetlistener/autoScalerService_test.go

@@ -64,7 +64,7 @@ func TestStart(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Run(func(args mock.Arguments) { cancel() }).Return(nil).Once()
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Run(func(mock.Arguments) { cancel() }).Return(nil).Once()
 
 	err = service.Start()
 
@@ -98,7 +98,7 @@ func TestStart_ScaleToMinRunners(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything).Run(func(args mock.Arguments) {
+	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
 		_ = service.scaleForAssignedJobCount(5)
 	}).Return(nil)
 
@@ -137,7 +137,7 @@ func TestStart_ScaleToMinRunnersFailed(t *testing.T) {
 	require.NoError(t, err)
 
 	c := mockKubeManager.On("ScaleEphemeralRunnerSet", ctx, service.settings.Namespace, service.settings.ResourceName, 5).Return(fmt.Errorf("error")).Once()
-	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything).Run(func(args mock.Arguments) {
+	mockRsClient.On("GetRunnerScaleSetMessage", ctx, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
 		_ = service.scaleForAssignedJobCount(5)
 	}).Return(c.ReturnArguments.Get(0))
 
@@ -172,8 +172,8 @@ func TestStart_GetMultipleMessages(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Return(nil).Times(5)
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Run(func(args mock.Arguments) { cancel() }).Return(nil).Once()
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Return(nil).Times(5)
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { cancel() }).Return(nil).Once()
 
 	err = service.Start()
 
@@ -207,8 +207,8 @@ func TestStart_ErrorOnMessage(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Return(nil).Times(2)
-	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything).Return(fmt.Errorf("error")).Once()
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Return(nil).Times(2)
+	mockRsClient.On("GetRunnerScaleSetMessage", service.ctx, mock.Anything, mock.Anything).Return(fmt.Errorf("error")).Once()
 
 	err = service.Start()
 

cmd/githubrunnerscalesetlistener/messageListener.go

@@ -8,6 +8,6 @@ import (
 
 //go:generate mockery --inpackage --name=RunnerScaleSetClient
 type RunnerScaleSetClient interface {
-	GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error) error
+	GetRunnerScaleSetMessage(ctx context.Context, handler func(msg *actions.RunnerScaleSetMessage) error, maxCapacity int) error
 	AcquireJobsForRunnerScaleSet(ctx context.Context, requestIds []int64) error
 }

cmd/githubrunnerscalesetlistener/mock_RunnerScaleSetClient.go

@@ -29,13 +29,13 @@ func (_m *MockRunnerScaleSetClient) AcquireJobsForRunnerScaleSet(ctx context.Con
 	return r0
 }
 
-// GetRunnerScaleSetMessage provides a mock function with given fields: ctx, handler
-func (_m *MockRunnerScaleSetClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(*actions.RunnerScaleSetMessage) error) error {
-	ret := _m.Called(ctx, handler)
+// GetRunnerScaleSetMessage provides a mock function with given fields: ctx, handler, maxCapacity
+func (_m *MockRunnerScaleSetClient) GetRunnerScaleSetMessage(ctx context.Context, handler func(*actions.RunnerScaleSetMessage) error, maxCapacity int) error {
+	ret := _m.Called(ctx, handler, maxCapacity)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, func(*actions.RunnerScaleSetMessage) error) error); ok {
-		r0 = rf(ctx, handler)
+	if rf, ok := ret.Get(0).(func(context.Context, func(*actions.RunnerScaleSetMessage) error, int) error); ok {
+		r0 = rf(ctx, handler, maxCapacity)
 	} else {
 		r0 = ret.Error(0)
 	}

cmd/githubrunnerscalesetlistener/sessionrefreshingclient.go

@@ -24,8 +24,12 @@ func newSessionClient(client actions.ActionsService, logger *logr.Logger, sessio
 	}
 }
 
-func (m *SessionRefreshingClient) GetMessage(ctx context.Context, lastMessageId int64) (*actions.RunnerScaleSetMessage, error) {
-	message, err := m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId)
+func (m *SessionRefreshingClient) GetMessage(ctx context.Context, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error) {
+	if maxCapacity < 0 {
+		return nil, fmt.Errorf("maxCapacity must be greater than or equal to 0")
+	}
+
+	message, err := m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId, maxCapacity)
 	if err == nil {
 		return message, nil
 	}
@@ -42,7 +46,7 @@ func (m *SessionRefreshingClient) GetMessage(ctx context.Context, lastMessageId
 	}
 
 	m.session = session
-	message, err = m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId)
+	message, err = m.client.GetMessage(ctx, m.session.MessageQueueUrl, m.session.MessageQueueAccessToken, lastMessageId, maxCapacity)
 	if err != nil {
 		return nil, fmt.Errorf("delete message failed after refresh message session. %w", err)
 	}

cmd/githubrunnerscalesetlistener/sessionrefreshingclient_test.go

@@ -31,17 +31,17 @@ func TestGetMessage(t *testing.T) {
 		},
 	}
 
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, nil).Once()
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(&actions.RunnerScaleSetMessage{MessageId: 1}, nil).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, nil).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(&actions.RunnerScaleSetMessage{MessageId: 1}, nil).Once()
 
 	client := newSessionClient(mockActionsClient, &logger, session)
 
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	require.NoError(t, err, "GetMessage should not return an error")
 
 	assert.Nil(t, msg, "GetMessage should return nil message")
 
-	msg, err = client.GetMessage(ctx, 0)
+	msg, err = client.GetMessage(ctx, 0, 10)
 	require.NoError(t, err, "GetMessage should not return an error")
 
 	assert.Equal(t, int64(1), msg.MessageId, "GetMessage should return a message with id 1")
@@ -146,11 +146,11 @@ func TestGetMessage_Error(t *testing.T) {
 		},
 	}
 
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, fmt.Errorf("error")).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, fmt.Errorf("error")).Once()
 
 	client := newSessionClient(mockActionsClient, &logger, session)
 
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	assert.ErrorContains(t, err, "get message failed. error", "GetMessage should return an error")
 	assert.Nil(t, msg, "GetMessage should return nil message")
 	assert.True(t, mockActionsClient.AssertExpectations(t), "All expected calls to mockActionsClient should have been made")
@@ -227,8 +227,8 @@ func TestGetMessage_RefreshToken(t *testing.T) {
 			Id: 1,
 		},
 	}
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, "token2", int64(0)).Return(&actions.RunnerScaleSetMessage{
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, "token2", int64(0), 10).Return(&actions.RunnerScaleSetMessage{
 		MessageId:   1,
 		MessageType: "test",
 		Body:        "test",
@@ -243,7 +243,7 @@ func TestGetMessage_RefreshToken(t *testing.T) {
 	}, nil).Once()
 
 	client := newSessionClient(mockActionsClient, &logger, session)
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	assert.NoError(t, err, "Error getting message")
 	assert.Equal(t, int64(1), msg.MessageId, "message id should be updated")
 	assert.Equal(t, "token2", client.session.MessageQueueAccessToken, "Message queue access token should be updated")
@@ -340,11 +340,11 @@ func TestGetMessage_RefreshToken_Failed(t *testing.T) {
 			Id: 1,
 		},
 	}
-	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0)).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
+	mockActionsClient.On("GetMessage", ctx, session.MessageQueueUrl, session.MessageQueueAccessToken, int64(0), 10).Return(nil, &actions.MessageQueueTokenExpiredError{}).Once()
 	mockActionsClient.On("RefreshMessageSession", ctx, session.RunnerScaleSet.Id, session.SessionId).Return(nil, fmt.Errorf("error"))
 
 	client := newSessionClient(mockActionsClient, &logger, session)
-	msg, err := client.GetMessage(ctx, 0)
+	msg, err := client.GetMessage(ctx, 0, 10)
 	assert.ErrorContains(t, err, "refresh message session failed. error", "Error should be returned")
 	assert.Nil(t, msg, "Message should be nil")
 	assert.Equal(t, "token", client.session.MessageQueueAccessToken, "Message queue access token should not be updated")

controllers/actions.github.com/autoscalingrunnerset_controller.go

@@ -277,6 +277,7 @@ func (r *AutoscalingRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl
 			// need to scale down to 0
 			err := patch(ctx, r.Client, latestRunnerSet, func(obj *v1alpha1.EphemeralRunnerSet) {
 				obj.Spec.Replicas = 0
+				obj.Spec.PatchID = 0
 			})
 			if err != nil {
 				log.Error(err, "Failed to patch runner set to set desired count to 0")

controllers/actions.github.com/ephemeralrunner_controller.go

@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
@@ -295,14 +294,17 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 }
 
 func (r *EphemeralRunnerReconciler) cleanupRunnerFromService(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) (ctrl.Result, error) {
+	if err := r.deleteRunnerFromService(ctx, ephemeralRunner, log); err != nil {
 		actionsError := &actions.ActionsError{}
-	err := r.deleteRunnerFromService(ctx, ephemeralRunner, log)
-	if err != nil {
-		if errors.As(err, &actionsError) &&
-			actionsError.StatusCode == http.StatusBadRequest &&
-			strings.Contains(actionsError.ExceptionName, "JobStillRunningException") {
+		if !errors.As(err, &actionsError) {
+			log.Error(err, "Failed to clean up runner from the service (not an ActionsError)")
+			return ctrl.Result{}, err
+		}
+
+		if actionsError.StatusCode == http.StatusBadRequest && actionsError.IsException("JobStillRunningException") {
 			log.Info("Runner is still running the job. Re-queue in 30 seconds")
 			return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
+
 		}
 
 		log.Error(err, "Failed clean up runner from the service")
@@ -310,10 +312,9 @@ func (r *EphemeralRunnerReconciler) cleanupRunnerFromService(ctx context.Context
 	}
 
 	log.Info("Successfully removed runner registration from service")
-	err = patch(ctx, r.Client, ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) {
+	if err := patch(ctx, r.Client, ephemeralRunner, func(obj *v1alpha1.EphemeralRunner) {
 		controllerutil.RemoveFinalizer(obj, ephemeralRunnerActionsFinalizerName)
-	})
-	if err != nil {
+	}); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -528,7 +529,7 @@ func (r *EphemeralRunnerReconciler) updateStatusWithRunnerConfig(ctx context.Con
 		}
 
 		if actionsError.StatusCode != http.StatusConflict ||
-			!strings.Contains(actionsError.ExceptionName, "AgentExistsException") {
+			!actionsError.IsException("AgentExistsException") {
 			return ctrl.Result{}, fmt.Errorf("failed to generate JIT config with Actions service error: %v", err)
 		}
 
@@ -784,7 +785,7 @@ func (r EphemeralRunnerReconciler) runnerRegisteredWithService(ctx context.Conte
 		}
 
 		if actionsError.StatusCode != http.StatusNotFound ||
-			!strings.Contains(actionsError.ExceptionName, "AgentNotFoundException") {
+			!actionsError.IsException("AgentNotFoundException") {
 			return false, fmt.Errorf("failed to check if runner exists in GitHub service: %v", err)
 		}
 

controllers/actions.github.com/ephemeralrunner_controller_test.go

@@ -672,8 +672,10 @@ var _ = Describe("EphemeralRunner", func() {
 								nil,
 								&actions.ActionsError{
 									StatusCode: http.StatusNotFound,
+									Err: &actions.ActionsExceptionError{
 										ExceptionName: "AgentNotFoundException",
 									},
+								},
 							),
 						),
 						nil,

controllers/actions.github.com/ephemeralrunnerset_controller.go

@@ -23,7 +23,6 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
-	"strings"
 
 	"github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
 	"github.com/actions/actions-runner-controller/controllers/actions.github.com/metrics"
@@ -197,7 +196,6 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
 				log.Error(err, "failed to cleanup finished ephemeral runners")
 			}
 		}()
-
 		log.Info("Scaling comparison", "current", total, "desired", ephemeralRunnerSet.Spec.Replicas)
 		switch {
 		case total < ephemeralRunnerSet.Spec.Replicas: // Handle scale up
@@ -208,8 +206,16 @@ func (r *EphemeralRunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.R
 				return ctrl.Result{}, err
 			}
 
-		case total > ephemeralRunnerSet.Spec.Replicas: // Handle scale down scenario.
+		case ephemeralRunnerSet.Spec.PatchID > 0 && total >= ephemeralRunnerSet.Spec.Replicas: // Handle scale down scenario.
+			// If ephemeral runner did not yet update the phase to succeeded, but the scale down
+			// request is issued, we should ignore the scale down request.
+			// Eventually, the ephemeral runner will be cleaned up on the next patch request, which happens
+			// on the next batch
+		case ephemeralRunnerSet.Spec.PatchID == 0 && total > ephemeralRunnerSet.Spec.Replicas:
 			count := total - ephemeralRunnerSet.Spec.Replicas
+			if count <= 0 {
+				break
+			}
 			log.Info("Deleting ephemeral runners (scale down)", "count", count)
 			if err := r.deleteIdleEphemeralRunners(
 				ctx,
@@ -428,6 +434,9 @@ func (r *EphemeralRunnerSetReconciler) createProxySecret(ctx context.Context, ep
 // When this happens, the next reconcile loop will try to delete the remaining ephemeral runners
 // after we get notified by any of the `v1alpha1.EphemeralRunner.Status` updates.
 func (r *EphemeralRunnerSetReconciler) deleteIdleEphemeralRunners(ctx context.Context, ephemeralRunnerSet *v1alpha1.EphemeralRunnerSet, pendingEphemeralRunners, runningEphemeralRunners []*v1alpha1.EphemeralRunner, count int, log logr.Logger) error {
+	if count <= 0 {
+		return nil
+	}
 	runners := newEphemeralRunnerStepper(pendingEphemeralRunners, runningEphemeralRunners)
 	if runners.len() == 0 {
 		log.Info("No pending or running ephemeral runners running at this time for scale down")
@@ -473,10 +482,14 @@ func (r *EphemeralRunnerSetReconciler) deleteIdleEphemeralRunners(ctx context.Co
 func (r *EphemeralRunnerSetReconciler) deleteEphemeralRunnerWithActionsClient(ctx context.Context, ephemeralRunner *v1alpha1.EphemeralRunner, actionsClient actions.ActionsService, log logr.Logger) (bool, error) {
 	if err := actionsClient.RemoveRunner(ctx, int64(ephemeralRunner.Status.RunnerId)); err != nil {
 		actionsError := &actions.ActionsError{}
-		if errors.As(err, &actionsError) &&
-			actionsError.StatusCode == http.StatusBadRequest &&
-			strings.Contains(actionsError.ExceptionName, "JobStillRunningException") {
-			// Runner is still running a job, proceed with the next one
+		if !errors.As(err, &actionsError) {
+			log.Error(err, "failed to remove runner from the service", "name", ephemeralRunner.Name, "runnerId", ephemeralRunner.Status.RunnerId)
+			return false, err
+		}
+
+		if actionsError.StatusCode == http.StatusBadRequest &&
+			actionsError.IsException("JobStillRunningException") {
+			log.Info("Runner is still running a job, skipping deletion", "name", ephemeralRunner.Name, "runnerId", ephemeralRunner.Status.RunnerId)
 			return false, nil
 		}
 

controllers/actions.github.com/ephemeralrunnerset_controller_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -275,21 +274,18 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() {
 	})
 
 	Context("When a new EphemeralRunnerSet scale up and down", func() {
-		It("Should scale only on patch ID change", func() {
-			created := new(actionsv1alpha1.EphemeralRunnerSet)
-			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, created)
+		It("Should scale up with patch ID 0", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
 			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
 
-			patchID := 1
-
-			// Scale up the EphemeralRunnerSet
-			updated := created.DeepCopy()
+			updated := ers.DeepCopy()
 			updated.Spec.Replicas = 5
-			updated.Spec.PatchID = patchID
-			err = k8sClient.Update(ctx, updated)
+			updated.Spec.PatchID = 0
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
 			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
 
-			// Wait for the EphemeralRunnerSet to be scaled up
 			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
 			Eventually(
 				func() (int, error) {
@@ -298,110 +294,282 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() {
 						return -1, err
 					}
 
-					// Set status to simulate a configured EphemeralRunner
-					refetch := false
-					for i, runner := range runnerList.Items {
-						if runner.Status.RunnerId == 0 {
-							updatedRunner := runner.DeepCopy()
-							updatedRunner.Status.Phase = corev1.PodRunning
-							updatedRunner.Status.RunnerId = i + 100
-							err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner))
-							Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-							refetch = true
-						}
-					}
-
-					if refetch {
-						err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
-						if err != nil {
-							return -1, err
-						}
-					}
-
 					return len(runnerList.Items), nil
 				},
 				ephemeralRunnerSetTestTimeout,
 				ephemeralRunnerSetTestInterval,
 			).Should(BeEquivalentTo(5), "5 EphemeralRunner should be created")
+		})
 
-			// Mark one of the EphemeralRunner as finished
-			finishedRunner := runnerList.Items[4].DeepCopy()
-			finishedRunner.Status.Phase = corev1.PodSucceeded
-			err = k8sClient.Status().Patch(ctx, finishedRunner, client.MergeFrom(&runnerList.Items[4]))
+		It("Should scale up when patch ID changes", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated := ers.DeepCopy()
+			updated.Spec.Replicas = 1
+			updated.Spec.PatchID = 0
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(1), "1 EphemeralRunner should be created")
+
+			ers = new(actionsv1alpha1.EphemeralRunnerSet)
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated = ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 1
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+		})
+
+		It("Should clean up finished ephemeral runner when scaling down", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated := ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 1
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+
+			updatedRunner := runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodSucceeded
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
 			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
 
-			// Wait for the finished EphemeralRunner to be set to succeeded
+			updatedRunner = runnerList.Items[1].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodRunning
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[1]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			// Keep the ephemeral runner until the next patch
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "1 EphemeralRunner should be up")
+
+			// The listener was slower to patch the completed, but we should still have 1 running
+			ers = new(actionsv1alpha1.EphemeralRunnerSet)
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated = ers.DeepCopy()
+			updated.Spec.Replicas = 1
+			updated.Spec.PatchID = 2
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(1), "1 Ephemeral runner should be up")
+		})
+
+		It("Should keep finished ephemeral runners until patch id changes", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated := ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 1
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+
+			updatedRunner := runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodSucceeded
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			updatedRunner = runnerList.Items[1].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodPending
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[1]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			// confirm they are not deleted
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Consistently(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				5*time.Second,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+		})
+
+		It("Should handle double scale up", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated := ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 1
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+
+			updatedRunner := runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodSucceeded
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			updatedRunner = runnerList.Items[1].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodRunning
+
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[1]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			ers = new(actionsv1alpha1.EphemeralRunnerSet)
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated = ers.DeepCopy()
+			updated.Spec.Replicas = 3
+			updated.Spec.PatchID = 2
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			// We should have 3 runners, and have no Succeeded ones
 			Eventually(
 				func() error {
-					runnerList := new(actionsv1alpha1.EphemeralRunnerList)
 					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
 					if err != nil {
 						return err
 					}
 
-					for _, runner := range runnerList.Items {
-						if runner.Name != finishedRunner.Name {
-							continue
-						}
-
-						if runner.Status.Phase != corev1.PodSucceeded {
-							return fmt.Errorf("EphemeralRunner is not finished")
-						}
-						// found pod succeeded
-						return nil
-					}
-
-					return errors.New("Finished ephemeral runner is not found")
-				},
-				ephemeralRunnerSetTestTimeout,
-				ephemeralRunnerSetTestInterval,
-			).Should(Succeed(), "Finished EphemeralRunner should be deleted")
-
-			// After one ephemeral runner is finished, simulate job done patch
-			patchID++
-			original := new(actionsv1alpha1.EphemeralRunnerSet)
-			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, original)
-			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
-			updated = original.DeepCopy()
-			updated.Spec.PatchID = patchID
-			updated.Spec.Replicas = 4
-			err = k8sClient.Patch(ctx, updated, client.MergeFrom(original))
-			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
-
-			// Only finished ephemeral runner should be deleted
-			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
-			Eventually(
-				func() (int, error) {
-					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
-					if err != nil {
-						return -1, err
+					if len(runnerList.Items) != 3 {
+						return fmt.Errorf("Expected 3 runners, got %d", len(runnerList.Items))
 					}
 
 					for _, runner := range runnerList.Items {
 						if runner.Status.Phase == corev1.PodSucceeded {
-							return -1, fmt.Errorf("Finished EphemeralRunner should be deleted")
+							return fmt.Errorf("Runner %s is in Succeeded phase", runner.Name)
 						}
 					}
 
-					return len(runnerList.Items), nil
+					return nil
 				},
 				ephemeralRunnerSetTestTimeout,
 				ephemeralRunnerSetTestInterval,
-			).Should(BeEquivalentTo(4), "4 EphemeralRunner should be created")
+			).Should(BeNil(), "3 EphemeralRunner should be created and none should be in Succeeded phase")
+		})
 
-			// Scaling down the EphemeralRunnerSet
-			patchID++
-			original = new(actionsv1alpha1.EphemeralRunnerSet)
-			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, original)
+		It("Should handle scale down without removing pending runners", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
 			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
-			updated = original.DeepCopy()
-			updated.Spec.PatchID = patchID
-			updated.Spec.Replicas = 3
-			err = k8sClient.Patch(ctx, updated, client.MergeFrom(original))
+
+			updated := ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 1
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
 			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
 
-			// Wait for the EphemeralRunnerSet to be scaled down
-			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
 			Eventually(
 				func() (int, error) {
 					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
@@ -409,150 +577,215 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() {
 						return -1, err
 					}
 
-					// Set status to simulate a configured EphemeralRunner
-					refetch := false
-					for i, runner := range runnerList.Items {
-						if runner.Status.RunnerId == 0 {
-							updatedRunner := runner.DeepCopy()
-							updatedRunner.Status.Phase = corev1.PodRunning
-							updatedRunner.Status.RunnerId = i + 100
-							err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner))
-							Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-							refetch = true
-						}
-					}
-
-					if refetch {
-						err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
-						if err != nil {
-							return -1, err
-						}
-					}
-
-					return len(runnerList.Items), nil
-				},
-				ephemeralRunnerSetTestTimeout,
-				ephemeralRunnerSetTestInterval,
-			).Should(BeEquivalentTo(3), "3 EphemeralRunner should be created")
-
-			// We will not scale down runner that is running jobs
-			runningRunner := runnerList.Items[0].DeepCopy()
-			runningRunner.Status.JobRequestId = 1000
-			err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[0]))
-			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-
-			runningRunner = runnerList.Items[1].DeepCopy()
-			runningRunner.Status.JobRequestId = 1001
-			err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[1]))
-			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-
-			// Scale down to 1 while 2 are running
-			patchID++
-			original = new(actionsv1alpha1.EphemeralRunnerSet)
-			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, original)
-			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
-			updated = original.DeepCopy()
-			updated.Spec.PatchID = patchID
-			updated.Spec.Replicas = 1
-			err = k8sClient.Patch(ctx, updated, client.MergeFrom(original))
-			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
-
-			// Wait for the EphemeralRunnerSet to be scaled down to 2 since we still have 2 runner running jobs
-			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
-			Eventually(
-				func() (int, error) {
-					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
-					if err != nil {
-						return -1, err
-					}
-
-					// Set status to simulate a configured EphemeralRunner
-					refetch := false
-					for i, runner := range runnerList.Items {
-						if runner.Status.RunnerId == 0 {
-							updatedRunner := runner.DeepCopy()
-							updatedRunner.Status.Phase = corev1.PodRunning
-							updatedRunner.Status.RunnerId = i + 100
-							err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner))
-							Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-							refetch = true
-						}
-					}
-
-					if refetch {
-						err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
-						if err != nil {
-							return -1, err
-						}
-					}
-
 					return len(runnerList.Items), nil
 				},
 				ephemeralRunnerSetTestTimeout,
 				ephemeralRunnerSetTestInterval,
 			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
 
-			// We will not scale down failed runner
-			failedRunner := runnerList.Items[0].DeepCopy()
-			failedRunner.Status.Phase = corev1.PodFailed
-			err = k8sClient.Status().Patch(ctx, failedRunner, client.MergeFrom(&runnerList.Items[0]))
+			updatedRunner := runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodSucceeded
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
 			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
 
+			updatedRunner = runnerList.Items[1].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodPending
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[1]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			// Wait for these statuses to actually be updated
 			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
 			Eventually(
-				func() (int, error) {
+				func() error {
 					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
 					if err != nil {
-						return -1, err
+						return err
 					}
-
-					// Set status to simulate a configured EphemeralRunner
-					refetch := false
-					for i, runner := range runnerList.Items {
-						if runner.Status.RunnerId == 0 {
-							updatedRunner := runner.DeepCopy()
-							updatedRunner.Status.Phase = corev1.PodRunning
-							updatedRunner.Status.RunnerId = i + 100
-							err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner))
-							Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-							refetch = true
+					pending := 0
+					succeeded := 0
+					for _, runner := range runnerList.Items {
+						switch runner.Status.Phase {
+						case corev1.PodSucceeded:
+							succeeded++
+						case corev1.PodPending:
+							pending++
 						}
 					}
 
-					if refetch {
-						err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
-						if err != nil {
-							return -1, err
-						}
+					if pending != 1 && succeeded != 1 {
+						return fmt.Errorf("Expected 1 runner in Pending and 1 in Succeeded, got %d in Pending and %d in Succeeded", pending, succeeded)
 					}
 
-					return len(runnerList.Items), nil
+					return nil
 				},
 				ephemeralRunnerSetTestTimeout,
 				ephemeralRunnerSetTestInterval,
-			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+			).Should(BeNil(), "1 EphemeralRunner should be in Pending and 1 in Succeeded phase")
 
-			// We will scale down to 0 when the running job is completed and the failed runner is deleted
-			runningRunner = runnerList.Items[1].DeepCopy()
-			runningRunner.Status.Phase = corev1.PodSucceeded
-			err = k8sClient.Status().Patch(ctx, runningRunner, client.MergeFrom(&runnerList.Items[1]))
-			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-
-			err = k8sClient.Delete(ctx, &runnerList.Items[0])
-			Expect(err).NotTo(HaveOccurred(), "failed to delete EphemeralRunner")
-
-			// Scale down to 0 while 1 ephemeral runner is failed
-			patchID++
-			original = new(actionsv1alpha1.EphemeralRunnerSet)
-			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, original)
+			// Scale down to 0, while 1 is still pending. This simulates the difference between the desired and actual state
+			ers = new(actionsv1alpha1.EphemeralRunnerSet)
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
 			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
-			updated = original.DeepCopy()
-			updated.Spec.PatchID = patchID
+
+			updated = ers.DeepCopy()
 			updated.Spec.Replicas = 0
-			err = k8sClient.Patch(ctx, updated, client.MergeFrom(original))
+			updated.Spec.PatchID = 2
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
 			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
 
-			// Wait for the EphemeralRunnerSet to be scaled down to 0
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			// We should have 1 runner up and pending
+			Eventually(
+				func() error {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return err
+					}
+
+					if len(runnerList.Items) != 1 {
+						return fmt.Errorf("Expected 1 runner, got %d", len(runnerList.Items))
+					}
+
+					if runnerList.Items[0].Status.Phase != corev1.PodPending {
+						return fmt.Errorf("Expected runner to be in Pending, got %s", runnerList.Items[0].Status.Phase)
+					}
+
+					return nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeNil(), "1 EphemeralRunner should be created and in Pending phase")
+
+			// Now, the ephemeral runner finally is done and we can scale down to 0
+			updatedRunner = runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodSucceeded
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(0), "2 EphemeralRunner should be created")
+		})
+
+		It("Should kill pending and running runners if they are up for some reason and the batch contains no jobs", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated := ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 1
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+
+			// Put one runner in Pending and one in Running
+			updatedRunner := runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodPending
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			updatedRunner = runnerList.Items[1].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodRunning
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[1]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			// Wait for these statuses to actually be updated
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() error {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return err
+					}
+
+					pending := 0
+					running := 0
+
+					for _, runner := range runnerList.Items {
+						switch runner.Status.Phase {
+						case corev1.PodPending:
+							pending++
+						case corev1.PodRunning:
+							running++
+
+						}
+					}
+
+					if pending != 1 && running != 1 {
+						return fmt.Errorf("Expected 1 runner in Pending and 1 in Running, got %d in Pending and %d in Running", pending, running)
+					}
+
+					return nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeNil(), "1 EphemeralRunner should be in Pending and 1 in Running phase")
+
+			// Scale down to 0 with patch ID 0. This forces the scale down to self correct on empty batch
+
+			ers = new(actionsv1alpha1.EphemeralRunnerSet)
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated = ers.DeepCopy()
+			updated.Spec.Replicas = 0
+			updated.Spec.PatchID = 0
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Consistently(
+				func() (int, error) {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return -1, err
+					}
+
+					return len(runnerList.Items), nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be up since they don't have an ID yet")
+
+			// Now, let's say ephemeral runner controller patched these ephemeral runners with the registration.
+
+			updatedRunner = runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.RunnerId = 1
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			updatedRunner = runnerList.Items[1].DeepCopy()
+			updatedRunner.Status.RunnerId = 2
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[1]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			// Now, eventually, they should be deleted
 			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
 			Eventually(
 				func() (int, error) {
@@ -561,31 +794,120 @@ var _ = Describe("Test EphemeralRunnerSet controller", func() {
 						return -1, err
 					}
 
-					// Set status to simulate a configured EphemeralRunner
-					refetch := false
-					for i, runner := range runnerList.Items {
-						if runner.Status.RunnerId == 0 {
-							updatedRunner := runner.DeepCopy()
-							updatedRunner.Status.Phase = corev1.PodRunning
-							updatedRunner.Status.RunnerId = i + 100
-							err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runner))
-							Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
-							refetch = true
-						}
-					}
+					return len(runnerList.Items), nil
 
-					if refetch {
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeEquivalentTo(0), "0 EphemeralRunner should exist")
+		})
+
+		It("Should replace finished ephemeral runners with new ones", func() {
+			ers := new(actionsv1alpha1.EphemeralRunnerSet)
+			err := k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated := ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 1
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList := new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() (int, error) {
 					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
 					if err != nil {
 						return -1, err
 					}
-					}
 
 					return len(runnerList.Items), nil
 				},
 				ephemeralRunnerSetTestTimeout,
 				ephemeralRunnerSetTestInterval,
-			).Should(BeEquivalentTo(0), "0 EphemeralRunner should be created")
+			).Should(BeEquivalentTo(2), "2 EphemeralRunner should be created")
+
+			// Put one runner in Succeeded and one in Running
+			updatedRunner := runnerList.Items[0].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodSucceeded
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[0]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			updatedRunner = runnerList.Items[1].DeepCopy()
+			updatedRunner.Status.Phase = corev1.PodRunning
+			err = k8sClient.Status().Patch(ctx, updatedRunner, client.MergeFrom(&runnerList.Items[1]))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunner")
+
+			// Wait for these statuses to actually be updated
+
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() error {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return err
+					}
+
+					succeeded := 0
+					running := 0
+
+					for _, runner := range runnerList.Items {
+						switch runner.Status.Phase {
+						case corev1.PodSucceeded:
+							succeeded++
+						case corev1.PodRunning:
+							running++
+						}
+					}
+
+					if succeeded != 1 && running != 1 {
+						return fmt.Errorf("Expected 1 runner in Succeeded and 1 in Running, got %d in Succeeded and %d in Running", succeeded, running)
+					}
+
+					return nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeNil(), "1 EphemeralRunner should be in Succeeded and 1 in Running phase")
+
+			// Now, let's simulate replacement. The desired count is still 2.
+			// This simulates that we got 1 job assigned, and 1 job completed.
+
+			ers = new(actionsv1alpha1.EphemeralRunnerSet)
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: ephemeralRunnerSet.Name, Namespace: ephemeralRunnerSet.Namespace}, ers)
+			Expect(err).NotTo(HaveOccurred(), "failed to get EphemeralRunnerSet")
+
+			updated = ers.DeepCopy()
+			updated.Spec.Replicas = 2
+			updated.Spec.PatchID = 2
+
+			err = k8sClient.Patch(ctx, updated, client.MergeFrom(ers))
+			Expect(err).NotTo(HaveOccurred(), "failed to update EphemeralRunnerSet")
+
+			runnerList = new(actionsv1alpha1.EphemeralRunnerList)
+			Eventually(
+				func() error {
+					err := k8sClient.List(ctx, runnerList, client.InNamespace(ephemeralRunnerSet.Namespace))
+					if err != nil {
+						return err
+					}
+
+					if len(runnerList.Items) != 2 {
+						return fmt.Errorf("Expected 2 runners, got %d", len(runnerList.Items))
+					}
+
+					for _, runner := range runnerList.Items {
+						if runner.Status.Phase == corev1.PodSucceeded {
+							return fmt.Errorf("Expected no runners in Succeeded phase, got one")
+						}
+					}
+
+					return nil
+				},
+				ephemeralRunnerSetTestTimeout,
+				ephemeralRunnerSetTestInterval,
+			).Should(BeNil(), "2 EphemeralRunner should be created and none should be in Succeeded phase")
 		})
 
 		It("Should update status on Ephemeral Runner state changes", func() {

docs/gha-runner-scale-set-controller/README.md

@@ -43,6 +43,16 @@ You can follow [this troubleshooting guide](https://docs.github.com/en/actions/h
 
 ## Changelog
 
+### v0.9.1
+
+#### Major changes
+
+1. Shutdown metrics server when listener exits [#3445](https://github.com/actions/actions-runner-controller/pull/3445)
+1. Propagate max capacity information to the actions back-end [#3431](https://github.com/actions/actions-runner-controller/pull/3431)
+1. Refactor actions client error to include request id [#3430](https://github.com/actions/actions-runner-controller/pull/3430)
+1. Include self correction on empty batch and avoid removing pending runners when cluster is busy [#3426](https://github.com/actions/actions-runner-controller/pull/3426)
+1. Add topologySpreadConstraint to gha-runner-scale-set-controller chart [#3405](https://github.com/actions/actions-runner-controller/pull/3405)
+
 ### v0.9.0
 
 #### ⚠️ Warning

github/actions/client.go

@@ -29,6 +29,9 @@ const (
 	apiVersionQueryParam = "api-version=6.0-preview"
 )
 
+// Header used to propagate capacity information to the back-end
+const HeaderScaleSetMaxCapacity = "X-ScaleSetMaxCapacity"
+
 //go:generate mockery --inpackage --name=ActionsService
 type ActionsService interface {
 	GetRunnerScaleSet(ctx context.Context, runnerGroupId int, runnerScaleSetName string) (*RunnerScaleSet, error)
@@ -45,7 +48,7 @@ type ActionsService interface {
 	AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQueueAccessToken string, requestIds []int64) ([]int64, error)
 	GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (*AcquirableJobList, error)
 
-	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*RunnerScaleSetMessage, error)
+	GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error)
 	DeleteMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, messageId int64) error
 
 	GenerateJitRunnerConfig(ctx context.Context, jitRunnerSetting *RunnerScaleSetJitRunnerSetting, scaleSetId int) (*RunnerScaleSetJitRunnerConfig, error)
@@ -104,6 +107,8 @@ type Client struct {
 	proxyFunc ProxyFunc
 }
 
+var _ ActionsService = &Client{}
+
 type ProxyFunc func(req *http.Request) (*url.URL, error)
 
 type ClientOption func(*Client)
@@ -355,15 +360,22 @@ func (c *Client) GetRunnerScaleSet(ctx context.Context, runnerGroupId int, runne
 	}
 
 	var runnerScaleSetList *runnerScaleSetsResponse
-	err = json.NewDecoder(resp.Body).Decode(&runnerScaleSetList)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerScaleSetList); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	if runnerScaleSetList.Count == 0 {
 		return nil, nil
 	}
 	if runnerScaleSetList.Count > 1 {
-		return nil, fmt.Errorf("multiple runner scale sets found with name %s", runnerScaleSetName)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("multiple runner scale sets found with name %q", runnerScaleSetName),
+		}
 	}
 
 	return &runnerScaleSetList.RunnerScaleSets[0], nil
@@ -386,9 +398,12 @@ func (c *Client) GetRunnerScaleSetById(ctx context.Context, runnerScaleSetId int
 	}
 
 	var runnerScaleSet *RunnerScaleSet
-	err = json.NewDecoder(resp.Body).Decode(&runnerScaleSet)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerScaleSet); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return runnerScaleSet, nil
 }
@@ -408,23 +423,43 @@ func (c *Client) GetRunnerGroupByName(ctx context.Context, runnerGroup string) (
 	if resp.StatusCode != http.StatusOK {
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			return nil, err
+			return nil, &ActionsError{
+				StatusCode: resp.StatusCode,
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				Err:        err,
 			}
-		return nil, fmt.Errorf("unexpected status code: %d - body: %s", resp.StatusCode, string(body))
+		}
+		return nil, fmt.Errorf("unexpected status code: %w", &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        errors.New(string(body)),
+		})
 	}
 
 	var runnerGroupList *RunnerGroupList
 	err = json.NewDecoder(resp.Body).Decode(&runnerGroupList)
 	if err != nil {
-		return nil, err
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	if runnerGroupList.Count == 0 {
-		return nil, fmt.Errorf("no runner group found with name '%s'", runnerGroup)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("no runner group found with name %q", runnerGroup),
+		}
 	}
 
 	if runnerGroupList.Count > 1 {
-		return nil, fmt.Errorf("multiple runner group found with name %s", runnerGroup)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("multiple runner group found with name %q", runnerGroup),
+		}
 	}
 
 	return &runnerGroupList.RunnerGroups[0], nil
@@ -450,9 +485,12 @@ func (c *Client) CreateRunnerScaleSet(ctx context.Context, runnerScaleSet *Runne
 		return nil, ParseActionsErrorFromResponse(resp)
 	}
 	var createdRunnerScaleSet *RunnerScaleSet
-	err = json.NewDecoder(resp.Body).Decode(&createdRunnerScaleSet)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&createdRunnerScaleSet); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return createdRunnerScaleSet, nil
 }
@@ -480,9 +518,12 @@ func (c *Client) UpdateRunnerScaleSet(ctx context.Context, runnerScaleSetId int,
 	}
 
 	var updatedRunnerScaleSet *RunnerScaleSet
-	err = json.NewDecoder(resp.Body).Decode(&updatedRunnerScaleSet)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&updatedRunnerScaleSet); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return updatedRunnerScaleSet, nil
 }
@@ -507,7 +548,7 @@ func (c *Client) DeleteRunnerScaleSet(ctx context.Context, runnerScaleSetId int)
 	return nil
 }
 
-func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*RunnerScaleSetMessage, error) {
+func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error) {
 	u, err := url.Parse(messageQueueUrl)
 	if err != nil {
 		return nil, err
@@ -519,6 +560,10 @@ func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAc
 		u.RawQuery = q.Encode()
 	}
 
+	if maxCapacity < 0 {
+		return nil, fmt.Errorf("maxCapacity must be greater than or equal to 0")
+	}
+
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 	if err != nil {
 		return nil, err
@@ -527,6 +572,7 @@ func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAc
 	req.Header.Set("Accept", "application/json; api-version=6.0-preview")
 	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", messageQueueAccessToken))
 	req.Header.Set("User-Agent", c.userAgent.String())
+	req.Header.Set(HeaderScaleSetMaxCapacity, strconv.Itoa(maxCapacity))
 
 	resp, err := c.Do(req)
 	if err != nil {
@@ -547,15 +593,26 @@ func (c *Client) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAc
 		body, err := io.ReadAll(resp.Body)
 		body = trimByteOrderMark(body)
 		if err != nil {
-			return nil, err
+			return nil, &ActionsError{
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				StatusCode: resp.StatusCode,
+				Err:        err,
+			}
+		}
+		return nil, &MessageQueueTokenExpiredError{
+			activityID: resp.Header.Get(HeaderActionsActivityID),
+			statusCode: resp.StatusCode,
+			msg:        string(body),
 		}
-		return nil, &MessageQueueTokenExpiredError{msg: string(body)}
 	}
 
 	var message *RunnerScaleSetMessage
-	err = json.NewDecoder(resp.Body).Decode(&message)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&message); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return message, nil
 }
@@ -591,9 +648,17 @@ func (c *Client) DeleteMessage(ctx context.Context, messageQueueUrl, messageQueu
 		body, err := io.ReadAll(resp.Body)
 		body = trimByteOrderMark(body)
 		if err != nil {
-			return err
+			return &ActionsError{
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				StatusCode: resp.StatusCode,
+				Err:        err,
+			}
+		}
+		return &MessageQueueTokenExpiredError{
+			activityID: resp.Header.Get(HeaderActionsActivityID),
+			statusCode: resp.StatusCode,
+			msg:        string(body),
 		}
-		return &MessageQueueTokenExpiredError{msg: string(body)}
 	}
 	return nil
 }
@@ -641,9 +706,18 @@ func (c *Client) doSessionRequest(ctx context.Context, method, path string, requ
 	}
 
 	if resp.StatusCode == expectedResponseStatusCode {
-		if responseUnmarshalTarget != nil {
-			return json.NewDecoder(resp.Body).Decode(responseUnmarshalTarget)
+		if responseUnmarshalTarget == nil {
+			return nil
 		}
+
+		if err := json.NewDecoder(resp.Body).Decode(responseUnmarshalTarget); err != nil {
+			return &ActionsError{
+				StatusCode: resp.StatusCode,
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				Err:        err,
+			}
+		}
+
 		return nil
 	}
 
@@ -655,10 +729,18 @@ func (c *Client) doSessionRequest(ctx context.Context, method, path string, requ
 	body, err := io.ReadAll(resp.Body)
 	body = trimByteOrderMark(body)
 	if err != nil {
-		return err
+		return &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
-	return fmt.Errorf("unexpected status code: %d - body: %s", resp.StatusCode, string(body))
+	return fmt.Errorf("unexpected status code: %w", &ActionsError{
+		StatusCode: resp.StatusCode,
+		ActivityID: resp.Header.Get(HeaderActionsActivityID),
+		Err:        errors.New(string(body)),
+	})
 }
 
 func (c *Client) AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQueueAccessToken string, requestIds []int64) ([]int64, error) {
@@ -692,16 +774,28 @@ func (c *Client) AcquireJobs(ctx context.Context, runnerScaleSetId int, messageQ
 		body, err := io.ReadAll(resp.Body)
 		body = trimByteOrderMark(body)
 		if err != nil {
-			return nil, err
+			return nil, &ActionsError{
+				ActivityID: resp.Header.Get(HeaderActionsActivityID),
+				StatusCode: resp.StatusCode,
+				Err:        err,
+			}
 		}
 
-		return nil, &MessageQueueTokenExpiredError{msg: string(body)}
+		return nil, &MessageQueueTokenExpiredError{
+			activityID: resp.Header.Get(HeaderActionsActivityID),
+			statusCode: resp.StatusCode,
+			msg:        string(body),
+		}
 	}
 
 	var acquiredJobs *Int64List
 	err = json.NewDecoder(resp.Body).Decode(&acquiredJobs)
 	if err != nil {
-		return nil, err
+		return nil, &ActionsError{
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			StatusCode: resp.StatusCode,
+			Err:        err,
+		}
 	}
 
 	return acquiredJobs.Value, nil
@@ -732,7 +826,11 @@ func (c *Client) GetAcquirableJobs(ctx context.Context, runnerScaleSetId int) (*
 	var acquirableJobList *AcquirableJobList
 	err = json.NewDecoder(resp.Body).Decode(&acquirableJobList)
 	if err != nil {
-		return nil, err
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	return acquirableJobList, nil
@@ -761,9 +859,12 @@ func (c *Client) GenerateJitRunnerConfig(ctx context.Context, jitRunnerSetting *
 	}
 
 	var runnerJitConfig *RunnerScaleSetJitRunnerConfig
-	err = json.NewDecoder(resp.Body).Decode(&runnerJitConfig)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerJitConfig); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 	return runnerJitConfig, nil
 }
@@ -786,9 +887,12 @@ func (c *Client) GetRunner(ctx context.Context, runnerId int64) (*RunnerReferenc
 	}
 
 	var runnerReference *RunnerReference
-	err = json.NewDecoder(resp.Body).Decode(&runnerReference)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerReference); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	return runnerReference, nil
@@ -812,9 +916,12 @@ func (c *Client) GetRunnerByName(ctx context.Context, runnerName string) (*Runne
 	}
 
 	var runnerList *RunnerReferenceList
-	err = json.NewDecoder(resp.Body).Decode(&runnerList)
-	if err != nil {
-		return nil, err
+	if err := json.NewDecoder(resp.Body).Decode(&runnerList); err != nil {
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        err,
+		}
 	}
 
 	if runnerList.Count == 0 {
@@ -822,7 +929,11 @@ func (c *Client) GetRunnerByName(ctx context.Context, runnerName string) (*Runne
 	}
 
 	if runnerList.Count > 1 {
-		return nil, fmt.Errorf("multiple runner found with name %s", runnerName)
+		return nil, &ActionsError{
+			StatusCode: resp.StatusCode,
+			ActivityID: resp.Header.Get(HeaderActionsActivityID),
+			Err:        fmt.Errorf("multiple runner found with name %s", runnerName),
+		}
 	}
 
 	return &runnerList.RunnerReferences[0], nil
@@ -895,12 +1006,20 @@ func (c *Client) getRunnerRegistrationToken(ctx context.Context) (*registrationT
 		if err != nil {
 			return nil, err
 		}
-		return nil, fmt.Errorf("unexpected response from Actions service during registration token call: %v - %v", resp.StatusCode, string(body))
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        errors.New(string(body)),
+		}
 	}
 
 	var registrationToken *registrationToken
 	if err := json.NewDecoder(resp.Body).Decode(&registrationToken); err != nil {
-		return nil, err
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        err,
+		}
 	}
 
 	return registrationToken, nil
@@ -937,8 +1056,14 @@ func (c *Client) fetchAccessToken(ctx context.Context, gitHubConfigURL string, c
 
 	// Format: https://docs.github.com/en/rest/apps/apps#create-an-installation-access-token-for-an-app
 	var accessToken *accessToken
-	err = json.NewDecoder(resp.Body).Decode(&accessToken)
-	return accessToken, err
+	if err = json.NewDecoder(resp.Body).Decode(&accessToken); err != nil {
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        err,
+		}
+	}
+	return accessToken, nil
 }
 
 type ActionsServiceAdminConnection struct {
@@ -989,21 +1114,29 @@ func (c *Client) getActionsServiceAdminConnection(ctx context.Context, rt *regis
 			break
 		}
 
-		errStr := fmt.Sprintf("unexpected response from Actions service during registration call: %v", resp.StatusCode)
+		var innerErr error
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
-			err = fmt.Errorf("%s - %w", errStr, err)
+			innerErr = err
 		} else {
-			err = fmt.Errorf("%s - %v", errStr, string(body))
+			innerErr = errors.New(string(body))
 		}
 
 		if resp.StatusCode != http.StatusUnauthorized && resp.StatusCode != http.StatusForbidden {
-			return nil, err
+			return nil, &GitHubAPIError{
+				StatusCode: resp.StatusCode,
+				RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+				Err:        innerErr,
+			}
 		}
 
 		retry++
 		if retry > 3 {
-			return nil, fmt.Errorf("unable to register runner after 3 retries: %v", err)
+			return nil, fmt.Errorf("unable to register runner after 3 retries: %w", &GitHubAPIError{
+				StatusCode: resp.StatusCode,
+				RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+				Err:        innerErr,
+			})
 		}
 		time.Sleep(time.Duration(500 * int(time.Millisecond) * (retry + 1)))
 
@@ -1011,7 +1144,11 @@ func (c *Client) getActionsServiceAdminConnection(ctx context.Context, rt *regis
 
 	var actionsServiceAdminConnection *ActionsServiceAdminConnection
 	if err := json.NewDecoder(resp.Body).Decode(&actionsServiceAdminConnection); err != nil {
-		return nil, err
+		return nil, &GitHubAPIError{
+			StatusCode: resp.StatusCode,
+			RequestID:  resp.Header.Get(HeaderGitHubRequestID),
+			Err:        err,
+		}
 	}
 
 	return actionsServiceAdminConnection, nil

github/actions/client_runner_scale_set_message_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
+	"strconv"
 	"testing"
 	"time"
 
@@ -35,7 +36,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(s.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		got, err := client.GetMessage(ctx, s.URL, token, 0)
+		got, err := client.GetMessage(ctx, s.URL, token, 0, 10)
 		require.NoError(t, err)
 		assert.Equal(t, want, got)
 	})
@@ -52,7 +53,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(s.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		got, err := client.GetMessage(ctx, s.URL, token, 1)
+		got, err := client.GetMessage(ctx, s.URL, token, 1, 10)
 		require.NoError(t, err)
 		assert.Equal(t, want, got)
 	})
@@ -76,7 +77,7 @@ func TestGetMessage(t *testing.T) {
 		)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		assert.NotNil(t, err)
 		assert.Equalf(t, actualRetry, expectedRetry, "A retry was expected after the first request but got: %v", actualRetry)
 	})
@@ -89,7 +90,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		require.NotNil(t, err)
 
 		var expectedErr *actions.MessageQueueTokenExpiredError
@@ -98,7 +99,7 @@ func TestGetMessage(t *testing.T) {
 
 	t.Run("Status code not found", func(t *testing.T) {
 		want := actions.ActionsError{
-			Message:    "Request returned status: 404 Not Found",
+			Err:        errors.New("unknown exception"),
 			StatusCode: 404,
 		}
 		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -108,7 +109,7 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		require.NotNil(t, err)
 		assert.Equal(t, want.Error(), err.Error())
 	})
@@ -122,9 +123,35 @@ func TestGetMessage(t *testing.T) {
 		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
 		require.NoError(t, err)
 
-		_, err = client.GetMessage(ctx, server.URL, token, 0)
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 10)
 		assert.NotNil(t, err)
 	})
+
+	t.Run("Capacity error handling", func(t *testing.T) {
+		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hc := r.Header.Get(actions.HeaderScaleSetMaxCapacity)
+			c, err := strconv.Atoi(hc)
+			require.NoError(t, err)
+			assert.GreaterOrEqual(t, c, 0)
+
+			w.WriteHeader(http.StatusBadRequest)
+			w.Header().Set("Content-Type", "text/plain")
+		}))
+
+		client, err := actions.NewClient(server.configURLForOrg("my-org"), auth)
+		require.NoError(t, err)
+
+		_, err = client.GetMessage(ctx, server.URL, token, 0, -1)
+		require.Error(t, err)
+		// Ensure we don't send requests with negative capacity
+		assert.False(t, errors.Is(err, &actions.ActionsError{}))
+
+		_, err = client.GetMessage(ctx, server.URL, token, 0, 0)
+		assert.Error(t, err)
+		var expectedErr *actions.ActionsError
+		assert.ErrorAs(t, err, &expectedErr)
+		assert.Equal(t, http.StatusBadRequest, expectedErr.StatusCode)
+	})
 }
 
 func TestDeleteMessage(t *testing.T) {

github/actions/client_runner_scale_set_session_test.go

@@ -13,6 +13,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const exampleRequestID = "5ddf2050-dae0-013c-9159-04421ad31b68"
+
 func TestCreateMessageSession(t *testing.T) {
 	ctx := context.Background()
 	auth := &actions.ActionsAuth{
@@ -69,13 +71,17 @@ func TestCreateMessageSession(t *testing.T) {
 		}
 
 		want := &actions.ActionsError{
+			ActivityID: exampleRequestID,
+			StatusCode: http.StatusBadRequest,
+			Err: &actions.ActionsExceptionError{
 				ExceptionName: "CSharpExceptionNameHere",
 				Message:       "could not do something",
-			StatusCode:    http.StatusBadRequest,
+			},
 		}
 
 		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set(actions.HeaderActionsActivityID, exampleRequestID)
 			w.WriteHeader(http.StatusBadRequest)
 			resp := []byte(`{"typeName": "CSharpExceptionNameHere","message": "could not do something"}`)
 			w.Write(resp)

github/actions/client_runner_scale_set_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/actions/actions-runner-controller/github/actions"
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -124,9 +125,15 @@ func TestGetRunnerScaleSet(t *testing.T) {
 	})
 
 	t.Run("Multiple runner scale sets found", func(t *testing.T) {
-		wantErr := fmt.Errorf("multiple runner scale sets found with name %s", scaleSetName)
+		reqID := uuid.NewString()
+		wantErr := &actions.ActionsError{
+			StatusCode: http.StatusOK,
+			ActivityID: reqID,
+			Err:        fmt.Errorf("multiple runner scale sets found with name %q", scaleSetName),
+		}
 		runnerScaleSetsResp := []byte(`{"count":2,"value":[{"id":1,"name":"ScaleSet"}]}`)
 		server := newActionsServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			w.Header().Set(actions.HeaderActionsActivityID, reqID)
 			w.Write(runnerScaleSetsResp)
 		}))
 

github/actions/errors.go

@@ -2,63 +2,117 @@ package actions
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 )
 
-type ActionsError struct {
-	ExceptionName string `json:"typeName,omitempty"`
-	Message       string `json:"message,omitempty"`
+// Header names for request IDs
+const (
+	HeaderActionsActivityID = "ActivityId"
+	HeaderGitHubRequestID   = "X-GitHub-Request-Id"
+)
+
+type GitHubAPIError struct {
 	StatusCode int
+	RequestID  string
+	Err        error
+}
+
+func (e *GitHubAPIError) Error() string {
+	return fmt.Sprintf("github api error: StatusCode %d, RequestID %q: %v", e.StatusCode, e.RequestID, e.Err)
+}
+
+func (e *GitHubAPIError) Unwrap() error {
+	return e.Err
+}
+
+type ActionsError struct {
+	ActivityID string
+	StatusCode int
+	Err        error
 }
 
 func (e *ActionsError) Error() string {
-	return fmt.Sprintf("%v - had issue communicating with Actions backend: %v", e.StatusCode, e.Message)
+	return fmt.Sprintf("actions error: StatusCode %d, AcivityId %q: %v", e.StatusCode, e.ActivityID, e.Err)
+}
+
+func (e *ActionsError) Unwrap() error {
+	return e.Err
+}
+
+func (e *ActionsError) IsException(target string) bool {
+	if ex, ok := e.Err.(*ActionsExceptionError); ok {
+		return strings.Contains(ex.ExceptionName, target)
+	}
+	return false
+}
+
+type ActionsExceptionError struct {
+	ExceptionName string `json:"typeName,omitempty"`
+	Message       string `json:"message,omitempty"`
+}
+
+func (e *ActionsExceptionError) Error() string {
+	return fmt.Sprintf("%s: %s", e.ExceptionName, e.Message)
 }
 
 func ParseActionsErrorFromResponse(response *http.Response) error {
 	if response.ContentLength == 0 {
-		message := "Request returned status: " + response.Status
 		return &ActionsError{
-			ExceptionName: "unknown",
-			Message:       message,
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
 			StatusCode: response.StatusCode,
+			Err:        errors.New("unknown exception"),
 		}
 	}
 
 	defer response.Body.Close()
 	body, err := io.ReadAll(response.Body)
 	if err != nil {
-		return err
+		return &ActionsError{
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
+			StatusCode: response.StatusCode,
+			Err:        err,
+		}
 	}
 
 	body = trimByteOrderMark(body)
 	contentType, ok := response.Header["Content-Type"]
 	if ok && len(contentType) > 0 && strings.Contains(contentType[0], "text/plain") {
 		message := string(body)
-		statusCode := response.StatusCode
 		return &ActionsError{
-			Message:    message,
-			StatusCode: statusCode,
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
+			StatusCode: response.StatusCode,
+			Err:        errors.New(message),
 		}
 	}
 
-	actionsError := &ActionsError{StatusCode: response.StatusCode}
-	if err := json.Unmarshal(body, &actionsError); err != nil {
-		return err
+	var exception ActionsExceptionError
+	if err := json.Unmarshal(body, &exception); err != nil {
+		return &ActionsError{
+			ActivityID: response.Header.Get(HeaderActionsActivityID),
+			StatusCode: response.StatusCode,
+			Err:        err,
+		}
 	}
 
-	return actionsError
+	return &ActionsError{
+		ActivityID: response.Header.Get(HeaderActionsActivityID),
+		StatusCode: response.StatusCode,
+		Err:        &exception,
+	}
 }
 
 type MessageQueueTokenExpiredError struct {
+	activityID string
+	statusCode int
 	msg        string
 }
 
 func (e *MessageQueueTokenExpiredError) Error() string {
-	return e.msg
+	return fmt.Sprintf("MessageQueueTokenExpiredError: AcivityId %q, StatusCode %d: %s", e.activityID, e.statusCode, e.msg)
 }
 
 type HttpClientSideError struct {

github/actions/errors_test.go

@@ -0,0 +1,206 @@
+package actions_test
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/actions/actions-runner-controller/github/actions"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestActionsError(t *testing.T) {
+	t.Run("contains the status code, activity ID, and error", func(t *testing.T) {
+		err := &actions.ActionsError{
+			ActivityID: "activity-id",
+			StatusCode: 404,
+			Err:        errors.New("example error description"),
+		}
+
+		s := err.Error()
+		assert.Contains(t, s, "StatusCode 404")
+		assert.Contains(t, s, "AcivityId \"activity-id\"")
+		assert.Contains(t, s, "example error description")
+	})
+
+	t.Run("unwraps the error", func(t *testing.T) {
+		err := &actions.ActionsError{
+			ActivityID: "activity-id",
+			StatusCode: 404,
+			Err: &actions.ActionsExceptionError{
+				ExceptionName: "exception-name",
+				Message:       "example error message",
+			},
+		}
+
+		assert.Equal(t, err.Unwrap(), err.Err)
+	})
+
+	t.Run("is exception is ok", func(t *testing.T) {
+		err := &actions.ActionsError{
+			ActivityID: "activity-id",
+			StatusCode: 404,
+			Err: &actions.ActionsExceptionError{
+				ExceptionName: "exception-name",
+				Message:       "example error message",
+			},
+		}
+
+		var exception *actions.ActionsExceptionError
+		assert.True(t, errors.As(err, &exception))
+
+		assert.True(t, err.IsException("exception-name"))
+	})
+
+	t.Run("is exception is not ok", func(t *testing.T) {
+		tt := map[string]*actions.ActionsError{
+			"not an exception": {
+				ActivityID: "activity-id",
+				StatusCode: 404,
+				Err:        errors.New("example error description"),
+			},
+			"not target exception": {
+				ActivityID: "activity-id",
+				StatusCode: 404,
+				Err: &actions.ActionsExceptionError{
+					ExceptionName: "exception-name",
+					Message:       "example error message",
+				},
+			},
+		}
+
+		targetException := "target-exception"
+		for name, err := range tt {
+			t.Run(name, func(t *testing.T) {
+				assert.False(t, err.IsException(targetException))
+			})
+		}
+	})
+}
+
+func TestActionsExceptionError(t *testing.T) {
+	t.Run("contains the exception name and message", func(t *testing.T) {
+		err := &actions.ActionsExceptionError{
+			ExceptionName: "exception-name",
+			Message:       "example error message",
+		}
+
+		s := err.Error()
+		assert.Contains(t, s, "exception-name")
+		assert.Contains(t, s, "example error message")
+	})
+}
+
+func TestGitHubAPIError(t *testing.T) {
+	t.Run("contains the status code, request ID, and error", func(t *testing.T) {
+		err := &actions.GitHubAPIError{
+			StatusCode: 404,
+			RequestID:  "request-id",
+			Err:        errors.New("example error description"),
+		}
+
+		s := err.Error()
+		assert.Contains(t, s, "StatusCode 404")
+		assert.Contains(t, s, "RequestID \"request-id\"")
+		assert.Contains(t, s, "example error description")
+	})
+
+	t.Run("unwraps the error", func(t *testing.T) {
+		err := &actions.GitHubAPIError{
+			StatusCode: 404,
+			RequestID:  "request-id",
+			Err:        errors.New("example error description"),
+		}
+
+		assert.Equal(t, err.Unwrap(), err.Err)
+	})
+}
+
+func ParseActionsErrorFromResponse(t *testing.T) {
+	t.Run("empty content length", func(t *testing.T) {
+		response := &http.Response{
+			ContentLength: 0,
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+			},
+			StatusCode: 404,
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+		assert.Equal(t, err.(*actions.ActionsError).ActivityID, "activity-id")
+		assert.Equal(t, err.(*actions.ActionsError).StatusCode, 404)
+		assert.Equal(t, err.(*actions.ActionsError).Err.Error(), "unknown exception")
+	})
+
+	t.Run("contains text plain error", func(t *testing.T) {
+		errorMessage := "example error message"
+		response := &http.Response{
+			ContentLength: int64(len(errorMessage)),
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+				"Content-Type":                  []string{"text/plain"},
+			},
+			StatusCode: 404,
+			Body:       io.NopCloser(strings.NewReader(errorMessage)),
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+		var actionsError *actions.ActionsError
+		assert.ErrorAs(t, err, &actionsError)
+		assert.Equal(t, actionsError.ActivityID, "activity-id")
+		assert.Equal(t, actionsError.StatusCode, 404)
+		assert.Equal(t, actionsError.Err.Error(), errorMessage)
+	})
+
+	t.Run("contains json error", func(t *testing.T) {
+		errorMessage := `{"typeName":"exception-name","message":"example error message"}`
+		response := &http.Response{
+			ContentLength: int64(len(errorMessage)),
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+				"Content-Type":                  []string{"application/json"},
+			},
+			StatusCode: 404,
+			Body:       io.NopCloser(strings.NewReader(errorMessage)),
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+		var actionsError *actions.ActionsError
+		assert.ErrorAs(t, err, &actionsError)
+		assert.Equal(t, actionsError.ActivityID, "activity-id")
+		assert.Equal(t, actionsError.StatusCode, 404)
+
+		inner, ok := actionsError.Err.(*actions.ActionsExceptionError)
+		require.True(t, ok)
+		assert.Equal(t, inner.ExceptionName, "exception-name")
+		assert.Equal(t, inner.Message, "example error message")
+	})
+
+	t.Run("wrapped exception error", func(t *testing.T) {
+		errorMessage := `{"typeName":"exception-name","message":"example error message"}`
+		response := &http.Response{
+			ContentLength: int64(len(errorMessage)),
+			Header: http.Header{
+				actions.HeaderActionsActivityID: []string{"activity-id"},
+				"Content-Type":                  []string{"application/json"},
+			},
+			StatusCode: 404,
+			Body:       io.NopCloser(strings.NewReader(errorMessage)),
+		}
+
+		err := actions.ParseActionsErrorFromResponse(response)
+		require.Error(t, err)
+
+		var actionsExceptionError *actions.ActionsExceptionError
+		assert.ErrorAs(t, err, &actionsExceptionError)
+
+		assert.Equal(t, actionsExceptionError.ExceptionName, "exception-name")
+		assert.Equal(t, actionsExceptionError.Message, "example error message")
+	})
+}

github/actions/fake/client.go

@@ -259,7 +259,7 @@ func (f *FakeClient) GetAcquirableJobs(ctx context.Context, runnerScaleSetId int
 	return f.getAcquirableJobsResult.AcquirableJobList, f.getAcquirableJobsResult.err
 }
 
-func (f *FakeClient) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64) (*actions.RunnerScaleSetMessage, error) {
+func (f *FakeClient) GetMessage(ctx context.Context, messageQueueUrl, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*actions.RunnerScaleSetMessage, error) {
 	return f.getMessageResult.RunnerScaleSetMessage, f.getMessageResult.err
 }
 

github/actions/github_api_request_test.go

@@ -139,7 +139,13 @@ func TestNewActionsServiceRequest(t *testing.T) {
 				w.WriteHeader(http.StatusUnauthorized)
 				w.Write([]byte(errMessage))
 			}
-			server := testserver.New(t, nil, testserver.WithActionsToken("random-token"), testserver.WithActionsToken(newToken), testserver.WithActionsRegistrationTokenHandler(unauthorizedHandler))
+			server := testserver.New(
+				t,
+				nil,
+				testserver.WithActionsToken("random-token"),
+				testserver.WithActionsToken(newToken),
+				testserver.WithActionsRegistrationTokenHandler(unauthorizedHandler),
+			)
 			client, err := actions.NewClient(server.ConfigURLForOrg("my-org"), defaultCreds)
 			require.NoError(t, err)
 			expiringToken := "expiring-token"

github/actions/mock_ActionsService.go

@@ -186,25 +186,25 @@ func (_m *MockActionsService) GetAcquirableJobs(ctx context.Context, runnerScale
 	return r0, r1
 }
 
-// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId
-func (_m *MockActionsService) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64) (*RunnerScaleSetMessage, error) {
-	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+// GetMessage provides a mock function with given fields: ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity
+func (_m *MockActionsService) GetMessage(ctx context.Context, messageQueueUrl string, messageQueueAccessToken string, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error) {
+	ret := _m.Called(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 
 	var r0 *RunnerScaleSetMessage
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) (*RunnerScaleSetMessage, error)); ok {
-		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) (*RunnerScaleSetMessage, error)); ok {
+		return rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) *RunnerScaleSetMessage); ok {
-		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int) *RunnerScaleSetMessage); ok {
+		r0 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*RunnerScaleSetMessage)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64) error); ok {
-		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId)
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int) error); ok {
+		r1 = rf(ctx, messageQueueUrl, messageQueueAccessToken, lastMessageId, maxCapacity)
 	} else {
 		r1 = ret.Error(1)
 	}

github/actions/mock_SessionService.go

@@ -67,25 +67,25 @@ func (_m *MockSessionService) DeleteMessage(ctx context.Context, messageId int64
 	return r0
 }
 
-// GetMessage provides a mock function with given fields: ctx, lastMessageId
-func (_m *MockSessionService) GetMessage(ctx context.Context, lastMessageId int64) (*RunnerScaleSetMessage, error) {
-	ret := _m.Called(ctx, lastMessageId)
+// GetMessage provides a mock function with given fields: ctx, lastMessageId, maxCapacity
+func (_m *MockSessionService) GetMessage(ctx context.Context, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error) {
+	ret := _m.Called(ctx, lastMessageId, maxCapacity)
 
 	var r0 *RunnerScaleSetMessage
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, int64) (*RunnerScaleSetMessage, error)); ok {
-		return rf(ctx, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, int64, int) (*RunnerScaleSetMessage, error)); ok {
+		return rf(ctx, lastMessageId, maxCapacity)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, int64) *RunnerScaleSetMessage); ok {
-		r0 = rf(ctx, lastMessageId)
+	if rf, ok := ret.Get(0).(func(context.Context, int64, int) *RunnerScaleSetMessage); ok {
+		r0 = rf(ctx, lastMessageId, maxCapacity)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*RunnerScaleSetMessage)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
-		r1 = rf(ctx, lastMessageId)
+	if rf, ok := ret.Get(1).(func(context.Context, int64, int) error); ok {
+		r1 = rf(ctx, lastMessageId, maxCapacity)
 	} else {
 		r1 = ret.Error(1)
 	}

github/actions/sessionservice.go

@@ -7,7 +7,7 @@ import (
 
 //go:generate mockery --inpackage --name=SessionService
 type SessionService interface {
-	GetMessage(ctx context.Context, lastMessageId int64) (*RunnerScaleSetMessage, error)
+	GetMessage(ctx context.Context, lastMessageId int64, maxCapacity int) (*RunnerScaleSetMessage, error)
 	DeleteMessage(ctx context.Context, messageId int64) error
 	AcquireJobs(ctx context.Context, requestIds []int64) ([]int64, error)
 	io.Closer

go.mod

@@ -5,7 +5,7 @@ go 1.22.1
 require (
 	github.com/bradleyfalzon/ghinstallation/v2 v2.8.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/evanphx/json-patch v5.7.0+incompatible
+	github.com/evanphx/json-patch v5.9.0+incompatible
 	github.com/go-logr/logr v1.4.1
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/go-cmp v0.6.0
@@ -17,15 +17,15 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/ginkgo/v2 v2.13.1
+	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.30.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.17.0
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/teambition/rrule-go v1.8.2
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.26.0
-	golang.org/x/net v0.19.0
+	golang.org/x/net v0.24.0
 	golang.org/x/oauth2 v0.15.0
 	golang.org/x/sync v0.6.0
 	gomodules.xyz/jsonpatch/v2 v2.4.0
@@ -43,7 +43,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boombuler/barcode v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cloudflare/circl v1.3.6 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
@@ -87,15 +87,15 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.5.1 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/urfave/cli v1.22.2 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.4.0 // indirect
-	golang.org/x/tools v0.15.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -19,8 +19,8 @@ github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7N
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
-github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg=
-github.com/cloudflare/circl v1.3.6/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
+github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
+github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -30,8 +30,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
-github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls=
+github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
 github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -169,8 +169,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.13.1 h1:LNGfMbR2OVGBfXjvRZIZ2YCTQdGKtPLvuI1rMCCj3OU=
-github.com/onsi/ginkgo/v2 v2.13.1/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
+github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8=
+github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
@@ -202,8 +202,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0=
-github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -212,9 +212,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/teambition/rrule-go v1.8.2 h1:lIjpjvWTj9fFUZCmuoVDrKVOtdiyzbzc93qTmRVe/J8=
 github.com/teambition/rrule-go v1.8.2/go.mod h1:Ieq5AbrKGciP1V//Wq8ktsTXwSwJHDD5mD/wLBGl3p4=
 github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
@@ -234,16 +233,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -256,8 +253,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
 golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -289,15 +286,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
+golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -316,8 +313,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
-golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

runner/Makefile

@@ -6,8 +6,8 @@ DIND_ROOTLESS_RUNNER_NAME ?= ${DOCKER_USER}/actions-runner-dind-rootless
 OS_IMAGE ?= ubuntu-22.04
 TARGETPLATFORM ?= $(shell arch)
 
-RUNNER_VERSION ?= 2.314.1
-RUNNER_CONTAINER_HOOKS_VERSION ?= 0.5.1
+RUNNER_VERSION ?= 2.315.0
+RUNNER_CONTAINER_HOOKS_VERSION ?= 0.6.0
 DOCKER_VERSION ?= 24.0.7
 
 # default list of platforms for which multiarch image is built

runner/VERSION

@@ -1,2 +1,2 @@
-RUNNER_VERSION=2.314.1
-RUNNER_CONTAINER_HOOKS_VERSION=0.5.1
+RUNNER_VERSION=2.315.0
+RUNNER_CONTAINER_HOOKS_VERSION=0.6.0

test/e2e/e2e_test.go

@@ -36,8 +36,8 @@ var (
 
 	testResultCMNamePrefix = "test-result-"
 
-	RunnerVersion               = "2.314.1"
-	RunnerContainerHooksVersion = "0.5.1"
+	RunnerVersion               = "2.315.0"
+	RunnerContainerHooksVersion = "0.6.0"
 )
 
 // If you're willing to run this test via VS Code "run test" or "debug test",