Prepare v2.314.0 release (#3172)

* Prepare v2.314.0 release

* update releaseNoteMd

.devcontainer/devcontainer.json

@@ -4,10 +4,13 @@
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:1": {},
     "ghcr.io/devcontainers/features/dotnet": {
-      "version": "6.0.418"
+      "version": "6.0.419"
     },
     "ghcr.io/devcontainers/features/node:1": {
       "version": "16"
+    },
+    "ghcr.io/devcontainers/features/sshd:1": {
+      "version": "latest"
     }
   },
   "customizations": {

.github/workflows/release.yml

@@ -163,7 +163,6 @@ jobs:
           core.setOutput('note', releaseNote);
 
     - name: Validate Packages HASH
-      working-directory: _package
       run: |
         ls -l
         echo "${{needs.build.outputs.win-x64-sha}}  actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip" | shasum -a 256 -c
@@ -193,7 +192,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
+        asset_path: ${{ github.workspace }}/actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
         asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip
         asset_content_type: application/octet-stream
 
@@ -203,7 +202,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
+        asset_path: ${{ github.workspace }}/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
         asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip
         asset_content_type: application/octet-stream
 
@@ -213,7 +212,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -223,7 +222,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -233,7 +232,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -243,7 +242,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-arm-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 
@@ -253,7 +252,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ steps.createRelease.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/_package/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
+        asset_path: ${{ github.workspace }}/actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz
         asset_content_type: application/octet-stream
 

images/Dockerfile

@@ -4,9 +4,9 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build
 ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.5.0
-ARG DOCKER_VERSION=24.0.6
-ARG BUILDX_VERSION=0.11.2
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.5.1
+ARG DOCKER_VERSION=25.0.2
+ARG BUILDX_VERSION=0.12.1
 
 RUN apt update -y && apt install curl unzip -y
 

releaseNote.md

@@ -1,29 +1,23 @@
 ## What's Changed
-* Fix `buildx` installation by @ajschmidt8 in https://github.com/actions/runner/pull/2952
-* Create close-features and close-bugs bot for runner issues by @ruvceskistefan in https://github.com/actions/runner/pull/2909
-* Send disableUpdate as query parameter  by @luketomlinson in https://github.com/actions/runner/pull/2970
-* Handle SelfUpdate Flow when Package is provided in Message  by @luketomlinson in https://github.com/actions/runner/pull/2926
-* Bump container hook version to 0.5.0 in runner image by @nikola-jokic in https://github.com/actions/runner/pull/3003
-* Set `ImageOS` environment variable in runner images by @int128 in https://github.com/actions/runner/pull/2878
-* Mark job as failed on worker crash. by @TingluoHuang in https://github.com/actions/runner/pull/3006
-* Include whether http proxy configured as part of UserAgent. by @TingluoHuang in https://github.com/actions/runner/pull/3009
-* Add codeload to the list of service we check during '--check'. by @TingluoHuang in https://github.com/actions/runner/pull/3011
-* close reason update by @ruvceskistefan in https://github.com/actions/runner/pull/3027
-* Update envlinux.md by @adjn in https://github.com/actions/runner/pull/3040
-* Extend `--check` to check Results-Receiver service. by @TingluoHuang in https://github.com/actions/runner/pull/3078
-* Use Azure SDK to upload files to Azure Blob by @yacaovsnc in https://github.com/actions/runner/pull/3033
-* Remove code in runner for handling trimmed packages. by @TingluoHuang in https://github.com/actions/runner/pull/3074
-* Update dotnet sdk to latest version @6.0.418 by @github-actions in https://github.com/actions/runner/pull/3085
-* Patch Curl to no longer use -k by @thboop in https://github.com/actions/runner/pull/3091
+* Prepare v2.313.0 Release by @luketomlinson in https://github.com/actions/runner/pull/3137
+* Pass RunnerOS during job acquire. by @TingluoHuang in https://github.com/actions/runner/pull/3140
+* Process `snapshot` tokens by @davidomid in https://github.com/actions/runner/pull/3135
+* Update dotnet sdk to latest version @6.0.419 by @github-actions in https://github.com/actions/runner/pull/3158
+* handle broker run service exception handling by @yaananth in https://github.com/actions/runner/pull/3163
+* Add a retry logic to docker login operation by @enescakir in https://github.com/actions/runner/pull/3089
+* Broker fixes for token refreshes and AccessDeniedException by @luketomlinson in https://github.com/actions/runner/pull/3161
+* Remove USE_BROKER_FLOW by @luketomlinson in https://github.com/actions/runner/pull/3162
+* Refresh Token for BrokerServer by @luketomlinson in https://github.com/actions/runner/pull/3167
+* Better step timeout message. by @TingluoHuang in https://github.com/actions/runner/pull/3166
 
 ## New Contributors
-* @int128 made their first contribution in https://github.com/actions/runner/pull/2878
-* @adjn made their first contribution in https://github.com/actions/runner/pull/3040
+* @davidomid made their first contribution in https://github.com/actions/runner/pull/3135
+* @enescakir made their first contribution in https://github.com/actions/runner/pull/3089
 
-**Full Changelog**: https://github.com/actions/runner/compare/v2.311.0...v2.312.0
+**Full Changelog**: https://github.com/actions/runner/compare/v2.313.0...v2.314.0
 
-_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. 
-To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. 
+_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet.
+To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.
 See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_
 
 ## Windows x64

src/Runner.Common/BrokerServer.cs

@@ -17,7 +17,14 @@ namespace GitHub.Runner.Common
     {
         Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
 
-        Task<TaskAgentMessage> GetRunnerMessageAsync(CancellationToken token, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate);
+        Task<TaskAgentSession> CreateSessionAsync(TaskAgentSession session, CancellationToken cancellationToken);
+        Task DeleteSessionAsync(CancellationToken cancellationToken);
+
+        Task<TaskAgentMessage> GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken token);
+
+        Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials);
+
+        Task ForceRefreshConnection(VssCredentials credentials);
     }
 
     public sealed class BrokerServer : RunnerService, IBrokerServer
@@ -44,13 +51,53 @@ namespace GitHub.Runner.Common
             }
         }
 
-        public Task<TaskAgentMessage> GetRunnerMessageAsync(CancellationToken cancellationToken, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate)
+        public async Task<TaskAgentSession> CreateSessionAsync(TaskAgentSession session, CancellationToken cancellationToken)
         {
             CheckConnection();
-            var jobMessage = RetryRequest<TaskAgentMessage>(
-                async () => await _brokerHttpClient.GetRunnerMessageAsync(version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken);
+            var jobMessage = await _brokerHttpClient.CreateSessionAsync(session, cancellationToken);
 
             return jobMessage;
         }
+
+        public Task<TaskAgentMessage> GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken)
+        {
+            CheckConnection();
+            var brokerSession = RetryRequest<TaskAgentMessage>(
+                async () => await _brokerHttpClient.GetRunnerMessageAsync(sessionId, version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken, shouldRetry: ShouldRetryException);
+
+
+            return brokerSession;
+        }
+
+        public async Task DeleteSessionAsync(CancellationToken cancellationToken)
+        {
+            CheckConnection();
+            await _brokerHttpClient.DeleteSessionAsync(cancellationToken);
+        }
+
+        public Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials)
+        {
+            if (_brokerUri != serverUri || !_hasConnection)
+            {
+                return ConnectAsync(serverUri, credentials);
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task ForceRefreshConnection(VssCredentials credentials)
+        {
+            return ConnectAsync(_brokerUri, credentials);
+        }
+
+        public bool ShouldRetryException(Exception ex)
+        {
+            if (ex is AccessDeniedException ade && ade.ErrorCode == 1)
+            {
+                return false;
+            }
+
+            return true;
+        }
     }
 }

src/Runner.Common/JobServerQueue.cs

@@ -551,6 +551,10 @@ namespace GitHub.Runner.Common
                             {
                                 await UploadSummaryFile(file);
                             }
+                            if (string.Equals(file.Type, CoreAttachmentType.ResultsDiagnosticLog, StringComparison.OrdinalIgnoreCase))
+                            {
+                                await UploadResultsDiagnosticLogsFile(file);
+                            }
                             else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase))
                             {
                                 if (file.RecordId != _jobTimelineRecordId)
@@ -922,6 +926,17 @@ namespace GitHub.Runner.Common
             await UploadResultsFile(file, summaryHandler);
         }
 
+        private async Task UploadResultsDiagnosticLogsFile(ResultsUploadFileInfo file)
+        {
+            Trace.Info($"Starting to upload diagnostic logs file to results service {file.Name}, {file.Path}");
+            ResultsFileUploadHandler diagnosticLogsHandler = async (file) =>
+            {
+                await _resultsServer.CreateResultsDiagnosticLogsAsync(file.PlanId, file.JobId, file.Path, CancellationToken.None);
+            };
+
+            await UploadResultsFile(file, diagnosticLogsHandler);
+        }
+
         private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file)
         {
             Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}");

src/Runner.Common/ResultsServer.cs

@@ -35,6 +35,8 @@ namespace GitHub.Runner.Common
 
         Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId,
             IEnumerable<TimelineRecord> records, CancellationToken cancellationToken);
+
+        Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file, CancellationToken cancellationToken);
     }
 
     public sealed class ResultServer : RunnerService, IResultsServer
@@ -141,6 +143,18 @@ namespace GitHub.Runner.Common
             throw new InvalidOperationException("Results client is not initialized.");
         }
 
+        public Task CreateResultsDiagnosticLogsAsync(string planId, string jobId, string file,
+            CancellationToken cancellationToken)
+        {
+            if (_resultsClient != null)
+            {
+                return _resultsClient.UploadResultsDiagnosticLogsAsync(planId, jobId, file,
+                    cancellationToken: cancellationToken);
+            }
+
+            throw new InvalidOperationException("Results client is not initialized.");
+        }
+
         public ValueTask DisposeAsync()
         {
             CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);

src/Runner.Common/RunServer.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using GitHub.Actions.RunService.WebApi;
 using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using Sdk.RSWebApi.Contracts;
@@ -60,7 +61,7 @@ namespace GitHub.Runner.Common
         {
             CheckConnection();
             return RetryRequest<AgentJobRequestMessage>(
-                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken,
+                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, cancellationToken), cancellationToken,
                 shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException);
         }
 

src/Runner.Listener/BrokerMessageListener.cs

@@ -24,7 +24,15 @@ namespace GitHub.Runner.Listener
         private TimeSpan _getNextMessageRetryInterval;
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
+        private VssCredentials _creds;
+        private TaskAgentSession _session;
         private IBrokerServer _brokerServer;
+        private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new();
+        private bool _accessTokenRevoked = false;
+        private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30);
+        private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4);
+        private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30);
+
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -36,13 +44,134 @@ namespace GitHub.Runner.Listener
 
         public async Task<Boolean> CreateSessionAsync(CancellationToken token)
         {
-            await RefreshBrokerConnection();
-            return await Task.FromResult(true);
+            Trace.Entering();
+
+            // Settings
+            var configManager = HostContext.GetService<IConfigurationManager>();
+            _settings = configManager.LoadSettings();
+            var serverUrl = _settings.ServerUrlV2;
+            Trace.Info(_settings);
+
+            if (string.IsNullOrEmpty(_settings.ServerUrlV2))
+            {
+                throw new InvalidOperationException("ServerUrlV2 is not set");
+            }
+
+            // Create connection.
+            Trace.Info("Loading Credentials");
+            var credMgr = HostContext.GetService<ICredentialManager>();
+            _creds = credMgr.LoadCredentials();
+
+            var agent = new TaskAgentReference
+            {
+                Id = _settings.AgentId,
+                Name = _settings.AgentName,
+                Version = BuildConstants.RunnerPackage.Version,
+                OSDescription = RuntimeInformation.OSDescription,
+            };
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
+            var taskAgentSession = new TaskAgentSession(sessionName, agent);
+
+            string errorMessage = string.Empty;
+            bool encounteringError = false;
+
+            while (true)
+            {
+                token.ThrowIfCancellationRequested();
+                Trace.Info($"Attempt to create session.");
+                try
+                {
+                    Trace.Info("Connecting to the Broker Server...");
+                    await _brokerServer.ConnectAsync(new Uri(serverUrl), _creds);
+                    Trace.Info("VssConnection created");
+
+                    _term.WriteLine();
+                    _term.WriteSuccessMessage("Connected to GitHub");
+                    _term.WriteLine();
+
+                    _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
+
+                    Trace.Info($"Session created.");
+                    if (encounteringError)
+                    {
+                        _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
+                        _sessionCreationExceptionTracker.Clear();
+                        encounteringError = false;
+                    }
+
+                    return true;
+                }
+                catch (OperationCanceledException) when (token.IsCancellationRequested)
+                {
+                    Trace.Info("Session creation has been cancelled.");
+                    throw;
+                }
+                catch (TaskAgentAccessTokenExpiredException)
+                {
+                    Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
+                    _accessTokenRevoked = true;
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    Trace.Error("Catch exception during create session.");
+                    Trace.Error(ex);
+
+                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _creds.Federated is VssOAuthCredential vssOAuthCred)
+                    {
+                        // "invalid_client" means the runner registration has been deleted from the server.
+                        if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
+                        {
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
+                            return false;
+                        }
+
+                        // Check whether we get 401 because the runner registration already removed by the service.
+                        // If the runner registration get deleted, we can't exchange oauth token.
+                        Trace.Error("Test oauth app registration.");
+                        var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrl));
+                        var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
+                        if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
+                        {
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
+                            return false;
+                        }
+                    }
+
+                    if (!IsSessionCreationExceptionRetriable(ex))
+                    {
+                        _term.WriteError($"Failed to create session. {ex.Message}");
+                        return false;
+                    }
+
+                    if (!encounteringError) //print the message only on the first error
+                    {
+                        _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
+                        encounteringError = true;
+                    }
+
+                    Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
+                    await HostContext.Delay(_sessionCreationRetryInterval, token);
+                }
+            }
         }
 
         public async Task DeleteSessionAsync()
         {
-            await Task.CompletedTask;
+            if (_session != null && _session.SessionId != Guid.Empty)
+            {
+                if (!_accessTokenRevoked)
+                {
+                    using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
+                    {
+                        await _brokerServer.DeleteSessionAsync(ts.Token);
+                    }
+                }
+                else
+                {
+                    Trace.Warning("Runner OAuth token has been revoked. Skip deleting session.");
+                }
+            }
         }
 
         public void OnJobStatus(object sender, JobStatusEventArgs e)
@@ -73,12 +202,13 @@ namespace GitHub.Runner.Listener
                 _getMessagesTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
                 try
                 {
-                    message = await _brokerServer.GetRunnerMessageAsync(_getMessagesTokenSource.Token,
+                    message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
                                                                         runnerStatus,
                                                                         BuildConstants.RunnerPackage.Version,
                                                                         VarUtil.OS,
                                                                         VarUtil.OSArchitecture,
-                                                                        _settings.DisableUpdate);
+                                                                        _settings.DisableUpdate,
+                                                                        _getMessagesTokenSource.Token);
 
                     if (message == null)
                     {
@@ -143,7 +273,7 @@ namespace GitHub.Runner.Listener
                         }
 
                         // re-create VssConnection before next retry
-                        await RefreshBrokerConnection();
+                        await RefreshBrokerConnectionAsync();
 
                         Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
                         await HostContext.Delay(_getNextMessageRetryInterval, token);
@@ -173,6 +303,11 @@ namespace GitHub.Runner.Listener
             }
         }
 
+        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        {
+            await RefreshBrokerConnectionAsync();
+        }
+
         public async Task DeleteMessageAsync(TaskAgentMessage message)
         {
             await Task.CompletedTask;
@@ -196,12 +331,84 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task RefreshBrokerConnection()
+        private bool IsSessionCreationExceptionRetriable(Exception ex)
+        {
+            if (ex is TaskAgentNotFoundException)
+            {
+                Trace.Info("The runner no longer exists on the server. Stopping the runner.");
+                _term.WriteError("The runner no longer exists on the server. Please reconfigure the runner.");
+                return false;
+            }
+            else if (ex is TaskAgentSessionConflictException)
+            {
+                Trace.Info("The session for this runner already exists.");
+                _term.WriteError("A session for this runner already exists.");
+                if (_sessionCreationExceptionTracker.ContainsKey(nameof(TaskAgentSessionConflictException)))
+                {
+                    _sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)]++;
+                    if (_sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)] * _sessionCreationRetryInterval.TotalSeconds >= _sessionConflictRetryLimit.TotalSeconds)
+                    {
+                        Trace.Info("The session conflict exception have reached retry limit.");
+                        _term.WriteError($"Stop retry on SessionConflictException after retried for {_sessionConflictRetryLimit.TotalSeconds} seconds.");
+                        return false;
+                    }
+                }
+                else
+                {
+                    _sessionCreationExceptionTracker[nameof(TaskAgentSessionConflictException)] = 1;
+                }
+
+                Trace.Info("The session conflict exception haven't reached retry limit.");
+                return true;
+            }
+            else if (ex is VssOAuthTokenRequestException && ex.Message.Contains("Current server time is"))
+            {
+                Trace.Info("Local clock might be skewed.");
+                _term.WriteError("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again.");
+                if (_sessionCreationExceptionTracker.ContainsKey(nameof(VssOAuthTokenRequestException)))
+                {
+                    _sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)]++;
+                    if (_sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)] * _sessionCreationRetryInterval.TotalSeconds >= _clockSkewRetryLimit.TotalSeconds)
+                    {
+                        Trace.Info("The OAuth token request exception have reached retry limit.");
+                        _term.WriteError($"Stopped retrying OAuth token request exception after {_clockSkewRetryLimit.TotalSeconds} seconds.");
+                        return false;
+                    }
+                }
+                else
+                {
+                    _sessionCreationExceptionTracker[nameof(VssOAuthTokenRequestException)] = 1;
+                }
+
+                Trace.Info("The OAuth token request exception haven't reached retry limit.");
+                return true;
+            }
+            else if (ex is TaskAgentPoolNotFoundException ||
+                     ex is AccessDeniedException ||
+                     ex is VssUnauthorizedException)
+            {
+                Trace.Info($"Non-retriable exception: {ex.Message}");
+                return false;
+            }
+
+            else if (ex is InvalidOperationException)
+            {
+                Trace.Info($"Non-retriable exception: {ex.Message}");
+                return false;
+            }
+            else
+            {
+                Trace.Info($"Retriable exception: {ex.Message}");
+                return true;
+            }
+        }
+
+        private async Task RefreshBrokerConnectionAsync()
         {
             var configManager = HostContext.GetService<IConfigurationManager>();
             _settings = configManager.LoadSettings();
 
-            if (_settings.ServerUrlV2 == null)
+            if (string.IsNullOrEmpty(_settings.ServerUrlV2))
             {
                 throw new InvalidOperationException("ServerUrlV2 is not set");
             }

src/Runner.Listener/JobDispatcher.cs

@@ -35,7 +35,7 @@ namespace GitHub.Runner.Listener
     // This implementation of IJobDispatcher is not thread safe.
     // It is based on the fact that the current design of the runner is a dequeue
     // and processes one message from the message queue at a time.
-    // In addition, it only executes one job every time, 
+    // In addition, it only executes one job every time,
     // and the server will not send another job while this one is still running.
     public sealed class JobDispatcher : RunnerService, IJobDispatcher
     {
@@ -546,13 +546,27 @@ namespace GitHub.Runner.Listener
                                     Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
 
                                     var jobServer = await InitializeJobServerAsync(systemConnection);
-                                    await LogWorkerProcessUnhandledException(jobServer, message, detailInfo);
-
-                                    // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
-                                    if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
+                                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
+                                    switch (jobServer)
                                     {
-                                        Trace.Info($"Finish job with result 'Failed' due to IOException.");
-                                        await ForceFailJob(jobServer, message, detailInfo);
+                                        case IJobServer js:
+                                            {
+                                                await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
+                                                // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
+                                                if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                                {
+                                                    Trace.Info($"Finish job with result 'Failed' due to IOException.");
+                                                    await ForceFailJob(js, message);
+                                                }
+
+                                                break;
+                                            }
+                                        case IRunServer rs:
+                                            await ForceFailJob(rs, message, unhandledExceptionIssue);
+                                            break;
+                                        default:
+                                            throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
                                     }
                                 }
 
@@ -629,8 +643,22 @@ namespace GitHub.Runner.Listener
                                     Trace.Info("worker process has been killed.");
                                 }
                             }
+                            catch (Exception ex)
+                            {
+                                // message send failed, this might indicate worker process is already exited or stuck.
+                                Trace.Info($"Job cancel message sending for job {message.JobId} failed, kill running worker. {ex}");
+                                workerProcessCancelTokenSource.Cancel();
+                                try
+                                {
+                                    await workerProcessTask;
+                                }
+                                catch (OperationCanceledException)
+                                {
+                                    Trace.Info("worker process has been killed.");
+                                }
+                            }
 
-                            // wait worker to exit 
+                            // wait worker to exit
                             // if worker doesn't exit within timeout, then kill worker.
                             completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
 
@@ -1117,86 +1145,70 @@ namespace GitHub.Runner.Listener
         }
 
         // log an error issue to job level timeline record
-        private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipelines.AgentJobRequestMessage message, Issue issue)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
+                var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
+                ArgUtil.NotNull(timeline, nameof(timeline));
+
+                TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
+                ArgUtil.NotNull(jobRecord, nameof(jobRecord));
+
+
+                jobRecord.ErrorCount++;
+                jobRecord.Issues.Add(issue);
+
+                if (message.Variables.TryGetValue("DistributedTask.MarkJobAsFailedOnWorkerCrash", out var markJobAsFailedOnWorkerCrash) &&
+                    StringUtil.ConvertToBoolean(markJobAsFailedOnWorkerCrash?.Value))
                 {
-                    var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
-                    ArgUtil.NotNull(timeline, nameof(timeline));
-
-                    TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
-                    ArgUtil.NotNull(jobRecord, nameof(jobRecord));
-
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
-                    jobRecord.ErrorCount++;
-                    jobRecord.Issues.Add(unhandledExceptionIssue);
-
-                    if (message.Variables.TryGetValue("DistributedTask.MarkJobAsFailedOnWorkerCrash", out var markJobAsFailedOnWorkerCrash) &&
-                        StringUtil.ConvertToBoolean(markJobAsFailedOnWorkerCrash?.Value))
-                    {
-                        Trace.Info("Mark the job as failed since the worker crashed");
-                        jobRecord.Result = TaskResult.Failed;
-                        // mark the job as completed so service will pickup the result
-                        jobRecord.State = TimelineRecordState.Completed;
-                    }
-
-                    await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to report unhandled exception from Runner.Worker process");
-                    Trace.Error(ex);
+                    Trace.Info("Mark the job as failed since the worker crashed");
+                    jobRecord.Result = TaskResult.Failed;
+                    // mark the job as completed so service will pickup the result
+                    jobRecord.State = TimelineRecordState.Completed;
                 }
+
+                await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
             }
-            else
+            catch (Exception ex)
             {
-                Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", detailInfo);
-                return;
+                Trace.Error("Fail to report unhandled exception from Runner.Worker process");
+                Trace.Error(ex);
             }
         }
 
         // raise job completed event to fail the job.
-        private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task ForceFailJob(IJobServer jobServer, Pipelines.AgentJobRequestMessage message)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
-                {
-                    var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
-                    await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise JobCompletedEvent back to service.");
-                    Trace.Error(ex);
-                }
+                var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
+                await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
             }
-            else if (server is IRunServer runServer)
+            catch (Exception ex)
             {
-                try
-                {
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    var unhandledAnnotation = unhandledExceptionIssue.ToAnnotation();
-                    var jobAnnotations = new List<Annotation>();
-                    if (unhandledAnnotation.HasValue)
-                    {
-                        jobAnnotations.Add(unhandledAnnotation.Value);
-                    }
+                Trace.Error("Fail to raise JobCompletedEvent back to service.");
+                Trace.Error(ex);
+            }
+        }
 
-                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise job completion back to service.");
-                    Trace.Error(ex);
-                }
-            }
-            else
+        private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestMessage message, Issue issue)
+        {
+            try
             {
-                throw new NotSupportedException($"Server type {server.GetType().FullName} is not supported.");
+                var annotation = issue.ToAnnotation();
+                var jobAnnotations = new List<Annotation>();
+                if (annotation.HasValue)
+                {
+                    jobAnnotations.Add(annotation.Value);
+                }
+
+                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("Fail to raise job completion back to service.");
+                Trace.Error(ex);
             }
         }
 

src/Runner.Listener/MessageListener.cs

@@ -14,6 +14,7 @@ using GitHub.Runner.Listener.Configuration;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.OAuth;
+using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
@@ -24,6 +25,8 @@ namespace GitHub.Runner.Listener
         Task DeleteSessionAsync();
         Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token);
         Task DeleteMessageAsync(TaskAgentMessage message);
+
+        Task RefreshListenerTokenAsync(CancellationToken token);
         void OnJobStatus(object sender, JobStatusEventArgs e);
     }
 
@@ -33,6 +36,7 @@ namespace GitHub.Runner.Listener
         private RunnerSettings _settings;
         private ITerminal _term;
         private IRunnerServer _runnerServer;
+        private IBrokerServer _brokerServer;
         private TaskAgentSession _session;
         private TimeSpan _getNextMessageRetryInterval;
         private bool _accessTokenRevoked = false;
@@ -42,6 +46,9 @@ namespace GitHub.Runner.Listener
         private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new();
         private TaskAgentStatus runnerStatus = TaskAgentStatus.Online;
         private CancellationTokenSource _getMessagesTokenSource;
+        private VssCredentials _creds;
+
+        private bool _isBrokerSession = false;
 
         public override void Initialize(IHostContext hostContext)
         {
@@ -49,6 +56,7 @@ namespace GitHub.Runner.Listener
 
             _term = HostContext.GetService<ITerminal>();
             _runnerServer = HostContext.GetService<IRunnerServer>();
+            _brokerServer = hostContext.GetService<IBrokerServer>();
         }
 
         public async Task<Boolean> CreateSessionAsync(CancellationToken token)
@@ -64,7 +72,7 @@ namespace GitHub.Runner.Listener
             // Create connection.
             Trace.Info("Loading Credentials");
             var credMgr = HostContext.GetService<ICredentialManager>();
-            VssCredentials creds = credMgr.LoadCredentials();
+            _creds = credMgr.LoadCredentials();
 
             var agent = new TaskAgentReference
             {
@@ -86,7 +94,7 @@ namespace GitHub.Runner.Listener
                 try
                 {
                     Trace.Info("Connecting to the Runner Server...");
-                    await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
+                    await _runnerServer.ConnectAsync(new Uri(serverUrl), _creds);
                     Trace.Info("VssConnection created");
 
                     _term.WriteLine();
@@ -98,6 +106,15 @@ namespace GitHub.Runner.Listener
                                                         taskAgentSession,
                                                         token);
 
+                    if (_session.BrokerMigrationMessage != null)
+                    {
+                        Trace.Info("Runner session is in migration mode: Creating Broker session with BrokerBaseUrl: {0}", _session.BrokerMigrationMessage.BrokerBaseUrl);
+
+                        await _brokerServer.UpdateConnectionIfNeeded(_session.BrokerMigrationMessage.BrokerBaseUrl, _creds);
+                        _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
+                        _isBrokerSession = true;
+                    }
+
                     Trace.Info($"Session created.");
                     if (encounteringError)
                     {
@@ -124,7 +141,7 @@ namespace GitHub.Runner.Listener
                     Trace.Error("Catch exception during create session.");
                     Trace.Error(ex);
 
-                    if (ex is VssOAuthTokenRequestException vssOAuthEx && creds.Federated is VssOAuthCredential vssOAuthCred)
+                    if (ex is VssOAuthTokenRequestException vssOAuthEx && _creds.Federated is VssOAuthCredential vssOAuthCred)
                     {
                         // "invalid_client" means the runner registration has been deleted from the server.
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
@@ -171,6 +188,11 @@ namespace GitHub.Runner.Listener
                 {
                     using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
                     {
+                        if (_isBrokerSession)
+                        {
+                            await _brokerServer.DeleteSessionAsync(ts.Token);
+                            return;
+                        }
                         await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token);
                     }
                 }
@@ -183,19 +205,17 @@ namespace GitHub.Runner.Listener
 
         public void OnJobStatus(object sender, JobStatusEventArgs e)
         {
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("USE_BROKER_FLOW")))
+            Trace.Info("Received job status event. JobState: {0}", e.Status);
+            runnerStatus = e.Status;
+            try
             {
-                Trace.Info("Received job status event. JobState: {0}", e.Status);
-                runnerStatus = e.Status;
-                try
-                {
-                    _getMessagesTokenSource?.Cancel();
-                }
-                catch (ObjectDisposedException)
-                {
-                    Trace.Info("_getMessagesTokenSource is already disposed.");
-                }
+                _getMessagesTokenSource?.Cancel();
             }
+            catch (ObjectDisposedException)
+            {
+                Trace.Info("_getMessagesTokenSource is already disposed.");
+            }
+
         }
 
         public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
@@ -228,6 +248,23 @@ namespace GitHub.Runner.Listener
                     // Decrypt the message body if the session is using encryption
                     message = DecryptMessage(message);
 
+
+                    if (message != null && message.MessageType == BrokerMigrationMessage.MessageType)
+                    {
+                        Trace.Info("BrokerMigration message received. Polling Broker for messages...");
+
+                        var migrationMessage = JsonUtility.FromString<BrokerMigrationMessage>(message.Body);
+
+                        await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _creds);
+                        message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
+                                                                        runnerStatus,
+                                                                        BuildConstants.RunnerPackage.Version,
+                                                                        VarUtil.OS,
+                                                                        VarUtil.OSArchitecture,
+                                                                        _settings.DisableUpdate,
+                                                                        token);
+                    }
+
                     if (message != null)
                     {
                         _lastMessageId = message.MessageId;
@@ -343,6 +380,12 @@ namespace GitHub.Runner.Listener
             }
         }
 
+        public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
+        {
+            await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+            await _brokerServer.ForceRefreshConnection(_creds);
+        }
+
         private TaskAgentMessage DecryptMessage(TaskAgentMessage message)
         {
             if (_session.EncryptionKey == null ||

src/Runner.Listener/Runner.cs

@@ -596,6 +596,11 @@ namespace GitHub.Runner.Listener
                                 Trace.Info($"Service requests the hosted runner to shutdown. Reason: '{HostedRunnerShutdownMessage.Reason}'.");
                                 return Constants.Runner.ReturnCode.Success;
                             }
+                            else if (string.Equals(message.MessageType, TaskAgentMessageTypes.ForceTokenRefresh))
+                            {
+                                Trace.Info("Received ForceTokenRefreshMessage");
+                                await _listener.RefreshListenerTokenAsync(messageQueueLoopTokenSource.Token);
+                            }
                             else
                             {
                                 Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
@@ -634,6 +639,7 @@ namespace GitHub.Runner.Listener
                     {
                         try
                         {
+                            Trace.Info("Deleting Runner Session...");
                             await _listener.DeleteSessionAsync();
                         }
                         catch (Exception ex) when (runOnce)

src/Runner.Listener/SelfUpdater.cs

@@ -34,6 +34,7 @@ namespace GitHub.Runner.Listener
         private IRunnerServer _runnerServer;
         private int _poolId;
         private ulong _agentId;
+        private const int _numberOfOldVersionsToKeep = 1;
         private readonly ConcurrentQueue<string> _updateTrace = new();
         public bool Busy { get; private set; }
 
@@ -509,9 +510,9 @@ namespace GitHub.Runner.Listener
 
             // delete old bin.2.99.0 folder, only leave the current version and the latest download version
             var allBinDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "bin.*");
-            if (allBinDirs.Length > 2)
+            if (allBinDirs.Length > _numberOfOldVersionsToKeep)
             {
-                // there are more than 2 bin.version folder.
+                // there are more than one bin.version folder.
                 // delete older bin.version folders.
                 foreach (var oldBinDir in allBinDirs)
                 {
@@ -538,9 +539,9 @@ namespace GitHub.Runner.Listener
 
             // delete old externals.2.99.0 folder, only leave the current version and the latest download version
             var allExternalsDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "externals.*");
-            if (allExternalsDirs.Length > 2)
+            if (allExternalsDirs.Length > _numberOfOldVersionsToKeep)
             {
-                // there are more than 2 externals.version folder.
+                // there are more than one externals.version folder.
                 // delete older externals.version folders.
                 foreach (var oldExternalDir in allExternalsDirs)
                 {

src/Runner.Sdk/Util/VssUtil.cs

@@ -23,7 +23,13 @@ namespace GitHub.Runner.Sdk
 
             if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0)
             {
-                headerValues.AddRange(VssClientHttpRequestSettings.Default.UserAgent);
+                foreach (var headerVal in VssClientHttpRequestSettings.Default.UserAgent)
+                {
+                    if (!headerValues.Contains(headerVal))
+                    {
+                        headerValues.Add(headerVal);
+                    }
+                }
             }
 
             VssClientHttpRequestSettings.Default.UserAgent = headerValues;
@@ -33,6 +39,23 @@ namespace GitHub.Runner.Sdk
             {
                 VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
             }
+
+            var rawHeaderValues = new List<ProductInfoHeaderValue>();
+            rawHeaderValues.AddRange(additionalUserAgents);
+            rawHeaderValues.Add(new ProductInfoHeaderValue($"({StringUtil.SanitizeUserAgentHeader(RuntimeInformation.OSDescription)})"));
+
+            if (RawClientHttpRequestSettings.Default.UserAgent != null && RawClientHttpRequestSettings.Default.UserAgent.Count > 0)
+            {
+                foreach (var headerVal in RawClientHttpRequestSettings.Default.UserAgent)
+                {
+                    if (!rawHeaderValues.Contains(headerVal))
+                    {
+                        rawHeaderValues.Add(headerVal);
+                    }
+                }
+            }
+
+            RawClientHttpRequestSettings.Default.UserAgent = rawHeaderValues;
         }
 
         public static VssConnection CreateConnection(
@@ -62,11 +85,6 @@ namespace GitHub.Runner.Sdk
                 settings.SendTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200));
             }
 
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("USE_BROKER_FLOW")))
-            {
-                settings.AllowAutoRedirectForBroker = true;
-            }
-
             // Remove Invariant from the list of accepted languages.
             //
             // The constructor of VssHttpRequestSettings (base class of VssClientHttpRequestSettings) adds the current

src/Runner.Worker/ActionManifestManager.cs

@@ -144,7 +144,7 @@ namespace GitHub.Runner.Worker
                     executionContext.Error(error.Message);
                 }
 
-                throw new ArgumentException($"Fail to load {fileRelativePath}");
+                throw new ArgumentException($"Failed to load {fileRelativePath}");
             }
 
             if (actionDefinition.Execution == null)

src/Runner.Worker/ContainerOperationProvider.cs

@@ -466,17 +466,39 @@ namespace GitHub.Runner.Worker
             {
                 throw new InvalidOperationException($"Failed to create directory to store registry client credentials: {e.Message}");
             }
-            var loginExitCode = await _dockerManager.DockerLogin(
-                executionContext,
-                configLocation,
-                container.RegistryServer,
-                container.RegistryAuthUsername,
-                container.RegistryAuthPassword);
 
-            if (loginExitCode != 0)
+            // Login docker with retry up to 3 times
+            int retryCount = 0;
+            int loginExitCode = 0;
+            while (retryCount < 3)
+            {
+                loginExitCode = await _dockerManager.DockerLogin(
+                    executionContext,
+                    configLocation,
+                    container.RegistryServer,
+                    container.RegistryAuthUsername,
+                    container.RegistryAuthPassword);
+                if (loginExitCode == 0)
+                {
+                    break;
+                }
+                else
+                {
+                    retryCount++;
+                    if (retryCount < 3)
+                    {
+                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
+                        executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
+                        await Task.Delay(backOff);
+                    }
+                }
+            }
+
+            if (retryCount == 3 && loginExitCode != 0)
             {
                 throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
             }
+
             return configLocation;
         }
 

src/Runner.Worker/DiagnosticLogManager.cs

@@ -108,6 +108,8 @@ namespace GitHub.Runner.Worker
 
             parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
 
+            parentContext.QueueDiagnosticLogFile(name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
+
             executionContext.Debug("Diagnostic file upload complete.");
         }
 

src/Runner.Worker/ExecutionContext.cs

@@ -90,6 +90,7 @@ namespace GitHub.Runner.Worker
         long Write(string tag, string message);
         void QueueAttachFile(string type, string name, string filePath);
         void QueueSummaryFile(string name, string filePath, Guid stepRecordId);
+        void QueueDiagnosticLogFile(string name, string filePath);
 
         // timeline record update methods
         void Start(string currentOperation = null);
@@ -397,11 +398,11 @@ namespace GitHub.Runner.Worker
 
             if (recordOrder != null)
             {
-                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, recordOrder);
+                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, recordOrder, embedded: isEmbedded);
             }
             else
             {
-                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder);
+                child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder, embedded: isEmbedded);
             }
             if (logger != null)
             {
@@ -432,7 +433,7 @@ namespace GitHub.Runner.Worker
             Dictionary<string, string> intraActionState = null,
             string siblingScopeName = null)
         {
-            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout());
+            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order);
         }
 
         public void Start(string currentOperation = null)
@@ -982,6 +983,18 @@ namespace GitHub.Runner.Worker
             _jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0);
         }
 
+        public void QueueDiagnosticLogFile(string name, string filePath)
+        {
+            ArgUtil.NotNullOrEmpty(name, nameof(name));
+            ArgUtil.NotNullOrEmpty(filePath, nameof(filePath));
+
+            if (!File.Exists(filePath))
+            {
+                throw new FileNotFoundException($"Can't upload diagnostic log file: {filePath}. File does not exist.");
+            }
+            _jobServerQueue.QueueResultsUpload(_record.Id, name, filePath, CoreAttachmentType.ResultsDiagnosticLog, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0);
+        }
+
         // Add OnMatcherChanged
         public void Add(OnMatcherChanged handler)
         {
@@ -1160,7 +1173,7 @@ namespace GitHub.Runner.Worker
             }
         }
 
-        private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order)
+        private void InitializeTimelineRecord(Guid timelineId, Guid timelineRecordId, Guid? parentTimelineRecordId, string recordType, string displayName, string refName, int? order, bool embedded = false)
         {
             _mainTimelineId = timelineId;
             _record.Id = timelineRecordId;
@@ -1186,7 +1199,11 @@ namespace GitHub.Runner.Worker
             var configuration = HostContext.GetService<IConfigurationStore>();
             _record.WorkerName = configuration.GetSettings().AgentName;
 
-            _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
+            // We don't want to update the timeline record for embedded steps since they are not really represented in the UI.
+            if (!embedded)
+            {
+                _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
+            }
         }
 
         private void JobServerQueueThrottling_EventReceived(object sender, ThrottlingEventArgs data)

src/Runner.Worker/JobExtension.cs

@@ -392,6 +392,18 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
+                    // Register custom image creation post-job step if the "snapshot" token is present in the message.
+                    var snapshotRequest = templateEvaluator.EvaluateJobSnapshotRequest(message.Snapshot, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
+                    if (snapshotRequest != null)
+                    {
+                        var snapshotOperationProvider = HostContext.GetService<ISnapshotOperationProvider>();
+                        jobContext.RegisterPostJobStep(new JobExtensionRunner(
+                            runAsync: (executionContext, _) => snapshotOperationProvider.CreateSnapshotRequestAsync(executionContext, snapshotRequest),
+                            condition: $"{PipelineTemplateConstants.Success}()",
+                            displayName: $"Create custom image",
+                            data: null));
+                    }
+
                     // Register Job Completed hook if the variable is set
                     var completedHookPath = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_COMPLETED");
                     if (!string.IsNullOrEmpty(completedHookPath))

src/Runner.Worker/JobRunner.cs

@@ -49,6 +49,9 @@ namespace GitHub.Runner.Worker
                 !string.IsNullOrEmpty(orchestrationId.Value))
             {
                 HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
+
+                // make sure orchestration id is in the user-agent header.
+                VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
             }
 
             var jobServerQueueTelemetry = false;
@@ -295,6 +298,8 @@ namespace GitHub.Runner.Worker
                 jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
             }
 
+            await ShutdownQueue(throwOnFailure: false);
+
             // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir.
             _tempDirectoryManager?.CleanupTempDirectory();
 

src/Runner.Worker/SnapshotOperationProvider.cs

@@ -0,0 +1,32 @@
+#nullable enable
+using System.IO;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+namespace GitHub.Runner.Worker;
+
+[ServiceLocator(Default = typeof(SnapshotOperationProvider))]
+public interface ISnapshotOperationProvider : IRunnerService
+{
+    Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest);
+}
+
+public class SnapshotOperationProvider : RunnerService, ISnapshotOperationProvider
+{
+    public Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest)
+    {
+        var snapshotRequestFilePath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), ".snapshot", "request.json");
+        var snapshotRequestDirectoryPath = Path.GetDirectoryName(snapshotRequestFilePath);
+        if (snapshotRequestDirectoryPath != null)
+        {
+            Directory.CreateDirectory(snapshotRequestDirectoryPath);
+        }
+
+        IOUtil.SaveObject(snapshotRequest, snapshotRequestFilePath);
+        executionContext.Output($"Request written to: {snapshotRequestFilePath}");
+        executionContext.Output("This request will be processed after the job completes. You will not receive any feedback on the snapshot process within the workflow logs of this job.");
+        executionContext.Output("If the snapshot process is successful, you should see a new image with the requested name in the list of available custom images when creating a new GitHub-hosted Runner.");
+        return Task.CompletedTask;
+    }
+}

src/Runner.Worker/StepsRunner.cs

@@ -295,7 +295,7 @@ namespace GitHub.Runner.Worker
                     !jobCancellationToken.IsCancellationRequested)
                 {
                     Trace.Error($"Caught timeout exception from step: {ex.Message}");
-                    step.ExecutionContext.Error("The action has timed out.");
+                    step.ExecutionContext.Error($"The action '{step.DisplayName}' has timed out after {timeoutMinutes} minutes.");
                     step.ExecutionContext.Result = TaskResult.Failed;
                 }
                 else

src/Sdk/Common/Common/RawHttpHeaders.cs

@@ -1,12 +1,11 @@
 using System;
 using System.ComponentModel;
-using System.Diagnostics.CodeAnalysis;
 
 namespace GitHub.Services.Common.Internal
 {
     [EditorBrowsable(EditorBrowsableState.Never)]
     public static class RawHttpHeaders
     {
-        public const String SessionHeader = "X-Runner-Session";
+        public const String SessionHeader = "X-Actions-Session";
     }
 }

src/Sdk/Common/Common/RawHttpMessageHandler.cs

@@ -138,6 +138,8 @@ namespace GitHub.Services.Common
                         response.Dispose();
                     }
 
+                    this.Settings.ApplyTo(request);
+
                     // Let's start with sending a token
                     IssuedToken token = null;
                     if (m_tokenProvider != null)

src/Sdk/Common/Common/VssHttpMessageHandler.cs

@@ -214,25 +214,7 @@ namespace GitHub.Services.Common
                     // ConfigureAwait(false) enables the continuation to be run outside any captured
                     // SyncronizationContext (such as ASP.NET's) which keeps things from deadlocking...
 
-                    var tmpResponse = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
-                    if (Settings.AllowAutoRedirectForBroker && tmpResponse.StatusCode == HttpStatusCode.Redirect)
-                    {
-                        //Dispose of the previous response
-                        tmpResponse?.Dispose();
-
-                        var location = tmpResponse.Headers.Location;
-                        request = new HttpRequestMessage(HttpMethod.Get, location);
-
-                        // Reapply the token to new redirected request
-                        ApplyToken(request, token, applyICredentialsToWebProxy: lastResponseDemandedProxyAuth);
-
-                        // Resend the request
-                        response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
-                    }
-                    else
-                    {
-                        response = tmpResponse;
-                    }
+                    response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
 
                     traceInfo?.TraceRequestSendTime();
 

src/Sdk/Common/Common/VssHttpRequestSettings.cs

@@ -110,16 +110,6 @@ namespace GitHub.Services.Common
             set;
         }
 
-        /// <summary>
-        /// Gets or sets a value indicating whether or not HttpClientHandler should follow redirect on outgoing broker requests
-        /// This is special since this also sends token in the request, where as default AllowAutoRedirect does not
-        /// </summary>
-        public Boolean AllowAutoRedirectForBroker
-        {
-            get;
-            set;
-        }
-
         /// <summary>
         /// Gets or sets a value indicating whether or not compression should be used on outgoing requests.
         /// The default value is true.

src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs

@@ -43,6 +43,7 @@ namespace GitHub.DistributedTask.Pipelines
             TemplateToken jobOutputs,
             IList<TemplateToken> defaults,
             ActionsEnvironmentReference actionsEnvironment,
+            TemplateToken snapshot,
             String messageType = JobRequestMessageTypes.PipelineAgentJobRequest)
         {
             this.MessageType = messageType;
@@ -57,6 +58,7 @@ namespace GitHub.DistributedTask.Pipelines
             this.Workspace = workspaceOptions;
             this.JobOutputs = jobOutputs;
             this.ActionsEnvironment = actionsEnvironment;
+            this.Snapshot = snapshot;
             m_variables = new Dictionary<String, VariableValue>(variables, StringComparer.OrdinalIgnoreCase);
             m_maskHints = new List<MaskHint>(maskHints);
             m_steps = new List<JobStep>(steps);
@@ -237,6 +239,13 @@ namespace GitHub.DistributedTask.Pipelines
             set;
         }
 
+        [DataMember(EmitDefaultValue = false)]
+        public TemplateToken Snapshot
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets the collection of variables associated with the current context.
         /// </summary>

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs

@@ -29,6 +29,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
         public const String Id = "id";
         public const String If = "if";
         public const String Image = "image";
+        public const String ImageName = "image-name";
         public const String Include = "include";
         public const String Inputs = "inputs";
         public const String Job = "job";
@@ -60,6 +61,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
         public const String Services = "services";
         public const String Shell = "shell";
         public const String Skipped = "skipped";
+        public const String Snapshot = "snapshot";
         public const String StepEnv = "step-env";
         public const String StepIfResult = "step-if-result";
         public const String StepWith = "step-with";

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs

@@ -346,6 +346,39 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
             return result;
         }
 
+        internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, TemplateToken token)
+        {
+            string imageName = null;
+            if (token is StringToken snapshotStringLiteral)
+            {
+                imageName = snapshotStringLiteral.Value;
+            }
+            else
+            {
+                var snapshotMapping = token.AssertMapping($"{PipelineTemplateConstants.Snapshot}");
+                foreach (var snapshotPropertyPair in snapshotMapping)
+                {
+                    var propertyName = snapshotPropertyPair.Key.AssertString($"{PipelineTemplateConstants.Snapshot} key");
+                    switch (propertyName.Value)
+                    {
+                        case PipelineTemplateConstants.ImageName:
+                            imageName = snapshotPropertyPair.Value.AssertString($"{PipelineTemplateConstants.Snapshot} {propertyName}").Value;
+                            break;
+                        default:
+                            propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Snapshot} key");
+                            break;
+                    }
+                }
+            }
+
+            if (String.IsNullOrEmpty(imageName))
+            {
+                return null;
+            }
+
+            return new Snapshot(imageName);
+        }
+
         private static ActionStep ConvertToStep(
             TemplateContext context,
             TemplateToken stepsItem,

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs

@@ -370,6 +370,32 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
             return result;
         }
 
+        public Snapshot EvaluateJobSnapshotRequest(TemplateToken token,
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
+        {
+            var result = default(Snapshot);
+
+            if (token != null && token.Type != TokenType.Null)
+            {
+                var context = CreateContext(contextData, expressionFunctions);
+                try
+                {
+                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Snapshot, token, 0, null, omitHeader: true);
+                    context.Errors.Check();
+                    result = PipelineTemplateConverter.ConvertToJobSnapshotRequest(context, token);
+                }
+                catch (Exception ex) when (!(ex is TemplateValidationException))
+                {
+                    context.Errors.Add(ex);
+                }
+
+                context.Errors.Check();
+            }
+
+            return result;
+        }
+
         private TemplateContext CreateContext(
             DictionaryContextData contextData,
             IList<IFunctionInfo> expressionFunctions,

src/Sdk/DTPipelines/Pipelines/Snapshot.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.Pipelines
+{
+    [DataContract]
+    public class Snapshot
+    {
+        public Snapshot(string imageName)
+        {
+            ImageName = imageName;
+        }
+
+        [DataMember(EmitDefaultValue = false)]
+        public String ImageName { get; set; }
+    }
+}

src/Sdk/DTPipelines/workflow-v1.0.json

@@ -71,7 +71,8 @@
           "env": "job-env",
           "outputs": "job-outputs",
           "defaults": "job-defaults",
-          "steps": "steps"
+          "steps": "steps",
+          "snapshot": "snapshot"
         }
       }
     },
@@ -155,6 +156,24 @@
       }
     },
 
+    "snapshot": {
+      "one-of": [
+        "non-empty-string",
+        "snapshot-mapping"
+      ]
+    },
+
+    "snapshot-mapping": {
+      "mapping": {
+        "properties": {
+          "image-name": {
+            "type": "non-empty-string",
+            "required": true
+          }
+        }
+      }
+    },
+
     "runs-on": {
       "context": [
         "github",

src/Sdk/DTWebApi/WebApi/BrokerMigrationMessage.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.WebApi
+{
+    /// <summary>
+    /// Message that tells the runner to redirect itself to BrokerListener for messages.
+    /// (Note that we use a special Message instead of a simple 302. This is because 
+    /// the runner will need to apply the runner's token to the request, and it is
+    /// a security best practice to *not* blindly add sensitive data to redirects
+    /// 302s.)
+    /// </summary>
+    [DataContract]
+    public class BrokerMigrationMessage
+    {
+        public static readonly string MessageType = "BrokerMigration";
+
+        public BrokerMigrationMessage()
+        {
+        }
+
+        public BrokerMigrationMessage(
+            Uri brokerUrl)
+        {
+            this.BrokerBaseUrl = brokerUrl;
+        }
+
+        /// <summary>
+        /// The base url for the broker listener
+        /// </summary>
+        [DataMember]
+        public Uri BrokerBaseUrl
+        {
+            get;
+            internal set;
+        }
+    }
+}

src/Sdk/DTWebApi/WebApi/TaskAgentMessageTypes.cs

@@ -0,0 +1,10 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.WebApi
+{
+    public sealed class TaskAgentMessageTypes
+    {
+        public static readonly string ForceTokenRefresh = "ForceTokenRefresh";
+    }
+}

src/Sdk/DTWebApi/WebApi/TaskAgentSession.cs

@@ -75,5 +75,12 @@ namespace GitHub.DistributedTask.WebApi
             get;
             set;
         }
+
+        [DataMember(EmitDefaultValue = false, IsRequired = false)]
+        public BrokerMigrationMessage BrokerMigrationMessage
+        {
+            get;
+            set;
+        }
     }
 }

src/Sdk/DTWebApi/WebApi/TaskAttachment.cs

@@ -101,6 +101,7 @@ namespace GitHub.DistributedTask.WebApi
         public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment";
         public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog";
         public static readonly String ResultsLog = "Results.Core.Log";
+        public static readonly String ResultsDiagnosticLog = "Results.Core.DiagnosticLog";
     }
 
     [GenerateAllConstants]

src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs

@@ -7,5 +7,8 @@ namespace GitHub.Actions.RunService.WebApi
     {
         [DataMember(Name = "jobMessageId", EmitDefaultValue = false)]
         public string JobMessageId { get; set; }
+
+        [DataMember(Name = "runnerOS", EmitDefaultValue = false)]
+        public string RunnerOS { get; set; }
     }
 }

src/Sdk/RSWebApi/RunServiceHttpClient.cs

@@ -59,12 +59,14 @@ namespace GitHub.Actions.RunService.WebApi
         public async Task<AgentJobRequestMessage> GetJobMessageAsync(
             Uri requestUri,
             string messageId,
+            string runnerOS,
             CancellationToken cancellationToken = default)
         {
             HttpMethod httpMethod = new HttpMethod("POST");
             var payload = new AcquireJobRequest
             {
                 JobMessageId = messageId,
+                RunnerOS = runnerOS
             };
 
             requestUri = new Uri(requestUri, "acquirejob");

src/Sdk/WebApi/WebApi/BrokerHttpClient.cs

@@ -57,6 +57,7 @@ namespace GitHub.Actions.RunService.WebApi
         }
 
         public async Task<TaskAgentMessage> GetRunnerMessageAsync(
+            Guid? sessionId,
             string runnerVersion,
             TaskAgentStatus? status,
             string os = null,
@@ -69,6 +70,11 @@ namespace GitHub.Actions.RunService.WebApi
 
             List<KeyValuePair<string, string>> queryParams = new List<KeyValuePair<string, string>>();
 
+            if (sessionId != null)
+            {
+                queryParams.Add("sessionId", sessionId.Value.ToString());
+            }
+
             if (status != null)
             {
                 queryParams.Add("status", status.Value.ToString());
@@ -104,12 +110,67 @@ namespace GitHub.Actions.RunService.WebApi
                 return result.Value;
             }
 
+            // the only time we throw a `Forbidden` exception from Listener /messages is when the runner is
+            // disable_update and is too old to poll
+            if (result.StatusCode == HttpStatusCode.Forbidden)
+            {
+                throw new AccessDeniedException($"{result.Error} Runner version v{runnerVersion} is deprecated and cannot receive messages.")
+                {
+                    ErrorCode = 1
+                };
+            }
+
+            throw new Exception($"Failed to get job message: {result.Error}");
+        }
+
+        public async Task<TaskAgentSession> CreateSessionAsync(
+
+           TaskAgentSession session,
+           CancellationToken cancellationToken = default)
+        {
+            var requestUri = new Uri(Client.BaseAddress, "session");
+            var requestContent = new ObjectContent<TaskAgentSession>(session, new VssJsonMediaTypeFormatter(true));
+
+            var result = await SendAsync<TaskAgentSession>(
+                new HttpMethod("POST"),
+                requestUri: requestUri,
+                content: requestContent,
+                cancellationToken: cancellationToken);
+
+            if (result.IsSuccess)
+            {
+                return result.Value;
+            }
+
             if (result.StatusCode == HttpStatusCode.Forbidden)
             {
                 throw new AccessDeniedException(result.Error);
             }
 
-            throw new Exception($"Failed to get job message: {result.Error}");
+            if (result.StatusCode == HttpStatusCode.Conflict)
+            {
+                throw new TaskAgentSessionConflictException(result.Error);
+            }
+
+            throw new Exception($"Failed to create broker session: {result.Error}");
+        }
+
+        public async Task DeleteSessionAsync(
+            CancellationToken cancellationToken = default)
+        {
+            var requestUri = new Uri(Client.BaseAddress, $"session");
+
+            var result = await SendAsync<object>(
+                new HttpMethod("DELETE"),
+                requestUri: requestUri,
+                cancellationToken: cancellationToken);
+
+            if (result.IsSuccess)
+            {
+                return;
+            }
+
+            throw new Exception($"Failed to delete broker session: {result.Error}");
         }
     }
 }

src/Sdk/WebApi/WebApi/Contracts.cs

@@ -89,6 +89,26 @@ namespace GitHub.Services.Results.Contracts
         public long SoftSizeLimit;
     }
 
+    [DataContract]
+    [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
+    public class GetSignedDiagnosticLogsURLRequest
+    {
+        [DataMember]
+        public string WorkflowJobRunBackendId;
+        [DataMember]
+        public string WorkflowRunBackendId;
+    }
+
+    [DataContract]
+    [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
+    public class GetSignedDiagnosticLogsURLResponse
+    {
+        [DataMember]
+        public string DiagLogsURL;
+        [DataMember]
+        public string BlobStorageType;
+    }
+
     [DataContract]
     [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
     public class JobLogsMetadataCreate

src/Sdk/WebApi/WebApi/ResultsHttpClient.cs

@@ -81,6 +81,19 @@ namespace GitHub.Services.Results.Client
             return await GetResultsSignedURLResponse<GetSignedStepLogsURLRequest, GetSignedStepLogsURLResponse>(getStepLogsSignedBlobURLEndpoint, cancellationToken, request);
         }
 
+        private async Task<GetSignedDiagnosticLogsURLResponse> GetDiagnosticLogsUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken)
+        {
+            var request = new GetSignedDiagnosticLogsURLRequest()
+            {
+                WorkflowJobRunBackendId = jobId,
+                WorkflowRunBackendId = planId,
+            };
+
+            var getDiagnosticLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetJobDiagLogsSignedBlobURL);
+
+            return await GetResultsSignedURLResponse<GetSignedDiagnosticLogsURLRequest, GetSignedDiagnosticLogsURLResponse>(getDiagnosticLogsSignedBlobURLEndpoint, cancellationToken, request);
+        }
+
         private async Task<GetSignedJobLogsURLResponse> GetJobLogUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken)
         {
             var request = new GetSignedJobLogsURLRequest()
@@ -209,14 +222,34 @@ namespace GitHub.Services.Results.Client
             return new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts);
         }
 
-        private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
+        private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
         {
             if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
             {
                 var blobClient = GetBlobClient(url);
+                var httpHeaders = new BlobHttpHeaders();
+                if (customHeaders != null)
+                {
+                    foreach (var header in customHeaders)
+                    {
+                        switch (header.Key)
+                        {
+                            case Constants.ContentTypeHeader:
+                                httpHeaders.ContentType = header.Value;
+                                break;
+                        }
+                    }
+                }
                 try
                 {
-                    await blobClient.UploadAsync(file, cancellationToken);
+                    await blobClient.UploadAsync(file, new BlobUploadOptions()
+                    {
+                        HttpHeaders = httpHeaders,
+                        Conditions = new BlobRequestConditions
+                        {
+                            IfNoneMatch = new ETag("*")
+                        }
+                    }, cancellationToken);
                 }
                 catch (RequestFailedException e)
                 {
@@ -236,6 +269,14 @@ namespace GitHub.Services.Results.Client
                     request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob);
                 }
 
+                if (customHeaders != null)
+                {
+                    foreach (var header in customHeaders)
+                    {
+                        request.Content.Headers.Add(header.Key, header.Value);
+                    }
+                };
+
                 using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
                 {
                     if (!response.IsSuccessStatusCode)
@@ -246,14 +287,34 @@ namespace GitHub.Services.Results.Client
             }
         }
 
-        private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken)
+        private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
         {
             if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage)
             {
                 var appendBlobClient = GetAppendBlobClient(url);
+                var httpHeaders = new BlobHttpHeaders();
+                if (customHeaders != null)
+                {
+                    foreach (var header in customHeaders)
+                    {
+                        switch (header.Key)
+                        {
+                            case Constants.ContentTypeHeader:
+                                httpHeaders.ContentType = header.Value;
+                                break;
+                        }
+                    }
+                }
                 try
                 {
-                    await appendBlobClient.CreateAsync(cancellationToken: cancellationToken);
+                    await appendBlobClient.CreateAsync(new AppendBlobCreateOptions()
+                    {
+                        HttpHeaders = httpHeaders,
+                        Conditions = new AppendBlobRequestConditions
+                        {
+                            IfNoneMatch = new ETag("*")
+                        }
+                    }, cancellationToken: cancellationToken);
                 }
                 catch (RequestFailedException e)
                 {
@@ -271,6 +332,13 @@ namespace GitHub.Services.Results.Client
                     request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob);
                     request.Content.Headers.Add("Content-Length", "0");
                 }
+                if (customHeaders != null)
+                {
+                    foreach (var header in customHeaders)
+                    {
+                        request.Content.Headers.Add(header.Key, header.Value);
+                    }
+                };
 
                 using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
                 {
@@ -353,14 +421,14 @@ namespace GitHub.Services.Results.Client
         }
 
         private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType,
-            CancellationToken cancellationToken)
+            CancellationToken cancellationToken, Dictionary<string, string> customHeaders = null)
         {
             if (firstBlock && finalize)
             {
                 // This is the one and only block, just use a block blob
                 using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
                 {
-                    await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken);
+                    await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken, customHeaders);
                 }
             }
             else
@@ -369,7 +437,7 @@ namespace GitHub.Services.Results.Client
                 // Create the Append blob
                 if (firstBlock)
                 {
-                    await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken);
+                    await CreateAppendFileAsync(sasUrl, blobStorageType, cancellationToken, customHeaders);
                 }
 
                 // Upload content
@@ -391,7 +459,12 @@ namespace GitHub.Services.Results.Client
                 throw new Exception("Failed to get step log upload url");
             }
 
-            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
+            var customHeaders = new Dictionary<string, string>
+            {
+                { Constants.ContentTypeHeader, Constants.TextPlainContentType }
+            };
+
+            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders);
 
             // Update metadata
             if (finalize)
@@ -411,7 +484,12 @@ namespace GitHub.Services.Results.Client
                 throw new Exception("Failed to get job log upload url");
             }
 
-            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
+            var customHeaders = new Dictionary<string, string>
+            {
+                { Constants.ContentTypeHeader, Constants.TextPlainContentType }
+            };
+
+            await UploadLogFile(file, finalize, firstBlock, uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken, customHeaders);
 
             // Update metadata
             if (finalize)
@@ -421,6 +499,18 @@ namespace GitHub.Services.Results.Client
             }
         }
 
+        public async Task UploadResultsDiagnosticLogsAsync(string planId, string jobId, string file, CancellationToken cancellationToken)
+        {
+            // Get the upload url
+            var uploadUrlResponse = await GetDiagnosticLogsUploadUrlAsync(planId, jobId, cancellationToken);
+            if (uploadUrlResponse == null || uploadUrlResponse.DiagLogsURL == null)
+            {
+                throw new Exception("Failed to get diagnostic logs upload url");
+            }
+
+            await UploadLogFile(file, true, true, uploadUrlResponse.DiagLogsURL, uploadUrlResponse.BlobStorageType, cancellationToken);
+        }
+
         private Step ConvertTimelineRecordToStep(TimelineRecord r)
         {
             return new Step()
@@ -511,6 +601,7 @@ namespace GitHub.Services.Results.Client
         public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata";
         public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL";
         public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata";
+        public static readonly string GetJobDiagLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobDiagLogsSignedBlobURL";
         public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/";
         public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate";
 
@@ -521,6 +612,9 @@ namespace GitHub.Services.Results.Client
         public static readonly string AzureBlobTypeHeader = "x-ms-blob-type";
         public static readonly string AzureBlockBlob = "BlockBlob";
         public static readonly string AzureAppendBlob = "AppendBlob";
+
+        public const string ContentTypeHeader = "Content-Type";
+        public const string TextPlainContentType = "text/plain";
     }
 
 }

src/Test/L0/Listener/BrokerMessageListenerL0.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Listener;
+using GitHub.Runner.Listener.Configuration;
+using GitHub.Services.Common;
+using Moq;
+using Xunit;
+
+namespace GitHub.Runner.Common.Tests.Listener
+{
+    public sealed class BrokerMessageListenerL0
+    {
+        private readonly RunnerSettings _settings;
+        private readonly Mock<IConfigurationManager> _config;
+        private readonly Mock<IBrokerServer> _brokerServer;
+        private readonly Mock<ICredentialManager> _credMgr;
+        private Mock<IConfigurationStore> _store;
+
+
+        public BrokerMessageListenerL0()
+        {
+            _settings = new RunnerSettings { AgentId = 1, AgentName = "myagent", PoolId = 123, PoolName = "default", ServerUrl = "http://myserver", WorkFolder = "_work", ServerUrlV2 = "http://myserverv2" };
+            _config = new Mock<IConfigurationManager>();
+            _config.Setup(x => x.LoadSettings()).Returns(_settings);
+            _credMgr = new Mock<ICredentialManager>();
+            _store = new Mock<IConfigurationStore>();
+            _brokerServer = new Mock<IBrokerServer>();
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void CreatesSession()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession();
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
+                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
+
+                // Act.
+                BrokerMessageListener listener = new();
+                listener.Initialize(tc);
+
+                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+
+                // Assert.
+                Assert.True(result);
+                _brokerServer
+                   .Verify(x => x.CreateSessionAsync(
+                       It.Is<TaskAgentSession>(y => y != null),
+                       tokenSource.Token), Times.Once());
+            }
+        }
+
+        private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
+        {
+            TestHostContext tc = new(this, testName);
+            tc.SetSingleton<IConfigurationManager>(_config.Object);
+            tc.SetSingleton<ICredentialManager>(_credMgr.Object);
+            tc.SetSingleton<IConfigurationStore>(_store.Object);
+            tc.SetSingleton<IBrokerServer>(_brokerServer.Object);
+            return tc;
+        }
+    }
+}

src/Test/L0/Listener/JobDispatcherL0.cs

@@ -41,7 +41,7 @@ namespace GitHub.Runner.Common.Tests.Listener
             TaskOrchestrationPlanReference plan = new();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
             result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
             return result;
         }
@@ -806,7 +806,8 @@ namespace GitHub.Runner.Common.Tests.Listener
                 },
                 null,
                 new List<TemplateToken>(),
-                new ActionsEnvironmentReference("env")
+                new ActionsEnvironmentReference("env"),
+                null
             );
             return message;
         }

src/Test/L0/Listener/MessageListenerL0.cs

@@ -24,6 +24,8 @@ namespace GitHub.Runner.Common.Tests.Listener
         private Mock<ICredentialManager> _credMgr;
         private Mock<IConfigurationStore> _store;
 
+        private Mock<IBrokerServer> _brokerServer;
+
         public MessageListenerL0()
         {
             _settings = new RunnerSettings { AgentId = 1, AgentName = "myagent", PoolId = 123, PoolName = "default", ServerUrl = "http://myserver", WorkFolder = "_work" };
@@ -32,6 +34,7 @@ namespace GitHub.Runner.Common.Tests.Listener
             _runnerServer = new Mock<IRunnerServer>();
             _credMgr = new Mock<ICredentialManager>();
             _store = new Mock<IConfigurationStore>();
+            _brokerServer = new Mock<IBrokerServer>();
         }
 
         private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
@@ -41,6 +44,7 @@ namespace GitHub.Runner.Common.Tests.Listener
             tc.SetSingleton<IRunnerServer>(_runnerServer.Object);
             tc.SetSingleton<ICredentialManager>(_credMgr.Object);
             tc.SetSingleton<IConfigurationStore>(_store.Object);
+            tc.SetSingleton<IBrokerServer>(_brokerServer.Object);
             return tc;
         }
 
@@ -81,6 +85,72 @@ namespace GitHub.Runner.Common.Tests.Listener
                         _settings.PoolId,
                         It.Is<TaskAgentSession>(y => y != null),
                         tokenSource.Token), Times.Once());
+                _brokerServer
+                   .Verify(x => x.CreateSessionAsync(
+                       It.Is<TaskAgentSession>(y => y != null),
+                       tokenSource.Token), Times.Never());
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void CreatesSessionWithBrokerMigration()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession()
+                {
+                    OwnerName = "legacy",
+                    BrokerMigrationMessage = new BrokerMigrationMessage(new Uri("https://broker.actions.github.com"))
+                };
+
+                var expectedBrokerSession = new TaskAgentSession()
+                {
+                    OwnerName = "broker"
+                };
+
+                _runnerServer
+                    .Setup(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedBrokerSession));
+
+                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
+                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
+
+                // Act.
+                MessageListener listener = new();
+                listener.Initialize(tc);
+
+                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+
+                // Assert.
+                Assert.True(result);
+
+                _runnerServer
+                    .Verify(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token), Times.Once());
+
+                _brokerServer
+                    .Verify(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token), Times.Once());
             }
         }
 
@@ -131,6 +201,83 @@ namespace GitHub.Runner.Common.Tests.Listener
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void DeleteSessionWithBrokerMigration()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession()
+                {
+                    OwnerName = "legacy",
+                    BrokerMigrationMessage = new BrokerMigrationMessage(new Uri("https://broker.actions.github.com"))
+                };
+
+                var expectedBrokerSession = new TaskAgentSession()
+                {
+                    SessionId = Guid.NewGuid(),
+                    OwnerName = "broker"
+                };
+
+                _runnerServer
+                    .Setup(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _brokerServer
+                    .Setup(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedBrokerSession));
+
+                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
+                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
+
+                // Act.
+                MessageListener listener = new();
+                listener.Initialize(tc);
+
+                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                trace.Info("result: {0}", result);
+
+                Assert.True(result);
+
+                _runnerServer
+                    .Verify(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token), Times.Once());
+
+                _brokerServer
+                    .Verify(x => x.CreateSessionAsync(
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token), Times.Once());
+
+                _brokerServer
+                    .Setup(x => x.DeleteSessionAsync(It.IsAny<CancellationToken>()))
+                    .Returns(Task.CompletedTask);
+
+                // Act.
+                await listener.DeleteSessionAsync();
+
+
+                //Assert
+                _runnerServer
+                    .Verify(x => x.DeleteAgentSessionAsync(
+                        _settings.PoolId, expectedSession.SessionId, It.IsAny<CancellationToken>()), Times.Never());
+                _brokerServer
+                    .Verify(x => x.DeleteSessionAsync(It.IsAny<CancellationToken>()), Times.Once());
+            }
+        }
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]
@@ -212,6 +359,112 @@ namespace GitHub.Runner.Common.Tests.Listener
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void GetNextMessageWithBrokerMigration()
+        {
+            using (TestHostContext tc = CreateTestContext())
+            using (var tokenSource = new CancellationTokenSource())
+            {
+                Tracing trace = tc.GetTrace();
+
+                // Arrange.
+                var expectedSession = new TaskAgentSession();
+                PropertyInfo sessionIdProperty = expectedSession.GetType().GetProperty("SessionId", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
+                Assert.NotNull(sessionIdProperty);
+                sessionIdProperty.SetValue(expectedSession, Guid.NewGuid());
+
+                _runnerServer
+                    .Setup(x => x.CreateAgentSessionAsync(
+                        _settings.PoolId,
+                        It.Is<TaskAgentSession>(y => y != null),
+                        tokenSource.Token))
+                    .Returns(Task.FromResult(expectedSession));
+
+                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
+                _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
+
+                // Act.
+                MessageListener listener = new();
+                listener.Initialize(tc);
+
+                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                Assert.True(result);
+
+                var brokerMigrationMesage = new BrokerMigrationMessage(new Uri("https://actions.broker.com"));
+
+                var arMessages = new TaskAgentMessage[]
+                {
+                        new TaskAgentMessage
+                        {
+                            Body = JsonUtility.ToString(brokerMigrationMesage),
+                            MessageType = BrokerMigrationMessage.MessageType
+                        },
+                };
+
+                var brokerMessages = new TaskAgentMessage[]
+                {
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody1",
+                            MessageId = 4234,
+                            MessageType = JobRequestMessageTypes.PipelineAgentJobRequest
+                        },
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody2",
+                            MessageId = 4235,
+                            MessageType = JobCancelMessage.MessageType
+                        },
+                        null,  //should be skipped by GetNextMessageAsync implementation
+                        null,
+                        new TaskAgentMessage
+                        {
+                            Body = "somebody3",
+                            MessageId = 4236,
+                            MessageType = JobRequestMessageTypes.PipelineAgentJobRequest
+                        }
+                };
+                var brokerMessageQueue = new Queue<TaskAgentMessage>(brokerMessages);
+
+                _runnerServer
+                    .Setup(x => x.GetAgentMessageAsync(
+                        _settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+                    .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, string runnerVersion, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken) =>
+                    {
+                        await Task.Yield();
+                        return arMessages[0]; // always send migration message
+                    });
+
+                _brokerServer
+                   .Setup(x => x.GetRunnerMessageAsync(
+                       expectedSession.SessionId, TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
+                   .Returns(async (Guid sessionId, TaskAgentStatus status, string runnerVersion, string os, string architecture, bool disableUpdate, CancellationToken cancellationToken) =>
+                   {
+                       await Task.Yield();
+                       return brokerMessageQueue.Dequeue();
+                   });
+
+                TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token);
+                TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token);
+                TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token);
+                Assert.Equal(brokerMessages[0], message1);
+                Assert.Equal(brokerMessages[1], message2);
+                Assert.Equal(brokerMessages[4], message3);
+
+                //Assert
+                _runnerServer
+                    .Verify(x => x.GetAgentMessageAsync(
+                        _settings.PoolId, expectedSession.SessionId, It.IsAny<long?>(), TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()), Times.Exactly(brokerMessages.Length));
+
+                _brokerServer
+                    .Verify(x => x.GetRunnerMessageAsync(
+                    expectedSession.SessionId, TaskAgentStatus.Online, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()), Times.Exactly(brokerMessages.Length));
+            }
+        }
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]

src/Test/L0/Listener/RunnerL0.cs

@@ -42,7 +42,7 @@ namespace GitHub.Runner.Common.Tests.Listener
             TaskOrchestrationPlanReference plan = new();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
         }
 
         private JobCancelMessage CreateJobCancelMessage()

src/Test/L0/Worker/ActionCommandManagerL0.cs

@@ -232,7 +232,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,

src/Test/L0/Worker/ActionManifestManagerL0.cs

@@ -757,7 +757,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 //Assert
                 var err = Assert.Throws<ArgumentException>(() => actionManifest.Load(_ec.Object, action_path));
-                Assert.Contains($"Fail to load {action_path}", err.Message);
+                Assert.Contains($"Failed to load {action_path}", err.Message);
                 _ec.Verify(x => x.AddIssue(It.Is<Issue>(s => s.Message.Contains("Missing 'using' value. 'using' requires 'composite', 'docker', 'node12', 'node16' or 'node20'.")), It.IsAny<ExecutionContextLogOptions>()), Times.Once);
             }
             finally

src/Test/L0/Worker/CreateStepSummaryCommandL0.cs

@@ -193,7 +193,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             TimelineReference timeline = new();
             Guid jobId = Guid.NewGuid();
             string jobName = "Summary Job";
-            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
             jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
             {
                 Alias = Pipelines.PipelineConstants.SelfAlias,

src/Test/L0/Worker/ExecutionContextL0.cs

@@ -29,7 +29,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -106,7 +106,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -162,7 +162,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -216,7 +216,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -271,7 +271,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -322,7 +322,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -373,7 +373,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -471,7 +471,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -555,7 +555,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -610,7 +610,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -653,7 +653,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -717,7 +717,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -781,7 +781,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -969,7 +969,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -1014,7 +1014,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new TimelineReference();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -1057,7 +1057,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new TimelineReference();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,

src/Test/L0/Worker/JobExtensionL0.cs

@@ -4,6 +4,8 @@ using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
+using GitHub.DistributedTask.Pipelines.ObjectTemplating;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Worker;
 using Moq;
@@ -25,6 +27,9 @@ namespace GitHub.Runner.Common.Tests.Worker
         private Mock<IContainerOperationProvider> _containerProvider;
         private Mock<IDiagnosticLogManager> _diagnosticLogManager;
         private Mock<IJobHookProvider> _jobHookProvider;
+        private Mock<ISnapshotOperationProvider> _snapshotOperationProvider;
+
+        private Pipelines.Snapshot _requestedSnapshot;
 
         private CancellationTokenSource _tokenSource;
         private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
@@ -41,7 +46,16 @@ namespace GitHub.Runner.Common.Tests.Worker
             _directoryManager.Setup(x => x.PrepareDirectory(It.IsAny<IExecutionContext>(), It.IsAny<Pipelines.WorkspaceOptions>()))
                              .Returns(new TrackingConfig() { PipelineDirectory = "runner", WorkspaceDirectory = "runner/runner" });
             _jobHookProvider = new Mock<IJobHookProvider>();
+            _snapshotOperationProvider = new Mock<ISnapshotOperationProvider>();
 
+            _requestedSnapshot = null;
+            _snapshotOperationProvider
+                .Setup(p => p.CreateSnapshotRequestAsync(It.IsAny<IExecutionContext>(), It.IsAny<Pipelines.Snapshot>()))
+                .Returns((IExecutionContext _, object data) =>
+                {
+                    _requestedSnapshot = data as Pipelines.Snapshot;
+                    return Task.CompletedTask;
+                });
             IActionRunner step1 = new ActionRunner();
             IActionRunner step2 = new ActionRunner();
             IActionRunner step3 = new ActionRunner();
@@ -100,7 +114,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             };
 
             Guid jobId = Guid.NewGuid();
-            _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null);
+            _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null, null);
             GitHubContext github = new();
             github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner");
             github["secret_source"] = new Pipelines.ContextData.StringContextData("Actions");
@@ -125,6 +139,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             hc.SetSingleton(_directoryManager.Object);
             hc.SetSingleton(_diagnosticLogManager.Object);
             hc.SetSingleton(_jobHookProvider.Object);
+            hc.SetSingleton(_snapshotOperationProvider.Object);
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // JobExecutionContext
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // job start hook
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // Initial Job
@@ -443,5 +458,80 @@ namespace GitHub.Runner.Common.Tests.Worker
                 Assert.Equal(0, _jobEc.PostJobSteps.Count);
             }
         }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public async Task EnsureNoSnapshotPostJobStep()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                var jobExtension = new JobExtension();
+                jobExtension.Initialize(hc);
+
+                _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny<IExecutionContext>(), It.IsAny<IEnumerable<Pipelines.JobStep>>(), It.IsAny<Guid>()))
+                    .Returns(Task.FromResult(new PrepareResult(new List<JobExtensionRunner>(), new Dictionary<Guid, IActionRunner>())));
+
+                _message.Snapshot = null;
+                await jobExtension.InitializeJob(_jobEc, _message);
+
+                var postJobSteps = _jobEc.PostJobSteps;
+                Assert.Equal(0, postJobSteps.Count);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public Task EnsureSnapshotPostJobStepForStringToken()
+        {
+            var snapshot = new Pipelines.Snapshot("TestImageNameFromStringToken");
+            var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName);
+            return EnsureSnapshotPostJobStepForToken(imageNameValueStringToken, snapshot);
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public Task EnsureSnapshotPostJobStepForMappingToken()
+        {
+            var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken");
+            var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName);
+            var mappingToken = new MappingToken(null, null, null)
+            {
+                { new StringToken(null,null,null, PipelineTemplateConstants.ImageName), imageNameValueStringToken }
+            };
+
+            return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot);
+        }
+
+        private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot)
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                var jobExtension = new JobExtension();
+                jobExtension.Initialize(hc);
+
+                _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny<IExecutionContext>(), It.IsAny<IEnumerable<Pipelines.JobStep>>(), It.IsAny<Guid>()))
+                    .Returns(Task.FromResult(new PrepareResult(new List<JobExtensionRunner>(), new Dictionary<Guid, IActionRunner>())));
+
+                _message.Snapshot = snapshotToken;
+
+                await jobExtension.InitializeJob(_jobEc, _message);
+
+                var postJobSteps = _jobEc.PostJobSteps;
+
+                Assert.Equal(1, postJobSteps.Count);
+                var snapshotStep = postJobSteps.First();
+                Assert.Equal("Create custom image", snapshotStep.DisplayName);
+                Assert.Equal($"{PipelineTemplateConstants.Success}()", snapshotStep.Condition);
+
+                // Run the mock snapshot step, so we can verify it was executed with the expected snapshot object.
+                await snapshotStep.RunAsync();
+
+                Assert.NotNull(_requestedSnapshot);
+                Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName);
+            }
+        }
     }
 }

src/Test/L0/Worker/JobRunnerL0.cs

@@ -101,6 +101,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 testName,
                 testName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null,
                 new ActionsEnvironmentReference("staging"),
+                null,
                 messageType: messageType);
             message.Variables[Constants.Variables.System.Culture] = "en-US";
             message.Resources.Endpoints.Add(new ServiceEndpoint()

src/Test/L0/Worker/SnapshotOperationProviderL0.cs

@@ -0,0 +1,78 @@
+#nullable enable
+using System;
+using System.IO;
+using System.Runtime.CompilerServices;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Runner.Sdk;
+using GitHub.Runner.Worker;
+using Moq;
+using Xunit;
+
+namespace GitHub.Runner.Common.Tests.Worker;
+
+public class SnapshotOperationProviderL0
+{
+    private Mock<IExecutionContext>? _ec;
+    private SnapshotOperationProvider? _snapshotOperationProvider;
+    private string? _snapshotRequestFilePath;
+    private string? _snapshotRequestDirectoryPath;
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    [Trait("Level", "L0")]
+    [Trait("Category", "Worker")]
+    public async void CreateSnapshotRequestAsync(bool shouldSnapshotDirectoryAlreadyExist)
+    {
+        using (TestHostContext testHostContext = CreateTestHostContext())
+        {
+            //Arrange
+            Setup(testHostContext, shouldSnapshotDirectoryAlreadyExist);
+            var expectedSnapshot = new Snapshot(Guid.NewGuid().ToString());
+
+            //Act
+            await _snapshotOperationProvider!.CreateSnapshotRequestAsync(_ec!.Object, expectedSnapshot);
+
+            //Assert
+            var actualSnapshot = IOUtil.LoadObject<Snapshot>(_snapshotRequestFilePath);
+            Assert.NotNull(actualSnapshot);
+            Assert.Equal(expectedSnapshot.ImageName, actualSnapshot!.ImageName);
+            _ec.Verify(ec => ec.Write(null, $"Request written to: {_snapshotRequestFilePath}"), Times.Once);
+            _ec.Verify(ec => ec.Write(null, "This request will be processed after the job completes. You will not receive any feedback on the snapshot process within the workflow logs of this job."), Times.Once);
+            _ec.Verify(ec => ec.Write(null, "If the snapshot process is successful, you should see a new image with the requested name in the list of available custom images when creating a new GitHub-hosted Runner."), Times.Once);
+            _ec.VerifyNoOtherCalls();
+        }
+    }
+
+    private void Setup(IHostContext hostContext, bool shouldSnapshotDirectoryAlreadyExist)
+    {
+        _ec = new Mock<IExecutionContext>();
+        _snapshotOperationProvider = new SnapshotOperationProvider();
+        _snapshotOperationProvider.Initialize(hostContext);
+        _snapshotRequestFilePath = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Root), ".snapshot", "request.json");
+        _snapshotRequestDirectoryPath = Path.GetDirectoryName(_snapshotRequestFilePath);
+
+        if (_snapshotRequestDirectoryPath != null)
+        {
+            // Clean up any existing the snapshot directory and its contents before starting the test. 
+            if (Directory.Exists(_snapshotRequestDirectoryPath))
+            {
+                Directory.Delete(_snapshotRequestDirectoryPath, true);
+            }
+
+            if (shouldSnapshotDirectoryAlreadyExist)
+            {
+                // Create a fresh snapshot directory if it's required for the test. 
+                Directory.CreateDirectory(_snapshotRequestDirectoryPath);
+            }
+        }
+    }
+
+    private TestHostContext CreateTestHostContext([CallerMemberName] string testName = "")
+    {
+        var testHostContext = new TestHostContext(this, testName);
+        _ec = new Mock<IExecutionContext>();
+        _ec.Object.Initialize(testHostContext);
+        return testHostContext;
+    }
+}

src/Test/L0/Worker/WorkerL0.cs

@@ -67,7 +67,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                     new Pipelines.ContextData.DictionaryContextData()
                 },
             };
-            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List<MaskHint>(), resources, context, null, actions, null, null, null, null);
+            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List<MaskHint>(), resources, context, null, actions, null, null, null, null, null);
             return jobRequest;
         }
 

src/dev.sh

@@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout"
 DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x"
 PACKAGE_DIR="$SCRIPT_DIR/../_package"
 DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk"
-DOTNETSDK_VERSION="6.0.418"
+DOTNETSDK_VERSION="6.0.419"
 DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION"
 RUNNER_VERSION=$(cat runnerversion)
 

src/global.json

@@ -1,5 +1,5 @@
 {
   "sdk": {
-    "version": "6.0.418"
+    "version": "6.0.419"
   }
 }

src/runnerversion

@@ -1 +1 @@
-2.312.0
+2.314.0