Release 2.316.0

* Cleanup FF 'DistributedTask.UseWhich2'.

* more ff.

.devcontainer/devcontainer.json

@@ -4,13 +4,13 @@
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:1": {},
     "ghcr.io/devcontainers/features/dotnet": {
-      "version": "6.0.418"
+      "version": "6.0.421"
     },
     "ghcr.io/devcontainers/features/node:1": {
       "version": "16"
     },
     "ghcr.io/devcontainers/features/sshd:1": {
-        "version": "latest"
+      "version": "latest"
     }
   },
   "customizations": {

images/Dockerfile

@@ -4,9 +4,9 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build
 ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.5.1
-ARG DOCKER_VERSION=25.0.2
-ARG BUILDX_VERSION=0.12.1
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.0
+ARG DOCKER_VERSION=25.0.4
+ARG BUILDX_VERSION=0.13.1
 
 RUN apt update -y && apt install curl unzip -y
 

releaseNote.md

@@ -1,29 +1,21 @@
 ## What's Changed
-* Fix `buildx` installation by @ajschmidt8 in https://github.com/actions/runner/pull/2952
-* Create close-features and close-bugs bot for runner issues by @ruvceskistefan in https://github.com/actions/runner/pull/2909
-* Send disableUpdate as query parameter  by @luketomlinson in https://github.com/actions/runner/pull/2970
-* Handle SelfUpdate Flow when Package is provided in Message  by @luketomlinson in https://github.com/actions/runner/pull/2926
-* Bump container hook version to 0.5.0 in runner image by @nikola-jokic in https://github.com/actions/runner/pull/3003
-* Set `ImageOS` environment variable in runner images by @int128 in https://github.com/actions/runner/pull/2878
-* Mark job as failed on worker crash. by @TingluoHuang in https://github.com/actions/runner/pull/3006
-* Include whether http proxy configured as part of UserAgent. by @TingluoHuang in https://github.com/actions/runner/pull/3009
-* Add codeload to the list of service we check during '--check'. by @TingluoHuang in https://github.com/actions/runner/pull/3011
-* close reason update by @ruvceskistefan in https://github.com/actions/runner/pull/3027
-* Update envlinux.md by @adjn in https://github.com/actions/runner/pull/3040
-* Extend `--check` to check Results-Receiver service. by @TingluoHuang in https://github.com/actions/runner/pull/3078
-* Use Azure SDK to upload files to Azure Blob by @yacaovsnc in https://github.com/actions/runner/pull/3033
-* Remove code in runner for handling trimmed packages. by @TingluoHuang in https://github.com/actions/runner/pull/3074
-* Update dotnet sdk to latest version @6.0.418 by @github-actions in https://github.com/actions/runner/pull/3085
-* Patch Curl to no longer use -k by @thboop in https://github.com/actions/runner/pull/3091
+* Load '_runnerSettings' in the early point of JobRunner.cs by @TingluoHuang in https://github.com/actions/runner/pull/3218
+* Add new SessionConflict return code by @eeSquared in https://github.com/actions/runner/pull/3215
+* backoff if we retried polling for more than 50 times in less than 30minutes by @aiqiaoy in https://github.com/actions/runner/pull/3232
+* Update dotnet sdk to latest version @6.0.421 by @github-actions in https://github.com/actions/runner/pull/3244
+* Cleanup enabled feature flags. by @TingluoHuang in https://github.com/actions/runner/pull/3246
+* Relax the condition to stop uploading to Results by @yacaovsnc in https://github.com/actions/runner/pull/3230
+* Cleanup enabled feature flags. by @TingluoHuang in https://github.com/actions/runner/pull/3248
+* Replace invalid file name chars in diag log name by @ericsciple in https://github.com/actions/runner/pull/3249
 
 ## New Contributors
-* @int128 made their first contribution in https://github.com/actions/runner/pull/2878
-* @adjn made their first contribution in https://github.com/actions/runner/pull/3040
+* @eeSquared made their first contribution in https://github.com/actions/runner/pull/3215
+* @aiqiaoy made their first contribution in https://github.com/actions/runner/pull/3232
 
-**Full Changelog**: https://github.com/actions/runner/compare/v2.311.0...v2.312.0
+**Full Changelog**: https://github.com/actions/runner/compare/v2.315.0...v2.316.0
 
-_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. 
-To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. 
+_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet.
+To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.
 See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_
 
 ## Windows x64

releaseVersion

@@ -1 +1 @@
-<Update to ./src/runnerversion when creating release>
+2.316.0

src/Misc/layoutbin/RunnerService.js

@@ -114,6 +114,11 @@ var runService = function () {
             );
             stopping = true;
           }
+        } else if (code === 5) {
+          console.log(
+            "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+          );
+          stopping = true;
         } else {
           var messagePrefix = "Runner listener exit with undefined return code";
           unknownFailureRetryCount++;

src/Misc/layoutroot/run-helper.cmd.template

@@ -49,5 +49,10 @@ if %ERRORLEVEL% EQU 4 (
   exit /b 1
 )
 
+if %ERRORLEVEL% EQU 5 (
+  echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+  exit /b 0
+)
+
 echo "Exiting after unknown error code: %ERRORLEVEL%"
 exit /b 0

src/Misc/layoutroot/run-helper.sh.template

@@ -70,6 +70,9 @@ elif [[ $returnCode == 4 ]]; then
         "$DIR"/safe_sleep.sh 1
     done
     exit 2
+elif [[ $returnCode == 5 ]]; then
+    echo "Runner listener exit with Session Conflict error, stop the service, no retry needed."
+    exit 0
 else
     echo "Exiting with unknown error code: ${returnCode}"
     exit 0

src/Misc/layoutroot/run.sh

@@ -38,7 +38,7 @@ runWithManualTrap() {
         cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh
         "$DIR"/run-helper.sh $* &
         PID=$!
-        wait -f $PID
+        wait $PID
         returnCode=$?
         if [[ $returnCode -eq 2 ]]; then
             echo "Restarting runner..."
@@ -84,4 +84,4 @@ if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then
     run $*
 else
     runWithManualTrap $*
-fi
+fi

src/Runner.Common/BrokerServer.cs

@@ -21,6 +21,10 @@ namespace GitHub.Runner.Common
         Task DeleteSessionAsync(CancellationToken cancellationToken);
 
         Task<TaskAgentMessage> GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken token);
+
+        Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials);
+
+        Task ForceRefreshConnection(VssCredentials credentials);
     }
 
     public sealed class BrokerServer : RunnerService, IBrokerServer
@@ -59,7 +63,8 @@ namespace GitHub.Runner.Common
         {
             CheckConnection();
             var brokerSession = RetryRequest<TaskAgentMessage>(
-                async () => await _brokerHttpClient.GetRunnerMessageAsync(sessionId, version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken);
+                async () => await _brokerHttpClient.GetRunnerMessageAsync(sessionId, version, status, os, architecture, disableUpdate, cancellationToken), cancellationToken, shouldRetry: ShouldRetryException);
+
 
             return brokerSession;
         }
@@ -69,5 +74,30 @@ namespace GitHub.Runner.Common
             CheckConnection();
             await _brokerHttpClient.DeleteSessionAsync(cancellationToken);
         }
+
+        public Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials)
+        {
+            if (_brokerUri != serverUri || !_hasConnection)
+            {
+                return ConnectAsync(serverUri, credentials);
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task ForceRefreshConnection(VssCredentials credentials)
+        {
+            return ConnectAsync(_brokerUri, credentials);
+        }
+
+        public bool ShouldRetryException(Exception ex)
+        {
+            if (ex is AccessDeniedException ade && ade.ErrorCode == 1)
+            {
+                return false;
+            }
+
+            return true;
+        }
     }
 }

src/Runner.Common/Constants.cs

@@ -153,6 +153,7 @@ namespace GitHub.Runner.Common
                 public const int RetryableError = 2;
                 public const int RunnerUpdating = 3;
                 public const int RunOnceRunnerUpdating = 4;
+                public const int SessionConflict = 5;
             }
 
             public static class Features
@@ -180,6 +181,9 @@ namespace GitHub.Runner.Common
             public static readonly string DeprecatedNodeVersion = "node16";
             public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/";
             public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings";
+            public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions uses Node.js version which is deprecated and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/";
+            public static readonly string EnforcedNode16DetectedAfterEndOfLifeEnvVariable = "Node20ForceActionsWarnings";
+
         }
 
         public static class RunnerEvent
@@ -251,6 +255,7 @@ namespace GitHub.Runner.Common
                 public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
                 public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
                 public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION";
+                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
             }
 
             public static class Agent
@@ -262,6 +267,7 @@ namespace GitHub.Runner.Common
                 public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION";
                 public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT";
                 public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE";
+                public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20";
             }
 
             public static class System

src/Runner.Common/JobServerQueue.cs

@@ -19,7 +19,7 @@ namespace GitHub.Runner.Common
         TaskCompletionSource<int> JobRecordUpdated { get; }
         event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
         Task ShutdownAsync();
-        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false);
+        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false);
         void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
         void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
         void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines);
@@ -74,6 +74,7 @@ namespace GitHub.Runner.Common
         private readonly List<JobTelemetry> _jobTelemetries = new();
         private bool _queueInProcess = false;
         private bool _resultsServiceOnly = false;
+        private int _resultsServiceExceptionsCount = 0;
         private Stopwatch _resultsUploadTimer = new();
         private Stopwatch _actionsUploadTimer = new();
 
@@ -104,11 +105,10 @@ namespace GitHub.Runner.Common
             _resultsServer = hostContext.GetService<IResultsServer>();
         }
 
-        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false)
+        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false)
         {
             Trace.Entering();
             _resultsServiceOnly = resultsServiceOnly;
-            _enableTelemetry = enableTelemetry;
 
             var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
 
@@ -139,6 +139,12 @@ namespace GitHub.Runner.Common
                 _resultsClientInitiated = true;
             }
 
+            // Enable telemetry if we have both results service and actions service
+            if (_resultsClientInitiated && !_resultsServiceOnly)
+            {
+                _enableTelemetry = true;
+            }
+
             if (_queueInProcess)
             {
                 Trace.Info("No-opt, all queue process tasks are running.");
@@ -574,9 +580,9 @@ namespace GitHub.Runner.Common
                             Trace.Info("Catch exception during file upload to results, keep going since the process is best effort.");
                             Trace.Error(ex);
                             errorCount++;
-
+                            _resultsServiceExceptionsCount++;
                             // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs
-                            if (!_resultsServiceOnly)
+                            if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3)
                             {
                                 _resultsClientInitiated = false;
                                 SendResultsTelemetry(ex);

src/Runner.Common/RunServer.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using GitHub.Actions.RunService.WebApi;
 using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using Sdk.RSWebApi.Contracts;
@@ -60,7 +61,7 @@ namespace GitHub.Runner.Common
         {
             CheckConnection();
             return RetryRequest<AgentJobRequestMessage>(
-                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken,
+                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, cancellationToken), cancellationToken,
                 shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException);
         }
 

src/Runner.Listener/BrokerMessageListener.cs

@@ -42,7 +42,7 @@ namespace GitHub.Runner.Listener
             _brokerServer = HostContext.GetService<IBrokerServer>();
         }
 
-        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
         {
             Trace.Entering();
 
@@ -99,7 +99,7 @@ namespace GitHub.Runner.Listener
                         encounteringError = false;
                     }
 
-                    return true;
+                    return CreateSessionResult.Success;
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
                 {
@@ -123,7 +123,7 @@ namespace GitHub.Runner.Listener
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
 
                         // Check whether we get 401 because the runner registration already removed by the service.
@@ -134,14 +134,18 @@ namespace GitHub.Runner.Listener
                         if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
                     }
 
                     if (!IsSessionCreationExceptionRetriable(ex))
                     {
                         _term.WriteError($"Failed to create session. {ex.Message}");
-                        return false;
+                        if (ex is TaskAgentSessionConflictException)
+                        {
+                            return CreateSessionResult.SessionConflict;
+                        }
+                        return CreateSessionResult.Failure;
                     }
 
                     if (!encounteringError) //print the message only on the first error

src/Runner.Listener/JobDispatcher.cs

@@ -35,7 +35,7 @@ namespace GitHub.Runner.Listener
     // This implementation of IJobDispatcher is not thread safe.
     // It is based on the fact that the current design of the runner is a dequeue
     // and processes one message from the message queue at a time.
-    // In addition, it only executes one job every time, 
+    // In addition, it only executes one job every time,
     // and the server will not send another job while this one is still running.
     public sealed class JobDispatcher : RunnerService, IJobDispatcher
     {
@@ -546,13 +546,27 @@ namespace GitHub.Runner.Listener
                                     Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
 
                                     var jobServer = await InitializeJobServerAsync(systemConnection);
-                                    await LogWorkerProcessUnhandledException(jobServer, message, detailInfo);
-
-                                    // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
-                                    if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
+                                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
+                                    switch (jobServer)
                                     {
-                                        Trace.Info($"Finish job with result 'Failed' due to IOException.");
-                                        await ForceFailJob(jobServer, message, detailInfo);
+                                        case IJobServer js:
+                                            {
+                                                await LogWorkerProcessUnhandledException(js, message, unhandledExceptionIssue);
+                                                // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
+                                                if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase))
+                                                {
+                                                    Trace.Info($"Finish job with result 'Failed' due to IOException.");
+                                                    await ForceFailJob(js, message);
+                                                }
+
+                                                break;
+                                            }
+                                        case IRunServer rs:
+                                            await ForceFailJob(rs, message, unhandledExceptionIssue);
+                                            break;
+                                        default:
+                                            throw new NotSupportedException($"JobServer type '{jobServer.GetType().Name}' is not supported.");
                                     }
                                 }
 
@@ -644,7 +658,7 @@ namespace GitHub.Runner.Listener
                                 }
                             }
 
-                            // wait worker to exit 
+                            // wait worker to exit
                             // if worker doesn't exit within timeout, then kill worker.
                             completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
 
@@ -1131,86 +1145,65 @@ namespace GitHub.Runner.Listener
         }
 
         // log an error issue to job level timeline record
-        private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipelines.AgentJobRequestMessage message, Issue issue)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
-                {
-                    var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
-                    ArgUtil.NotNull(timeline, nameof(timeline));
+                var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
+                ArgUtil.NotNull(timeline, nameof(timeline));
 
-                    TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
-                    ArgUtil.NotNull(jobRecord, nameof(jobRecord));
+                TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
+                ArgUtil.NotNull(jobRecord, nameof(jobRecord));
 
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
-                    jobRecord.ErrorCount++;
-                    jobRecord.Issues.Add(unhandledExceptionIssue);
+                jobRecord.ErrorCount++;
+                jobRecord.Issues.Add(issue);
 
-                    if (message.Variables.TryGetValue("DistributedTask.MarkJobAsFailedOnWorkerCrash", out var markJobAsFailedOnWorkerCrash) &&
-                        StringUtil.ConvertToBoolean(markJobAsFailedOnWorkerCrash?.Value))
-                    {
-                        Trace.Info("Mark the job as failed since the worker crashed");
-                        jobRecord.Result = TaskResult.Failed;
-                        // mark the job as completed so service will pickup the result
-                        jobRecord.State = TimelineRecordState.Completed;
-                    }
+                Trace.Info("Mark the job as failed since the worker crashed");
+                jobRecord.Result = TaskResult.Failed;
+                // mark the job as completed so service will pickup the result
+                jobRecord.State = TimelineRecordState.Completed;
 
-                    await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to report unhandled exception from Runner.Worker process");
-                    Trace.Error(ex);
-                }
+                await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
             }
-            else
+            catch (Exception ex)
             {
-                Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", detailInfo);
-                return;
+                Trace.Error("Fail to report unhandled exception from Runner.Worker process");
+                Trace.Error(ex);
             }
         }
 
         // raise job completed event to fail the job.
-        private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo)
+        private async Task ForceFailJob(IJobServer jobServer, Pipelines.AgentJobRequestMessage message)
         {
-            if (server is IJobServer jobServer)
+            try
             {
-                try
-                {
-                    var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
-                    await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise JobCompletedEvent back to service.");
-                    Trace.Error(ex);
-                }
+                var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed);
+                await jobServer.RaisePlanEventAsync<JobCompletedEvent>(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None);
             }
-            else if (server is IRunServer runServer)
+            catch (Exception ex)
             {
-                try
-                {
-                    var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo };
-                    var unhandledAnnotation = unhandledExceptionIssue.ToAnnotation();
-                    var jobAnnotations = new List<Annotation>();
-                    if (unhandledAnnotation.HasValue)
-                    {
-                        jobAnnotations.Add(unhandledAnnotation.Value);
-                    }
+                Trace.Error("Fail to raise JobCompletedEvent back to service.");
+                Trace.Error(ex);
+            }
+        }
 
-                    await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error("Fail to raise job completion back to service.");
-                    Trace.Error(ex);
-                }
-            }
-            else
+        private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestMessage message, Issue issue)
+        {
+            try
             {
-                throw new NotSupportedException($"Server type {server.GetType().FullName} is not supported.");
+                var annotation = issue.ToAnnotation();
+                var jobAnnotations = new List<Annotation>();
+                if (annotation.HasValue)
+                {
+                    jobAnnotations.Add(annotation.Value);
+                }
+
+                await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None);
+            }
+            catch (Exception ex)
+            {
+                Trace.Error("Fail to raise job completion back to service.");
+                Trace.Error(ex);
             }
         }
 

src/Runner.Listener/MessageListener.cs

@@ -18,10 +18,17 @@ using GitHub.Services.WebApi;
 
 namespace GitHub.Runner.Listener
 {
+    public enum CreateSessionResult
+    {
+        Success,
+        Failure,
+        SessionConflict
+    }
+
     [ServiceLocator(Default = typeof(MessageListener))]
     public interface IMessageListener : IRunnerService
     {
-        Task<Boolean> CreateSessionAsync(CancellationToken token);
+        Task<CreateSessionResult> CreateSessionAsync(CancellationToken token);
         Task DeleteSessionAsync();
         Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token);
         Task DeleteMessageAsync(TaskAgentMessage message);
@@ -59,7 +66,7 @@ namespace GitHub.Runner.Listener
             _brokerServer = hostContext.GetService<IBrokerServer>();
         }
 
-        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        public async Task<CreateSessionResult> CreateSessionAsync(CancellationToken token)
         {
             Trace.Entering();
 
@@ -109,7 +116,8 @@ namespace GitHub.Runner.Listener
                     if (_session.BrokerMigrationMessage != null)
                     {
                         Trace.Info("Runner session is in migration mode: Creating Broker session with BrokerBaseUrl: {0}", _session.BrokerMigrationMessage.BrokerBaseUrl);
-                        await _brokerServer.ConnectAsync(_session.BrokerMigrationMessage.BrokerBaseUrl, _creds);
+
+                        await _brokerServer.UpdateConnectionIfNeeded(_session.BrokerMigrationMessage.BrokerBaseUrl, _creds);
                         _session = await _brokerServer.CreateSessionAsync(taskAgentSession, token);
                         _isBrokerSession = true;
                     }
@@ -122,7 +130,7 @@ namespace GitHub.Runner.Listener
                         encounteringError = false;
                     }
 
-                    return true;
+                    return CreateSessionResult.Success;
                 }
                 catch (OperationCanceledException) when (token.IsCancellationRequested)
                 {
@@ -146,7 +154,7 @@ namespace GitHub.Runner.Listener
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
 
                         // Check whether we get 401 because the runner registration already removed by the service.
@@ -157,14 +165,18 @@ namespace GitHub.Runner.Listener
                         if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
                             _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
-                            return false;
+                            return CreateSessionResult.Failure;
                         }
                     }
 
                     if (!IsSessionCreationExceptionRetriable(ex))
                     {
                         _term.WriteError($"Failed to create session. {ex.Message}");
-                        return false;
+                        if (ex is TaskAgentSessionConflictException)
+                        {
+                            return CreateSessionResult.SessionConflict;
+                        }
+                        return CreateSessionResult.Failure;
                     }
 
                     if (!encounteringError) //print the message only on the first error
@@ -187,12 +199,12 @@ namespace GitHub.Runner.Listener
                 {
                     using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30)))
                     {
+                        await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token);
+
                         if (_isBrokerSession)
                         {
                             await _brokerServer.DeleteSessionAsync(ts.Token);
-                            return;
                         }
-                        await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token);
                     }
                 }
                 else
@@ -204,19 +216,17 @@ namespace GitHub.Runner.Listener
 
         public void OnJobStatus(object sender, JobStatusEventArgs e)
         {
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("USE_BROKER_FLOW")))
+            Trace.Info("Received job status event. JobState: {0}", e.Status);
+            runnerStatus = e.Status;
+            try
             {
-                Trace.Info("Received job status event. JobState: {0}", e.Status);
-                runnerStatus = e.Status;
-                try
-                {
-                    _getMessagesTokenSource?.Cancel();
-                }
-                catch (ObjectDisposedException)
-                {
-                    Trace.Info("_getMessagesTokenSource is already disposed.");
-                }
+                _getMessagesTokenSource?.Cancel();
             }
+            catch (ObjectDisposedException)
+            {
+                Trace.Info("_getMessagesTokenSource is already disposed.");
+            }
+
         }
 
         public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
@@ -226,6 +236,7 @@ namespace GitHub.Runner.Listener
             ArgUtil.NotNull(_settings, nameof(_settings));
             bool encounteringError = false;
             int continuousError = 0;
+            int continuousEmptyMessage = 0;
             string errorMessage = string.Empty;
             Stopwatch heartbeat = new();
             heartbeat.Restart();
@@ -256,7 +267,7 @@ namespace GitHub.Runner.Listener
 
                         var migrationMessage = JsonUtility.FromString<BrokerMigrationMessage>(message.Body);
 
-                        await _brokerServer.ConnectAsync(migrationMessage.BrokerBaseUrl, _creds);
+                        await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _creds);
                         message = await _brokerServer.GetRunnerMessageAsync(_session.SessionId,
                                                                         runnerStatus,
                                                                         BuildConstants.RunnerPackage.Version,
@@ -304,7 +315,7 @@ namespace GitHub.Runner.Listener
                     Trace.Error(ex);
 
                     // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
-                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token))
+                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && (await CreateSessionAsync(token) == CreateSessionResult.Success))
                     {
                         Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session.");
                     }
@@ -349,16 +360,27 @@ namespace GitHub.Runner.Listener
 
                 if (message == null)
                 {
+                    continuousEmptyMessage++;
                     if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
                     {
                         Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
                         heartbeat.Restart();
+                        continuousEmptyMessage = 0;
                     }
                     else
                     {
                         Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
                     }
 
+                    if (continuousEmptyMessage > 50)
+                    {
+                        // retried more than 50 times in less than 30mins and still getting empty message
+                        // something is not right on the service side, backoff for 15-30s before retry
+                        _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
+                        Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                        await HostContext.Delay(_getNextMessageRetryInterval, token);
+                    }
+
                     continue;
                 }
 
@@ -384,6 +406,7 @@ namespace GitHub.Runner.Listener
         public async Task RefreshListenerTokenAsync(CancellationToken cancellationToken)
         {
             await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+            await _brokerServer.ForceRefreshConnection(_creds);
         }
 
         private TaskAgentMessage DecryptMessage(TaskAgentMessage message)

src/Runner.Listener/Runner.cs

@@ -359,7 +359,12 @@ namespace GitHub.Runner.Listener
             {
                 Trace.Info(nameof(RunAsync));
                 _listener = GetMesageListener(settings);
-                if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken))
+                CreateSessionResult createSessionResult = await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken);
+                if (createSessionResult == CreateSessionResult.SessionConflict)
+                {
+                    return Constants.Runner.ReturnCode.SessionConflict;
+                }
+                else if (createSessionResult == CreateSessionResult.Failure)
                 {
                     return Constants.Runner.ReturnCode.TerminatedError;
                 }
@@ -567,6 +572,11 @@ namespace GitHub.Runner.Listener
                                             Trace.Info("Job is already acquired, skip this message.");
                                             continue;
                                         }
+                                        catch (Exception ex)
+                                        {
+                                            Trace.Error($"Caught exception from acquiring job message: {ex}");
+                                            continue;
+                                        }
                                     }
 
                                     jobDispatcher.Run(jobRequestMessage, runOnce);
@@ -598,6 +608,7 @@ namespace GitHub.Runner.Listener
                             }
                             else if (string.Equals(message.MessageType, TaskAgentMessageTypes.ForceTokenRefresh))
                             {
+                                Trace.Info("Received ForceTokenRefreshMessage");
                                 await _listener.RefreshListenerTokenAsync(messageQueueLoopTokenSource.Token);
                             }
                             else

src/Runner.Sdk/Util/IOUtil.cs

@@ -459,6 +459,34 @@ namespace GitHub.Runner.Sdk
             File.WriteAllText(path, null);
         }
 
+        /// <summary>
+        /// Replaces invalid file name characters with '_'
+        /// </summary>
+        public static string ReplaceInvalidFileNameChars(string fileName)
+        {
+            var result = new StringBuilder();
+            var invalidChars = Path.GetInvalidFileNameChars();
+
+            var current = 0; // Current index
+            while (current < fileName?.Length)
+            {
+                var next = fileName.IndexOfAny(invalidChars, current);
+                if (next >= 0)
+                {
+                    result.Append(fileName.Substring(current, next - current));
+                    result.Append('_');
+                    current = next + 1;
+                }
+                else
+                {
+                    result.Append(fileName.Substring(current));
+                    break;
+                }
+            }
+
+            return result.ToString();
+        }
+
         /// <summary>
         /// Recursively enumerates a directory without following directory reparse points.
         /// </summary>

src/Runner.Sdk/Util/VssUtil.cs

@@ -85,11 +85,6 @@ namespace GitHub.Runner.Sdk
                 settings.SendTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200));
             }
 
-            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("USE_BROKER_FLOW")))
-            {
-                settings.AllowAutoRedirectForBroker = true;
-            }
-
             // Remove Invariant from the list of accepted languages.
             //
             // The constructor of VssHttpRequestSettings (base class of VssClientHttpRequestSettings) adds the current

src/Runner.Sdk/Util/WhichUtil.cs

@@ -7,129 +7,6 @@ namespace GitHub.Runner.Sdk
     public static class WhichUtil
     {
         public static string Which(string command, bool require = false, ITraceWriter trace = null, string prependPath = null)
-        {
-            ArgUtil.NotNullOrEmpty(command, nameof(command));
-            trace?.Info($"Which: '{command}'");
-            if (Path.IsPathFullyQualified(command) && File.Exists(command))
-            {
-                trace?.Info($"Fully qualified path: '{command}'");
-                return command;
-            }
-            string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
-            if (string.IsNullOrEmpty(path))
-            {
-                trace?.Info("PATH environment variable not defined.");
-                path = path ?? string.Empty;
-            }
-            if (!string.IsNullOrEmpty(prependPath))
-            {
-                path = PathUtil.PrependPath(prependPath, path);
-            }
-
-            string[] pathSegments = path.Split(new Char[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries);
-            for (int i = 0; i < pathSegments.Length; i++)
-            {
-                pathSegments[i] = Environment.ExpandEnvironmentVariables(pathSegments[i]);
-            }
-
-            foreach (string pathSegment in pathSegments)
-            {
-                if (!string.IsNullOrEmpty(pathSegment) && Directory.Exists(pathSegment))
-                {
-                    string[] matches = null;
-#if OS_WINDOWS
-                    string pathExt = Environment.GetEnvironmentVariable("PATHEXT");
-                    if (string.IsNullOrEmpty(pathExt))
-                    {
-                        // XP's system default value for PATHEXT system variable
-                        pathExt = ".com;.exe;.bat;.cmd;.vbs;.vbe;.js;.jse;.wsf;.wsh";
-                    }
-
-                    string[] pathExtSegments = pathExt.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
-
-                    // if command already has an extension.
-                    if (pathExtSegments.Any(ext => command.EndsWith(ext, StringComparison.OrdinalIgnoreCase)))
-                    {
-                        try
-                        {
-                            matches = Directory.GetFiles(pathSegment, command);
-                        }
-                        catch (UnauthorizedAccessException ex)
-                        {
-                            trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                            trace?.Verbose(ex.ToString());
-                        }
-
-                        if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
-                        {
-                            trace?.Info($"Location: '{matches.First()}'");
-                            return matches.First();
-                        }
-                    }
-                    else
-                    {
-                        string searchPattern;
-                        searchPattern = StringUtil.Format($"{command}.*");
-                        try
-                        {
-                            matches = Directory.GetFiles(pathSegment, searchPattern);
-                        }
-                        catch (UnauthorizedAccessException ex)
-                        {
-                            trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                            trace?.Verbose(ex.ToString());
-                        }
-
-                        if (matches != null && matches.Length > 0)
-                        {
-                            // add extension.
-                            for (int i = 0; i < pathExtSegments.Length; i++)
-                            {
-                                string fullPath = Path.Combine(pathSegment, $"{command}{pathExtSegments[i]}");
-                                if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)) && IsPathValid(fullPath, trace))
-                                {
-                                    trace?.Info($"Location: '{fullPath}'");
-                                    return fullPath;
-                                }
-                            }
-                        }
-                    }
-#else
-                    try
-                    {
-                        matches = Directory.GetFiles(pathSegment, command);
-                    }
-                    catch (UnauthorizedAccessException ex)
-                    {
-                        trace?.Info("Ignore UnauthorizedAccess exception during Which.");
-                        trace?.Verbose(ex.ToString());
-                    }
-
-                    if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
-                    {
-                        trace?.Info($"Location: '{matches.First()}'");
-                        return matches.First();
-                    }
-#endif
-                }
-            }
-
-#if OS_WINDOWS
-            trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'Path' environment variable.");
-#else
-            trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'PATH' environment variable.");
-#endif
-            if (require)
-            {
-                throw new FileNotFoundException(
-                    message: $"{command}: command not found",
-                    fileName: command);
-            }
-
-            return null;
-        }
-
-        public static string Which2(string command, bool require = false, ITraceWriter trace = null, string prependPath = null)
         {
             ArgUtil.NotNullOrEmpty(command, nameof(command));
             trace?.Info($"Which2: '{command}'");

src/Runner.Worker/ActionManager.cs

@@ -483,10 +483,6 @@ namespace GitHub.Runner.Worker
                             {
                                 // Load stored Ids for later load actions
                                 compositeAction.Steps[i].Id = _cachedEmbeddedStepIds[action.Id][i];
-                                if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && compositeAction.Steps[i].Reference.Type != Pipelines.ActionSourceType.Script)
-                                {
-                                    throw new Exception("`uses:` keyword is not currently supported.");
-                                }
                             }
                         }
                         else
@@ -703,11 +699,12 @@ namespace GitHub.Runner.Worker
                 catch (Exception ex) when (!executionContext.CancellationToken.IsCancellationRequested) // Do not retry if the run is cancelled.
                 {
                     // UnresolvableActionDownloadInfoException is a 422 client error, don't retry
+                    // NonRetryableActionDownloadInfoException is an non-retryable exception from Actions
                     // Some possible cases are:
                     // * Repo is rate limited
                     // * Repo or tag doesn't exist, or isn't public
                     // * Policy validation failed
-                    if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException))
+                    if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException) && !(ex is WebApi.NonRetryableActionDownloadInfoException))
                     {
                         executionContext.Output($"Failed to resolve action download info. Error: {ex.Message}");
                         executionContext.Debug(ex.ToString());
@@ -796,43 +793,40 @@ namespace GitHub.Runner.Worker
             try
             {
                 var useActionArchiveCache = false;
-                if (executionContext.Global.Variables.GetBoolean("DistributedTask.UseActionArchiveCache") == true)
+                var hasActionArchiveCache = false;
+                var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory);
+                if (!string.IsNullOrEmpty(actionArchiveCacheDir) &&
+                    Directory.Exists(actionArchiveCacheDir))
                 {
-                    var hasActionArchiveCache = false;
-                    var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory);
-                    if (!string.IsNullOrEmpty(actionArchiveCacheDir) &&
-                        Directory.Exists(actionArchiveCacheDir))
-                    {
-                        hasActionArchiveCache = true;
-                        Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'");
+                    hasActionArchiveCache = true;
+                    Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'");
 #if OS_WINDOWS
-                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip");
+                    var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip");
 #else
-                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz");
+                    var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz");
 #endif
-                        if (File.Exists(cacheArchiveFile))
+                    if (File.Exists(cacheArchiveFile))
+                    {
+                        try
                         {
-                            try
-                            {
-                                Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'");
-                                File.Copy(cacheArchiveFile, archiveFile);
-                                useActionArchiveCache = true;
-                                executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'");
-                            }
-                            catch (Exception ex)
-                            {
-                                Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}");
-                            }
+                            Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'");
+                            File.Copy(cacheArchiveFile, archiveFile);
+                            useActionArchiveCache = true;
+                            executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'");
+                        }
+                        catch (Exception ex)
+                        {
+                            Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}");
                         }
                     }
-
-                    executionContext.Global.JobTelemetry.Add(new JobTelemetry()
-                    {
-                        Type = JobTelemetryType.General,
-                        Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
-                    });
                 }
 
+                executionContext.Global.JobTelemetry.Add(new JobTelemetry()
+                {
+                    Type = JobTelemetryType.General,
+                    Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
+                });
+
                 if (!useActionArchiveCache)
                 {
                     await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile);
@@ -878,16 +872,9 @@ namespace GitHub.Runner.Worker
                     int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
                     if (exitCode != 0)
                     {
-                        if (executionContext.Global.Variables.GetBoolean("DistributedTask.DetailUntarFailure") == true)
-                        {
-                            var fileInfo = new FileInfo(archiveFile);
-                            var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile);
-                            throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
-                        }
-                        else
-                        {
-                            throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
-                        }
+                        var fileInfo = new FileInfo(archiveFile);
+                        var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile);
+                        throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}.");
                     }
                 }
 #endif
@@ -1031,13 +1018,6 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
-                    foreach (var step in compositeAction.Steps)
-                    {
-                        if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && step.Reference.Type != Pipelines.ActionSourceType.Script)
-                        {
-                            throw new Exception("`uses:` keyword is not currently supported.");
-                        }
-                    }
                     return setupInfo;
                 }
                 else

src/Runner.Worker/ContainerOperationProvider.cs

@@ -466,17 +466,39 @@ namespace GitHub.Runner.Worker
             {
                 throw new InvalidOperationException($"Failed to create directory to store registry client credentials: {e.Message}");
             }
-            var loginExitCode = await _dockerManager.DockerLogin(
-                executionContext,
-                configLocation,
-                container.RegistryServer,
-                container.RegistryAuthUsername,
-                container.RegistryAuthPassword);
 
-            if (loginExitCode != 0)
+            // Login docker with retry up to 3 times
+            int retryCount = 0;
+            int loginExitCode = 0;
+            while (retryCount < 3)
+            {
+                loginExitCode = await _dockerManager.DockerLogin(
+                    executionContext,
+                    configLocation,
+                    container.RegistryServer,
+                    container.RegistryAuthUsername,
+                    container.RegistryAuthPassword);
+                if (loginExitCode == 0)
+                {
+                    break;
+                }
+                else
+                {
+                    retryCount++;
+                    if (retryCount < 3)
+                    {
+                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
+                        executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
+                        await Task.Delay(backOff);
+                    }
+                }
+            }
+
+            if (retryCount == 3 && loginExitCode != 0)
             {
                 throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
             }
+
             return configLocation;
         }
 

src/Runner.Worker/DiagnosticLogManager.cs

@@ -91,13 +91,13 @@ namespace GitHub.Runner.Worker
             string phaseName = executionContext.Global.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
 
             // zip the files
-            string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
+            string diagnosticsZipFileName = $"{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.zip";
             string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
             ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
 
             // upload the json metadata file
             executionContext.Debug("Uploading diagnostic metadata file.");
-            string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
+            string metadataFileName = $"diagnostics-{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.json";
             string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
             string phaseResult = GetTaskResultAsString(executionContext.Result);
 

src/Runner.Worker/FileCommandManager.cs

@@ -244,7 +244,7 @@ namespace GitHub.Runner.Worker
                 if (resultsReceiverEndpoint != null)
                 {
                     Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})");
-                    var stepId = context.Id;
+                    var stepId = context.IsEmbedded ? context.EmbeddedId : context.Id;
                     // Attachments must be added to the parent context (job), not the current context (step)
                     context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId);
                 }

src/Runner.Worker/Handlers/ContainerActionHandler.cs

@@ -223,6 +223,10 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Environment["ACTIONS_CACHE_URL"] = cacheUrl;
             }
+            if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl))
+            {
+                Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl;
+            }
             if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
             {
                 Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;

src/Runner.Worker/Handlers/HandlerFactory.cs

@@ -84,6 +84,45 @@ namespace GitHub.Runner.Worker.Handlers
                     }
                     nodeData.NodeVersion = "node16";
                 }
+
+                var localForceActionsToNode20 = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Agent.ManualForceActionsToNode20));
+                executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.ManualForceActionsToNode20, out var workflowForceActionsToNode20);
+                var enforceNode20Locally = !string.IsNullOrWhiteSpace(workflowForceActionsToNode20) ? StringUtil.ConvertToBoolean(workflowForceActionsToNode20) : localForceActionsToNode20;
+                if (string.Equals(nodeData.NodeVersion, "node16")
+                && ((executionContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false) || enforceNode20Locally))
+                {
+                    executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion, out var workflowOptOut);
+                    var isWorkflowOptOutSet = !string.IsNullOrWhiteSpace(workflowOptOut);
+                    var isLocalOptOut = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion));
+                    bool isOptOut = isWorkflowOptOutSet ? StringUtil.ConvertToBoolean(workflowOptOut) : isLocalOptOut;
+
+                    if (!isOptOut)
+                    {
+                        var repoAction = action as Pipelines.RepositoryPathReference;
+                        if (repoAction != null)
+                        {
+                            var warningActions = new HashSet<string>();
+                            if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings))
+                            {
+                                warningActions = StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings);
+                            }
+
+                            string repoActionFullName;
+                            if (string.IsNullOrEmpty(repoAction.Name))
+                            {
+                                repoActionFullName = repoAction.Path; // local actions don't have a 'Name'
+                            }
+                            else
+                            {
+                                repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}";
+                            }
+
+                            warningActions.Add(repoActionFullName);
+                            executionContext.Global.Variables.Set(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, StringUtil.ConvertToJson(warningActions));
+                        }
+                        nodeData.NodeVersion = "node20";
+                    }
+                }
                 (handler as INodeScriptActionHandler).Data = nodeData;
             }
             else if (data.ExecutionType == ActionExecutionType.Script)

src/Runner.Worker/Handlers/NodeScriptActionHandler.cs

@@ -58,6 +58,10 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Environment["ACTIONS_CACHE_URL"] = cacheUrl;
             }
+            if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl))
+            {
+                Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl;
+            }
             if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
             {
                 Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;
@@ -114,6 +118,11 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 Data.NodeVersion = "node16";
             }
+
+            if (forcedNodeVersion == "node20" && Data.NodeVersion != "node20")
+            {
+                Data.NodeVersion = "node20";
+            }
             var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion);
             string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}");
 

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -83,40 +83,19 @@ namespace GitHub.Runner.Worker.Handlers
                 shellCommand = "pwsh";
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
                     if (string.IsNullOrEmpty(shellCommandPath))
                     {
                         shellCommand = "powershell";
-                        Trace.Info($"Defaulting to {shellCommand}");
-                        if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                        {
-                            shellCommandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath);
-                        }
-                        else
-                        {
-                            shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
-                        }
+                        Trace.Info($"Defaulting to {shellCommand}");                        
+                        shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
                     }
                 }
 #else
                 shellCommand = "sh";
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
                 }
 #endif
                 argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
@@ -127,14 +106,7 @@ namespace GitHub.Runner.Worker.Handlers
                 shellCommand = parsed.shellCommand;
                 if (validateShellOnHost)
                 {
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        shellCommandPath = WhichUtil.Which2(parsed.shellCommand, true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath);
-                    }
+                    shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath);
                 }
 
                 argFormat = $"{parsed.shellArgs}".TrimStart();
@@ -216,38 +188,17 @@ namespace GitHub.Runner.Worker.Handlers
             {
 #if OS_WINDOWS
                 shellCommand = "pwsh";
-                if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                {
-                    commandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath);
-                }
-                else
-                {
-                    commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
-                }
+                commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath);
                 if (string.IsNullOrEmpty(commandPath))
                 {
                     shellCommand = "powershell";
                     Trace.Info($"Defaulting to {shellCommand}");
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath);
                 }
                 ArgUtil.NotNullOrEmpty(commandPath, "Default Shell");
 #else
                 shellCommand = "sh";
-                if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                {
-                    commandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath);
-                }
-                else
-                {
-                    commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
-                }
+                commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath);
 #endif
                 argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
             }
@@ -258,14 +209,7 @@ namespace GitHub.Runner.Worker.Handlers
                 if (!IsActionStep && systemShells.Contains(shell))
                 {
                     shellCommand = shell;
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(shell, !isContainerStepHost, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
                     if (shell == "bash")
                     {
                         argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat("sh");
@@ -280,14 +224,7 @@ namespace GitHub.Runner.Worker.Handlers
                     var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell);
                     shellCommand = parsed.shellCommand;
                     // For non-ContainerStepHost, the command must be located on the host by Which
-                    if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true)
-                    {
-                        commandPath = WhichUtil.Which2(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
-                    }
-                    else
-                    {
-                        commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
-                    }
+                    commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
                     argFormat = $"{parsed.shellArgs}".TrimStart();
                     if (string.IsNullOrEmpty(argFormat))
                     {

src/Runner.Worker/JobExtension.cs

@@ -392,6 +392,18 @@ namespace GitHub.Runner.Worker
                         }
                     }
 
+                    // Register custom image creation post-job step if the "snapshot" token is present in the message.
+                    var snapshotRequest = templateEvaluator.EvaluateJobSnapshotRequest(message.Snapshot, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
+                    if (snapshotRequest != null)
+                    {
+                        var snapshotOperationProvider = HostContext.GetService<ISnapshotOperationProvider>();
+                        jobContext.RegisterPostJobStep(new JobExtensionRunner(
+                            runAsync: (executionContext, _) => snapshotOperationProvider.CreateSnapshotRequestAsync(executionContext, snapshotRequest),
+                            condition: $"{PipelineTemplateConstants.Success}()",
+                            displayName: $"Create custom image",
+                            data: null));
+                    }
+
                     // Register Job Completed hook if the variable is set
                     var completedHookPath = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_HOOK_JOB_COMPLETED");
                     if (!string.IsNullOrEmpty(completedHookPath))

src/Runner.Worker/JobRunner.cs

@@ -42,6 +42,7 @@ namespace GitHub.Runner.Worker
             Trace.Info("Job ID {0}", message.JobId);
 
             DateTime jobStartTimeUtc = DateTime.UtcNow;
+            _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
             IRunnerService server = null;
 
             // add orchestration id to useragent for better correlation.
@@ -54,13 +55,6 @@ namespace GitHub.Runner.Worker
                 VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
             }
 
-            var jobServerQueueTelemetry = false;
-            if (message.Variables.TryGetValue("DistributedTask.EnableJobServerQueueTelemetry", out VariableValue enableJobServerQueueTelemetry) &&
-                !string.IsNullOrEmpty(enableJobServerQueueTelemetry?.Value))
-            {
-                jobServerQueueTelemetry = StringUtil.ConvertToBoolean(enableJobServerQueueTelemetry.Value);
-            }
-
             ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
             if (MessageUtil.IsRunServiceJob(message.MessageType))
             {
@@ -82,7 +76,7 @@ namespace GitHub.Runner.Worker
                     launchServer.InitializeLaunchClient(new Uri(launchReceiverEndpoint), accessToken);
                 }
                 _jobServerQueue = HostContext.GetService<IJobServerQueue>();
-                _jobServerQueue.Start(message, resultsServiceOnly: true, enableTelemetry: jobServerQueueTelemetry);
+                _jobServerQueue.Start(message, resultsServiceOnly: true);
             }
             else
             {
@@ -104,7 +98,7 @@ namespace GitHub.Runner.Worker
                 VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, delegatingHandlers);
                 await jobServer.ConnectAsync(jobConnection);
 
-                _jobServerQueue.Start(message, enableTelemetry: jobServerQueueTelemetry);
+                _jobServerQueue.Start(message);
                 server = jobServer;
             }
 
@@ -164,8 +158,6 @@ namespace GitHub.Runner.Worker
 
                 jobContext.SetRunnerContext("os", VarUtil.OS);
                 jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture);
-
-                _runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
                 jobContext.SetRunnerContext("name", _runnerSettings.AgentName);
 
                 if (jobContext.Global.Variables.TryGetValue(WellKnownDistributedTaskVariables.RunnerEnvironment, out var runnerEnvironment))
@@ -298,6 +290,12 @@ namespace GitHub.Runner.Worker
                 jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
             }
 
+            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings) && (jobContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false))
+            {
+                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings));
+                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions));
+            }
+
             await ShutdownQueue(throwOnFailure: false);
 
             // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir.
@@ -405,6 +403,12 @@ namespace GitHub.Runner.Worker
                 jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions));
             }
 
+            if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings))
+            {
+                var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node20ForceWarnings));
+                jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions));
+            }
+
             try
             {
                 var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true);

src/Runner.Worker/SnapshotOperationProvider.cs

@@ -0,0 +1,32 @@
+#nullable enable
+using System.IO;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Runner.Common;
+using GitHub.Runner.Sdk;
+namespace GitHub.Runner.Worker;
+
+[ServiceLocator(Default = typeof(SnapshotOperationProvider))]
+public interface ISnapshotOperationProvider : IRunnerService
+{
+    Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest);
+}
+
+public class SnapshotOperationProvider : RunnerService, ISnapshotOperationProvider
+{
+    public Task CreateSnapshotRequestAsync(IExecutionContext executionContext, Snapshot snapshotRequest)
+    {
+        var snapshotRequestFilePath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), ".snapshot", "request.json");
+        var snapshotRequestDirectoryPath = Path.GetDirectoryName(snapshotRequestFilePath);
+        if (snapshotRequestDirectoryPath != null)
+        {
+            Directory.CreateDirectory(snapshotRequestDirectoryPath);
+        }
+
+        IOUtil.SaveObject(snapshotRequest, snapshotRequestFilePath);
+        executionContext.Output($"Request written to: {snapshotRequestFilePath}");
+        executionContext.Output("This request will be processed after the job completes. You will not receive any feedback on the snapshot process within the workflow logs of this job.");
+        executionContext.Output("If the snapshot process is successful, you should see a new image with the requested name in the list of available custom images when creating a new GitHub-hosted Runner.");
+        return Task.CompletedTask;
+    }
+}

src/Runner.Worker/StepsRunner.cs

@@ -295,7 +295,7 @@ namespace GitHub.Runner.Worker
                     !jobCancellationToken.IsCancellationRequested)
                 {
                     Trace.Error($"Caught timeout exception from step: {ex.Message}");
-                    step.ExecutionContext.Error("The action has timed out.");
+                    step.ExecutionContext.Error($"The action '{step.DisplayName}' has timed out after {timeoutMinutes} minutes.");
                     step.ExecutionContext.Result = TaskResult.Failed;
                 }
                 else

src/Sdk/Common/Common/VssHttpMessageHandler.cs

@@ -214,25 +214,7 @@ namespace GitHub.Services.Common
                     // ConfigureAwait(false) enables the continuation to be run outside any captured
                     // SyncronizationContext (such as ASP.NET's) which keeps things from deadlocking...
 
-                    var tmpResponse = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
-                    if (Settings.AllowAutoRedirectForBroker && tmpResponse.StatusCode == HttpStatusCode.Redirect)
-                    {
-                        //Dispose of the previous response
-                        tmpResponse?.Dispose();
-
-                        var location = tmpResponse.Headers.Location;
-                        request = new HttpRequestMessage(HttpMethod.Get, location);
-
-                        // Reapply the token to new redirected request
-                        ApplyToken(request, token, applyICredentialsToWebProxy: lastResponseDemandedProxyAuth);
-
-                        // Resend the request
-                        response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
-                    }
-                    else
-                    {
-                        response = tmpResponse;
-                    }
+                    response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);
 
                     traceInfo?.TraceRequestSendTime();
 

src/Sdk/Common/Common/VssHttpRequestSettings.cs

@@ -110,16 +110,6 @@ namespace GitHub.Services.Common
             set;
         }
 
-        /// <summary>
-        /// Gets or sets a value indicating whether or not HttpClientHandler should follow redirect on outgoing broker requests
-        /// This is special since this also sends token in the request, where as default AllowAutoRedirect does not
-        /// </summary>
-        public Boolean AllowAutoRedirectForBroker
-        {
-            get;
-            set;
-        }
-
         /// <summary>
         /// Gets or sets a value indicating whether or not compression should be used on outgoing requests.
         /// The default value is true.

src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs

@@ -43,6 +43,7 @@ namespace GitHub.DistributedTask.Pipelines
             TemplateToken jobOutputs,
             IList<TemplateToken> defaults,
             ActionsEnvironmentReference actionsEnvironment,
+            TemplateToken snapshot,
             String messageType = JobRequestMessageTypes.PipelineAgentJobRequest)
         {
             this.MessageType = messageType;
@@ -57,6 +58,7 @@ namespace GitHub.DistributedTask.Pipelines
             this.Workspace = workspaceOptions;
             this.JobOutputs = jobOutputs;
             this.ActionsEnvironment = actionsEnvironment;
+            this.Snapshot = snapshot;
             m_variables = new Dictionary<String, VariableValue>(variables, StringComparer.OrdinalIgnoreCase);
             m_maskHints = new List<MaskHint>(maskHints);
             m_steps = new List<JobStep>(steps);
@@ -237,6 +239,13 @@ namespace GitHub.DistributedTask.Pipelines
             set;
         }
 
+        [DataMember(EmitDefaultValue = false)]
+        public TemplateToken Snapshot
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets the collection of variables associated with the current context.
         /// </summary>

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs

@@ -29,6 +29,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
         public const String Id = "id";
         public const String If = "if";
         public const String Image = "image";
+        public const String ImageName = "image-name";
         public const String Include = "include";
         public const String Inputs = "inputs";
         public const String Job = "job";
@@ -60,6 +61,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
         public const String Services = "services";
         public const String Shell = "shell";
         public const String Skipped = "skipped";
+        public const String Snapshot = "snapshot";
         public const String StepEnv = "step-env";
         public const String StepIfResult = "step-if-result";
         public const String StepWith = "step-with";

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs

@@ -346,6 +346,39 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
             return result;
         }
 
+        internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, TemplateToken token)
+        {
+            string imageName = null;
+            if (token is StringToken snapshotStringLiteral)
+            {
+                imageName = snapshotStringLiteral.Value;
+            }
+            else
+            {
+                var snapshotMapping = token.AssertMapping($"{PipelineTemplateConstants.Snapshot}");
+                foreach (var snapshotPropertyPair in snapshotMapping)
+                {
+                    var propertyName = snapshotPropertyPair.Key.AssertString($"{PipelineTemplateConstants.Snapshot} key");
+                    switch (propertyName.Value)
+                    {
+                        case PipelineTemplateConstants.ImageName:
+                            imageName = snapshotPropertyPair.Value.AssertString($"{PipelineTemplateConstants.Snapshot} {propertyName}").Value;
+                            break;
+                        default:
+                            propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Snapshot} key");
+                            break;
+                    }
+                }
+            }
+
+            if (String.IsNullOrEmpty(imageName))
+            {
+                return null;
+            }
+
+            return new Snapshot(imageName);
+        }
+
         private static ActionStep ConvertToStep(
             TemplateContext context,
             TemplateToken stepsItem,

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs

@@ -370,6 +370,32 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
             return result;
         }
 
+        public Snapshot EvaluateJobSnapshotRequest(TemplateToken token,
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
+        {
+            var result = default(Snapshot);
+
+            if (token != null && token.Type != TokenType.Null)
+            {
+                var context = CreateContext(contextData, expressionFunctions);
+                try
+                {
+                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Snapshot, token, 0, null, omitHeader: true);
+                    context.Errors.Check();
+                    result = PipelineTemplateConverter.ConvertToJobSnapshotRequest(context, token);
+                }
+                catch (Exception ex) when (!(ex is TemplateValidationException))
+                {
+                    context.Errors.Add(ex);
+                }
+
+                context.Errors.Check();
+            }
+
+            return result;
+        }
+
         private TemplateContext CreateContext(
             DictionaryContextData contextData,
             IList<IFunctionInfo> expressionFunctions,

src/Sdk/DTPipelines/Pipelines/Snapshot.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Runtime.Serialization;
+
+namespace GitHub.DistributedTask.Pipelines
+{
+    [DataContract]
+    public class Snapshot
+    {
+        public Snapshot(string imageName)
+        {
+            ImageName = imageName;
+        }
+
+        [DataMember(EmitDefaultValue = false)]
+        public String ImageName { get; set; }
+    }
+}

src/Sdk/DTPipelines/workflow-v1.0.json

@@ -71,7 +71,8 @@
           "env": "job-env",
           "outputs": "job-outputs",
           "defaults": "job-defaults",
-          "steps": "steps"
+          "steps": "steps",
+          "snapshot": "snapshot"
         }
       }
     },
@@ -155,6 +156,24 @@
       }
     },
 
+    "snapshot": {
+      "one-of": [
+        "non-empty-string",
+        "snapshot-mapping"
+      ]
+    },
+
+    "snapshot-mapping": {
+      "mapping": {
+        "properties": {
+          "image-name": {
+            "type": "non-empty-string",
+            "required": true
+          }
+        }
+      }
+    },
+
     "runs-on": {
       "context": [
         "github",

src/Sdk/DTWebApi/WebApi/Exceptions.cs

@@ -2498,6 +2498,25 @@ namespace GitHub.DistributedTask.WebApi
         }
     }
 
+    [Serializable]
+    public class NonRetryableActionDownloadInfoException : DistributedTaskException
+    {
+        public NonRetryableActionDownloadInfoException(String message)
+            : base(message)
+        {
+        }
+
+        public NonRetryableActionDownloadInfoException(String message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+
+        protected NonRetryableActionDownloadInfoException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+        }
+    }
+
     [Serializable]
     public sealed class FailedToResolveActionDownloadInfoException : DistributedTaskException
     {

src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs

@@ -7,5 +7,8 @@ namespace GitHub.Actions.RunService.WebApi
     {
         [DataMember(Name = "jobMessageId", EmitDefaultValue = false)]
         public string JobMessageId { get; set; }
+
+        [DataMember(Name = "runnerOS", EmitDefaultValue = false)]
+        public string RunnerOS { get; set; }
     }
 }

src/Sdk/RSWebApi/RunServiceHttpClient.cs

@@ -59,12 +59,14 @@ namespace GitHub.Actions.RunService.WebApi
         public async Task<AgentJobRequestMessage> GetJobMessageAsync(
             Uri requestUri,
             string messageId,
+            string runnerOS,
             CancellationToken cancellationToken = default)
         {
             HttpMethod httpMethod = new HttpMethod("POST");
             var payload = new AcquireJobRequest
             {
                 JobMessageId = messageId,
+                RunnerOS = runnerOS
             };
 
             requestUri = new Uri(requestUri, "acquirejob");

src/Sdk/WebApi/WebApi/BrokerHttpClient.cs

@@ -110,9 +110,14 @@ namespace GitHub.Actions.RunService.WebApi
                 return result.Value;
             }
 
+            // the only time we throw a `Forbidden` exception from Listener /messages is when the runner is
+            // disable_update and is too old to poll
             if (result.StatusCode == HttpStatusCode.Forbidden)
             {
-                throw new AccessDeniedException(result.Error);
+                throw new AccessDeniedException($"{result.Error} Runner version v{runnerVersion} is deprecated and cannot receive messages.")
+                {
+                    ErrorCode = 1
+                };
             }
 
             throw new Exception($"Failed to get job message: {result.Error}");

src/Test/L0/Listener/BrokerMessageListenerL0.cs

@@ -56,11 +56,11 @@ namespace GitHub.Runner.Common.Tests.Listener
                 BrokerMessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
                 trace.Info("result: {0}", result);
 
                 // Assert.
-                Assert.True(result);
+                Assert.Equal(CreateSessionResult.Success, result);
                 _brokerServer
                    .Verify(x => x.CreateSessionAsync(
                        It.Is<TaskAgentSession>(y => y != null),

src/Test/L0/Listener/JobDispatcherL0.cs

@@ -41,7 +41,7 @@ namespace GitHub.Runner.Common.Tests.Listener
             TaskOrchestrationPlanReference plan = new();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+            var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
             result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
             return result;
         }
@@ -806,7 +806,8 @@ namespace GitHub.Runner.Common.Tests.Listener
                 },
                 null,
                 new List<TemplateToken>(),
-                new ActionsEnvironmentReference("env")
+                new ActionsEnvironmentReference("env"),
+                null
             );
             return message;
         }

src/Test/L0/Listener/MessageListenerL0.cs

@@ -75,11 +75,11 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
                 trace.Info("result: {0}", result);
 
                 // Assert.
-                Assert.True(result);
+                Assert.Equal(CreateSessionResult.Success, result);
                 _runnerServer
                     .Verify(x => x.CreateAgentSessionAsync(
                         _settings.PoolId,
@@ -135,11 +135,11 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
                 trace.Info("result: {0}", result);
 
                 // Assert.
-                Assert.True(result);
+                Assert.Equal(CreateSessionResult.Success, result);
 
                 _runnerServer
                     .Verify(x => x.CreateAgentSessionAsync(
@@ -185,8 +185,8 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
-                Assert.True(result);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                Assert.Equal(CreateSessionResult.Success, result);
 
                 _runnerServer
                     .Setup(x => x.DeleteAgentSessionAsync(
@@ -245,10 +245,10 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
                 trace.Info("result: {0}", result);
 
-                Assert.True(result);
+                Assert.Equal(CreateSessionResult.Success, result);
 
                 _runnerServer
                     .Verify(x => x.CreateAgentSessionAsync(
@@ -272,7 +272,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                 //Assert
                 _runnerServer
                     .Verify(x => x.DeleteAgentSessionAsync(
-                        _settings.PoolId, expectedSession.SessionId, It.IsAny<CancellationToken>()), Times.Never());
+                        _settings.PoolId, expectedBrokerSession.SessionId, It.IsAny<CancellationToken>()), Times.Once());
                 _brokerServer
                     .Verify(x => x.DeleteSessionAsync(It.IsAny<CancellationToken>()), Times.Once());
             }
@@ -309,8 +309,8 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
-                Assert.True(result);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                Assert.Equal(CreateSessionResult.Success, result);
 
                 var arMessages = new TaskAgentMessage[]
                 {
@@ -390,8 +390,8 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
-                Assert.True(result);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                Assert.Equal(CreateSessionResult.Success, result);
 
                 var brokerMigrationMesage = new BrokerMigrationMessage(new Uri("https://actions.broker.com"));
 
@@ -497,11 +497,11 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
                 trace.Info("result: {0}", result);
 
                 // Assert.
-                Assert.True(result);
+                Assert.Equal(CreateSessionResult.Success, result);
                 _runnerServer
                     .Verify(x => x.CreateAgentSessionAsync(
                         _settings.PoolId,
@@ -541,8 +541,8 @@ namespace GitHub.Runner.Common.Tests.Listener
                 MessageListener listener = new();
                 listener.Initialize(tc);
 
-                bool result = await listener.CreateSessionAsync(tokenSource.Token);
-                Assert.True(result);
+                CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token);
+                Assert.Equal(CreateSessionResult.Success, result);
 
                 _runnerServer
                     .Setup(x => x.GetAgentMessageAsync(

src/Test/L0/Listener/RunnerL0.cs

@@ -42,7 +42,7 @@ namespace GitHub.Runner.Common.Tests.Listener
             TaskOrchestrationPlanReference plan = new();
             TimelineReference timeline = null;
             Guid jobId = Guid.NewGuid();
-            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+            return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
         }
 
         private JobCancelMessage CreateJobCancelMessage()
@@ -88,7 +88,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                 _configurationManager.Setup(x => x.IsConfigured())
                     .Returns(true);
                 _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()))
-                    .Returns(Task.FromResult<bool>(true));
+                    .Returns(Task.FromResult<CreateSessionResult>(CreateSessionResult.Success));
                 _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny<CancellationToken>()))
                     .Returns(async () =>
                         {
@@ -184,7 +184,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                 _configStore.Setup(x => x.IsServiceConfigured()).Returns(configureAsService);
 
                 _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()))
-                    .Returns(Task.FromResult(false));
+                    .Returns(Task.FromResult<CreateSessionResult>(CreateSessionResult.Failure));
 
                 var runner = new Runner.Listener.Runner();
                 runner.Initialize(hc);
@@ -217,7 +217,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                     .Returns(false);
 
                 _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()))
-                    .Returns(Task.FromResult(false));
+                    .Returns(Task.FromResult<CreateSessionResult>(CreateSessionResult.Failure));
 
                 var runner = new Runner.Listener.Runner();
                 runner.Initialize(hc);
@@ -263,7 +263,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                 _configurationManager.Setup(x => x.IsConfigured())
                     .Returns(true);
                 _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()))
-                    .Returns(Task.FromResult<bool>(true));
+                    .Returns(Task.FromResult<CreateSessionResult>(CreateSessionResult.Success));
                 _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny<CancellationToken>()))
                     .Returns(async () =>
                         {
@@ -363,7 +363,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                 _configurationManager.Setup(x => x.IsConfigured())
                     .Returns(true);
                 _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()))
-                    .Returns(Task.FromResult<bool>(true));
+                    .Returns(Task.FromResult<CreateSessionResult>(CreateSessionResult.Success));
                 _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny<CancellationToken>()))
                     .Returns(async () =>
                         {
@@ -458,7 +458,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                 _configurationManager.Setup(x => x.IsConfigured())
                     .Returns(true);
                 _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny<CancellationToken>()))
-                    .Returns(Task.FromResult<bool>(true));
+                    .Returns(Task.FromResult<CreateSessionResult>(CreateSessionResult.Success));
                 _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny<CancellationToken>()))
                     .Returns(async () =>
                         {

src/Test/L0/Util/IOUtilL0.cs

@@ -960,6 +960,33 @@ namespace GitHub.Runner.Common.Tests.Util
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void ReplaceInvalidFileNameChars()
+        {
+            Assert.Equal(string.Empty, IOUtil.ReplaceInvalidFileNameChars(null));
+            Assert.Equal(string.Empty, IOUtil.ReplaceInvalidFileNameChars(string.Empty));
+            Assert.Equal("hello.txt", IOUtil.ReplaceInvalidFileNameChars("hello.txt"));
+#if OS_WINDOWS
+            // Refer https://github.com/dotnet/runtime/blob/ce84f1d8a3f12711bad678a33efbc37b461f684f/src/libraries/System.Private.CoreLib/src/System/IO/Path.Windows.cs#L15
+            Assert.Equal(
+                "1_ 2_ 3_ 4_ 5_ 6_ 7_ 8_ 9_ 10_ 11_ 12_ 13_ 14_ 15_ 16_ 17_ 18_ 19_ 20_ 21_ 22_ 23_ 24_ 25_ 26_ 27_ 28_ 29_ 30_ 31_ 32_ 33_ 34_ 35_ 36_ 37_ 38_ 39_ 40_ 41_",
+                IOUtil.ReplaceInvalidFileNameChars($"1\" 2< 3> 4| 5\0 6{(char)1} 7{(char)2} 8{(char)3} 9{(char)4} 10{(char)5} 11{(char)6} 12{(char)7} 13{(char)8} 14{(char)9} 15{(char)10} 16{(char)11} 17{(char)12} 18{(char)13} 19{(char)14} 20{(char)15} 21{(char)16} 22{(char)17} 23{(char)18} 24{(char)19} 25{(char)20} 26{(char)21} 27{(char)22} 28{(char)23} 29{(char)24} 30{(char)25} 31{(char)26} 32{(char)27} 33{(char)28} 34{(char)29} 35{(char)30} 36{(char)31} 37: 38* 39? 40\\ 41/"));
+#else
+            // Refer https://github.com/dotnet/runtime/blob/ce84f1d8a3f12711bad678a33efbc37b461f684f/src/libraries/System.Private.CoreLib/src/System/IO/Path.Unix.cs#L12
+            Assert.Equal("1_ 2_", IOUtil.ReplaceInvalidFileNameChars("1\0 2/"));
+#endif
+            Assert.Equal("_leading", IOUtil.ReplaceInvalidFileNameChars("/leading"));
+            Assert.Equal("__consecutive leading", IOUtil.ReplaceInvalidFileNameChars("//consecutive leading"));
+            Assert.Equal("trailing_", IOUtil.ReplaceInvalidFileNameChars("trailing/"));
+            Assert.Equal("consecutive trailing__", IOUtil.ReplaceInvalidFileNameChars("consecutive trailing//"));
+            Assert.Equal("middle_middle", IOUtil.ReplaceInvalidFileNameChars("middle/middle"));
+            Assert.Equal("consecutive middle__consecutive middle", IOUtil.ReplaceInvalidFileNameChars("consecutive middle//consecutive middle"));
+            Assert.Equal("_leading_middle_trailing_", IOUtil.ReplaceInvalidFileNameChars("/leading/middle/trailing/"));
+            Assert.Equal("__consecutive leading__consecutive middle__consecutive trailing__", IOUtil.ReplaceInvalidFileNameChars("//consecutive leading//consecutive middle//consecutive trailing//"));
+        }
+
         private static async Task CreateDirectoryReparsePoint(IHostContext context, string link, string target)
         {
 #if OS_WINDOWS

src/Test/L0/Util/WhichUtilL0.cs

@@ -212,210 +212,5 @@ namespace GitHub.Runner.Common.Tests.Util
             File.Delete(brokenSymlink);
             Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue);
         }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void UseWhich2FindGit()
-        {
-            using (TestHostContext hc = new(this))
-            {
-                //Arrange
-                Tracing trace = hc.GetTrace();
-
-                // Act.
-                string gitPath = WhichUtil.Which2("git", trace: trace);
-
-                trace.Info($"Which(\"git\") returns: {gitPath ?? string.Empty}");
-
-                // Assert.
-                Assert.True(!string.IsNullOrEmpty(gitPath) && File.Exists(gitPath), $"Unable to find Git through: {nameof(WhichUtil.Which)}");
-            }
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void Which2ReturnsNullWhenNotFound()
-        {
-            using (TestHostContext hc = new(this))
-            {
-                //Arrange
-                Tracing trace = hc.GetTrace();
-
-                // Act.
-                string nosuch = WhichUtil.Which2("no-such-file-cf7e351f", trace: trace);
-
-                trace.Info($"result: {nosuch ?? string.Empty}");
-
-                // Assert.
-                Assert.True(string.IsNullOrEmpty(nosuch), "Path should not be resolved");
-            }
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void Which2ThrowsWhenRequireAndNotFound()
-        {
-            using (TestHostContext hc = new(this))
-            {
-                //Arrange
-                Tracing trace = hc.GetTrace();
-
-                // Act.
-                try
-                {
-                    WhichUtil.Which2("no-such-file-cf7e351f", require: true, trace: trace);
-                    throw new Exception("which should have thrown");
-                }
-                catch (FileNotFoundException ex)
-                {
-                    Assert.Equal("no-such-file-cf7e351f", ex.FileName);
-                }
-            }
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void Which2HandleFullyQualifiedPath()
-        {
-            using (TestHostContext hc = new(this))
-            {
-                //Arrange
-                Tracing trace = hc.GetTrace();
-
-                // Act.
-                var gitPath = WhichUtil.Which2("git", require: true, trace: trace);
-                var gitPath2 = WhichUtil.Which2(gitPath, require: true, trace: trace);
-
-                // Assert.
-                Assert.Equal(gitPath, gitPath2);
-            }
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void Which2HandlesSymlinkToTargetFullPath()
-        {
-            // Arrange
-            using TestHostContext hc = new TestHostContext(this);
-            Tracing trace = hc.GetTrace();
-            string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
-#if OS_WINDOWS
-            string newValue = oldValue + @$";{Path.GetTempPath()}";
-            string symlinkName = $"symlink-{Guid.NewGuid()}";
-            string symlink = Path.GetTempPath() + $"{symlinkName}.exe";
-            string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}.exe";
-#else
-            string newValue = oldValue + @$":{Path.GetTempPath()}";
-            string symlinkName = $"symlink-{Guid.NewGuid()}";
-            string symlink = Path.GetTempPath() + $"{symlinkName}";
-            string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}";
-#endif
-
-            Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue);
-
-
-            using (File.Create(target))
-            {
-                File.CreateSymbolicLink(symlink, target);
-
-                // Act.
-                var result = WhichUtil.Which2(symlinkName, require: true, trace: trace);
-
-                // Assert
-                Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find symlink through: {nameof(WhichUtil.Which)}");
-
-            }
-
-
-            // Cleanup
-            File.Delete(symlink);
-            File.Delete(target);
-            Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue);
-
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void Which2HandlesSymlinkToTargetRelativePath()
-        {
-            // Arrange
-            using TestHostContext hc = new TestHostContext(this);
-            Tracing trace = hc.GetTrace();
-            string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
-#if OS_WINDOWS
-            string newValue = oldValue + @$";{Path.GetTempPath()}";
-            string symlinkName = $"symlink-{Guid.NewGuid()}";
-            string symlink = Path.GetTempPath() + $"{symlinkName}.exe";
-            string targetName = $"target-{Guid.NewGuid()}.exe";
-            string target = Path.GetTempPath() + targetName;
-#else
-            string newValue = oldValue + @$":{Path.GetTempPath()}";
-            string symlinkName = $"symlink-{Guid.NewGuid()}";
-            string symlink = Path.GetTempPath() + $"{symlinkName}";
-            string targetName = $"target-{Guid.NewGuid()}";
-            string target = Path.GetTempPath() + targetName;
-#endif
-            Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue);
-
-
-            using (File.Create(target))
-            {
-                File.CreateSymbolicLink(symlink, targetName);
-
-                // Act.
-                var result = WhichUtil.Which2(symlinkName, require: true, trace: trace);
-
-                // Assert
-                Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find {symlinkName} through: {nameof(WhichUtil.Which)}");
-            }
-
-            // Cleanup
-            File.Delete(symlink);
-            File.Delete(target);
-            Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue);
-
-        }
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public void Which2ThrowsWhenSymlinkBroken()
-        {
-            // Arrange
-            using TestHostContext hc = new TestHostContext(this);
-            Tracing trace = hc.GetTrace();
-            string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
-
-#if OS_WINDOWS
-            string newValue = oldValue + @$";{Path.GetTempPath()}";
-            string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}";
-            string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}.exe";
-#else
-            string newValue = oldValue + @$":{Path.GetTempPath()}";
-            string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}";
-            string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}";
-#endif
-
-
-            string target = "no-such-file-cf7e351f";
-            Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue);
-
-            File.CreateSymbolicLink(brokenSymlink, target);
-
-            // Act.
-            var exception = Assert.Throws<FileNotFoundException>(() => WhichUtil.Which2(brokenSymlinkName, require: true, trace: trace));
-
-            // Assert
-            Assert.Equal(brokenSymlinkName, exception.FileName);
-
-            // Cleanup
-            File.Delete(brokenSymlink);
-            Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue);
-        }
     }
 }

src/Test/L0/Worker/ActionCommandManagerL0.cs

@@ -232,7 +232,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,

src/Test/L0/Worker/ActionManagerL0.cs

@@ -382,8 +382,6 @@ runs:
                     }
                 };
 
-                _ec.Object.Global.Variables.Set("DistributedTask.UseActionArchiveCache", bool.TrueString);
-
                 //Act
                 await _actionManager.PrepareActionsAsync(_ec.Object, actions);
 
@@ -2375,10 +2373,6 @@ runs:
             _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token);
             _ec.Setup(x => x.Root).Returns(new GitHub.Runner.Worker.ExecutionContext());
             var variables = new Dictionary<string, VariableValue>();
-            if (enableComposite)
-            {
-                variables["DistributedTask.EnableCompositeActions"] = "true";
-            }
             _ec.Object.Global.Variables = new Variables(_hc, variables);
             _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData());
             _ec.Setup(x => x.ExpressionFunctions).Returns(new List<IFunctionInfo>());

src/Test/L0/Worker/CreateStepSummaryCommandL0.cs

@@ -193,7 +193,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             TimelineReference timeline = new();
             Guid jobId = Guid.NewGuid();
             string jobName = "Summary Job";
-            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
             jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
             {
                 Alias = Pipelines.PipelineConstants.SelfAlias,

src/Test/L0/Worker/ExecutionContextL0.cs

@@ -29,7 +29,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -106,7 +106,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -162,7 +162,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -216,7 +216,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -271,7 +271,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -322,7 +322,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -373,7 +373,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -471,7 +471,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -555,7 +555,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -610,7 +610,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -653,7 +653,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -717,7 +717,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -781,7 +781,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -969,7 +969,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -1014,7 +1014,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new TimelineReference();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,
@@ -1057,7 +1057,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 TimelineReference timeline = new TimelineReference();
                 Guid jobId = Guid.NewGuid();
                 string jobName = "some job name";
-                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
                 jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                 {
                     Alias = Pipelines.PipelineConstants.SelfAlias,

src/Test/L0/Worker/JobExtensionL0.cs

@@ -4,6 +4,8 @@ using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
+using GitHub.DistributedTask.Pipelines.ObjectTemplating;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Worker;
 using Moq;
@@ -25,6 +27,9 @@ namespace GitHub.Runner.Common.Tests.Worker
         private Mock<IContainerOperationProvider> _containerProvider;
         private Mock<IDiagnosticLogManager> _diagnosticLogManager;
         private Mock<IJobHookProvider> _jobHookProvider;
+        private Mock<ISnapshotOperationProvider> _snapshotOperationProvider;
+
+        private Pipelines.Snapshot _requestedSnapshot;
 
         private CancellationTokenSource _tokenSource;
         private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
@@ -41,7 +46,16 @@ namespace GitHub.Runner.Common.Tests.Worker
             _directoryManager.Setup(x => x.PrepareDirectory(It.IsAny<IExecutionContext>(), It.IsAny<Pipelines.WorkspaceOptions>()))
                              .Returns(new TrackingConfig() { PipelineDirectory = "runner", WorkspaceDirectory = "runner/runner" });
             _jobHookProvider = new Mock<IJobHookProvider>();
+            _snapshotOperationProvider = new Mock<ISnapshotOperationProvider>();
 
+            _requestedSnapshot = null;
+            _snapshotOperationProvider
+                .Setup(p => p.CreateSnapshotRequestAsync(It.IsAny<IExecutionContext>(), It.IsAny<Pipelines.Snapshot>()))
+                .Returns((IExecutionContext _, object data) =>
+                {
+                    _requestedSnapshot = data as Pipelines.Snapshot;
+                    return Task.CompletedTask;
+                });
             IActionRunner step1 = new ActionRunner();
             IActionRunner step2 = new ActionRunner();
             IActionRunner step3 = new ActionRunner();
@@ -100,7 +114,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             };
 
             Guid jobId = Guid.NewGuid();
-            _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null);
+            _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null, null);
             GitHubContext github = new();
             github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner");
             github["secret_source"] = new Pipelines.ContextData.StringContextData("Actions");
@@ -125,6 +139,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             hc.SetSingleton(_directoryManager.Object);
             hc.SetSingleton(_diagnosticLogManager.Object);
             hc.SetSingleton(_jobHookProvider.Object);
+            hc.SetSingleton(_snapshotOperationProvider.Object);
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // JobExecutionContext
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // job start hook
             hc.EnqueueInstance<IPagingLogger>(_logger.Object); // Initial Job
@@ -443,5 +458,80 @@ namespace GitHub.Runner.Common.Tests.Worker
                 Assert.Equal(0, _jobEc.PostJobSteps.Count);
             }
         }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public async Task EnsureNoSnapshotPostJobStep()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                var jobExtension = new JobExtension();
+                jobExtension.Initialize(hc);
+
+                _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny<IExecutionContext>(), It.IsAny<IEnumerable<Pipelines.JobStep>>(), It.IsAny<Guid>()))
+                    .Returns(Task.FromResult(new PrepareResult(new List<JobExtensionRunner>(), new Dictionary<Guid, IActionRunner>())));
+
+                _message.Snapshot = null;
+                await jobExtension.InitializeJob(_jobEc, _message);
+
+                var postJobSteps = _jobEc.PostJobSteps;
+                Assert.Equal(0, postJobSteps.Count);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public Task EnsureSnapshotPostJobStepForStringToken()
+        {
+            var snapshot = new Pipelines.Snapshot("TestImageNameFromStringToken");
+            var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName);
+            return EnsureSnapshotPostJobStepForToken(imageNameValueStringToken, snapshot);
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public Task EnsureSnapshotPostJobStepForMappingToken()
+        {
+            var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken");
+            var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName);
+            var mappingToken = new MappingToken(null, null, null)
+            {
+                { new StringToken(null,null,null, PipelineTemplateConstants.ImageName), imageNameValueStringToken }
+            };
+
+            return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot);
+        }
+
+        private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot)
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                var jobExtension = new JobExtension();
+                jobExtension.Initialize(hc);
+
+                _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny<IExecutionContext>(), It.IsAny<IEnumerable<Pipelines.JobStep>>(), It.IsAny<Guid>()))
+                    .Returns(Task.FromResult(new PrepareResult(new List<JobExtensionRunner>(), new Dictionary<Guid, IActionRunner>())));
+
+                _message.Snapshot = snapshotToken;
+
+                await jobExtension.InitializeJob(_jobEc, _message);
+
+                var postJobSteps = _jobEc.PostJobSteps;
+
+                Assert.Equal(1, postJobSteps.Count);
+                var snapshotStep = postJobSteps.First();
+                Assert.Equal("Create custom image", snapshotStep.DisplayName);
+                Assert.Equal($"{PipelineTemplateConstants.Success}()", snapshotStep.Condition);
+
+                // Run the mock snapshot step, so we can verify it was executed with the expected snapshot object.
+                await snapshotStep.RunAsync();
+
+                Assert.NotNull(_requestedSnapshot);
+                Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName);
+            }
+        }
     }
 }

src/Test/L0/Worker/JobRunnerL0.cs

@@ -101,6 +101,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 testName,
                 testName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null,
                 new ActionsEnvironmentReference("staging"),
+                null,
                 messageType: messageType);
             message.Variables[Constants.Variables.System.Culture] = "en-US";
             message.Resources.Endpoints.Add(new ServiceEndpoint()

src/Test/L0/Worker/SnapshotOperationProviderL0.cs

@@ -0,0 +1,78 @@
+#nullable enable
+using System;
+using System.IO;
+using System.Runtime.CompilerServices;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Runner.Sdk;
+using GitHub.Runner.Worker;
+using Moq;
+using Xunit;
+
+namespace GitHub.Runner.Common.Tests.Worker;
+
+public class SnapshotOperationProviderL0
+{
+    private Mock<IExecutionContext>? _ec;
+    private SnapshotOperationProvider? _snapshotOperationProvider;
+    private string? _snapshotRequestFilePath;
+    private string? _snapshotRequestDirectoryPath;
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    [Trait("Level", "L0")]
+    [Trait("Category", "Worker")]
+    public async void CreateSnapshotRequestAsync(bool shouldSnapshotDirectoryAlreadyExist)
+    {
+        using (TestHostContext testHostContext = CreateTestHostContext())
+        {
+            //Arrange
+            Setup(testHostContext, shouldSnapshotDirectoryAlreadyExist);
+            var expectedSnapshot = new Snapshot(Guid.NewGuid().ToString());
+
+            //Act
+            await _snapshotOperationProvider!.CreateSnapshotRequestAsync(_ec!.Object, expectedSnapshot);
+
+            //Assert
+            var actualSnapshot = IOUtil.LoadObject<Snapshot>(_snapshotRequestFilePath);
+            Assert.NotNull(actualSnapshot);
+            Assert.Equal(expectedSnapshot.ImageName, actualSnapshot!.ImageName);
+            _ec.Verify(ec => ec.Write(null, $"Request written to: {_snapshotRequestFilePath}"), Times.Once);
+            _ec.Verify(ec => ec.Write(null, "This request will be processed after the job completes. You will not receive any feedback on the snapshot process within the workflow logs of this job."), Times.Once);
+            _ec.Verify(ec => ec.Write(null, "If the snapshot process is successful, you should see a new image with the requested name in the list of available custom images when creating a new GitHub-hosted Runner."), Times.Once);
+            _ec.VerifyNoOtherCalls();
+        }
+    }
+
+    private void Setup(IHostContext hostContext, bool shouldSnapshotDirectoryAlreadyExist)
+    {
+        _ec = new Mock<IExecutionContext>();
+        _snapshotOperationProvider = new SnapshotOperationProvider();
+        _snapshotOperationProvider.Initialize(hostContext);
+        _snapshotRequestFilePath = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Root), ".snapshot", "request.json");
+        _snapshotRequestDirectoryPath = Path.GetDirectoryName(_snapshotRequestFilePath);
+
+        if (_snapshotRequestDirectoryPath != null)
+        {
+            // Clean up any existing the snapshot directory and its contents before starting the test. 
+            if (Directory.Exists(_snapshotRequestDirectoryPath))
+            {
+                Directory.Delete(_snapshotRequestDirectoryPath, true);
+            }
+
+            if (shouldSnapshotDirectoryAlreadyExist)
+            {
+                // Create a fresh snapshot directory if it's required for the test. 
+                Directory.CreateDirectory(_snapshotRequestDirectoryPath);
+            }
+        }
+    }
+
+    private TestHostContext CreateTestHostContext([CallerMemberName] string testName = "")
+    {
+        var testHostContext = new TestHostContext(this, testName);
+        _ec = new Mock<IExecutionContext>();
+        _ec.Object.Initialize(testHostContext);
+        return testHostContext;
+    }
+}

src/Test/L0/Worker/WorkerL0.cs

@@ -67,7 +67,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                     new Pipelines.ContextData.DictionaryContextData()
                 },
             };
-            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List<MaskHint>(), resources, context, null, actions, null, null, null, null);
+            var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List<MaskHint>(), resources, context, null, actions, null, null, null, null, null);
             return jobRequest;
         }
 

src/dev.sh

@@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout"
 DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x"
 PACKAGE_DIR="$SCRIPT_DIR/../_package"
 DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk"
-DOTNETSDK_VERSION="6.0.418"
+DOTNETSDK_VERSION="6.0.421"
 DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION"
 RUNNER_VERSION=$(cat runnerversion)
 

src/global.json

@@ -1,5 +1,5 @@
 {
   "sdk": {
-    "version": "6.0.418"
+    "version": "6.0.421"
   }
 }

src/runnerversion

@@ -1 +1 @@
-2.312.0
+2.316.0