Create 2.310.0 runner release.

* Fix typo

* Remove double space

docs/checks/nodejs.md

@@ -4,9 +4,9 @@
 
 Make sure the built-in node.js has access to GitHub.com or GitHub Enterprise Server.
 
-The runner carries its own copy of node.js executable under `<runner_root>/externals/node16/`.
+The runner carries its own copy of node.js executable under `<runner_root>/externals/node20/`.
 
-All javascript base Actions will get executed by the built-in `node` at `<runner_root>/externals/node16/`.
+All javascript base Actions will get executed by the built-in `node` at `<runner_root>/externals/node20/`.
 
 > Not the `node` from `$PATH`
 

images/Dockerfile

@@ -5,7 +5,7 @@ ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
 ARG RUNNER_CONTAINER_HOOKS_VERSION=0.4.0
-ARG DOCKER_VERSION=23.0.6
+ARG DOCKER_VERSION=24.0.6
 
 RUN apt update -y && apt install curl unzip -y
 

releaseNote.md

@@ -1,37 +1,44 @@
 ## What's Changed
-* Bump @types/node from 12.12.14 to 20.4.10 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2759
-* Trace x-github-request-id when download action tarball. by @TingluoHuang in https://github.com/actions/runner/pull/2755
-* Fix typo by @kyanny in https://github.com/actions/runner/pull/2741
-* Bump prettier from 3.0.1 to 3.0.2 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2772
-* Bump @types/node from 20.4.10 to 20.5.0 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2773
-* Revert "Fixed a bug where a misplaced `=` character could bypass here… by @cory-miller in https://github.com/actions/runner/pull/2774
-* Filter NODE_OPTIONS from env for file output by @cory-miller in https://github.com/actions/runner/pull/2775
-* Bump @types/node from 20.5.0 to 20.5.1 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2781
-* Update Docker Version in Images by @ajschmidt8 in https://github.com/actions/runner/pull/2694
-* Bump @types/node from 20.5.1 to 20.5.4 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2789
-* Bump @typescript-eslint/parser from 6.4.0 to 6.4.1 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2785
-* Bump Microsoft.AspNet.WebApi.Client from 5.2.4 to 5.2.9 in /src by @dependabot in https://github.com/actions/runner/pull/2751
-* Bump System.Buffers from 4.3.0 to 4.5.1 in /src by @dependabot in https://github.com/actions/runner/pull/2749
-* Bump dotnet/runtime-deps from 6.0-jammy to 7.0-jammy in /images by @dependabot in https://github.com/actions/runner/pull/2745
-* Remove need to manually compile JS binary for hashFiles utility by @vanZeben in https://github.com/actions/runner/pull/2770
-* Revert "Bump dotnet/runtime-deps from 6.0-jammy to 7.0-jammy in /images" by @TingluoHuang in https://github.com/actions/runner/pull/2790
-* Query runner by name on server side. by @TingluoHuang in https://github.com/actions/runner/pull/2771
-* Bump typescript from 5.1.6 to 5.2.2 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2795
-* Bump @types/node from 20.5.4 to 20.5.6 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2796
-* Bump Newtonsoft.Json from 13.0.1 to 13.0.3 in /src by @dependabot in https://github.com/actions/runner/pull/2797
-* Support replacing runners in v2 flow by @luketomlinson in https://github.com/actions/runner/pull/2791
-* Delegating handler for Http redirects by @paveliak in https://github.com/actions/runner/pull/2814
-* Add references to the firewall requirements docs by @paveliak in https://github.com/actions/runner/pull/2815
-* Create automated workflow that will auto-generate dotnet sdk patches by @vanZeben in https://github.com/actions/runner/pull/2776
-* Fixes minor issues with using proper output varaibles by @vanZeben in https://github.com/actions/runner/pull/2818
-* Throw NonRetryableException on GetNextMessage from broker as needed. by @TingluoHuang in https://github.com/actions/runner/pull/2828
-* Mark action download failures as infra failures by @cory-miller in https://github.com/actions/runner/pull/2827
+* Prepare runner release 2.309.0 by @johnsudol in https://github.com/actions/runner/pull/2833
+* remove debug-only flag from stale bot action by @ruvceskistefan in https://github.com/actions/runner/pull/2834
+* Calculate docker instance label based on the hash of the config by @nikola-jokic in https://github.com/actions/runner/pull/2683
+* Correcting `zen` address by @Pantelis-Santorinios in https://github.com/actions/runner/pull/2855
+* Update dotnet sdk to latest version @6.0.414 by @github-actions in https://github.com/actions/runner/pull/2852
+* Bump @typescript-eslint/parser from 6.4.1 to 6.7.0 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2845
+* Bump @types/node from 20.5.6 to 20.6.2 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2854
+* Bump eslint-plugin-github from 4.9.2 to 4.10.0 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2808
+* Bump @typescript-eslint/parser from 6.7.0 to 6.7.2 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2858
+* Bump prettier from 3.0.2 to 3.0.3 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2860
+* Bump @vercel/ncc from 0.36.1 to 0.38.0 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2859
+* Bump @typescript-eslint/eslint-plugin from 6.4.1 to 6.7.2 in /src/Misc/expressionFunc/hashFiles by @dependabot in https://github.com/actions/runner/pull/2861
+* Remove unused code in AgentManager. by @TingluoHuang in https://github.com/actions/runner/pull/2866
+* GetAgents from all runner groups durning config. by @TingluoHuang in https://github.com/actions/runner/pull/2865
+* Change alpine from vst blobs to OSS gha alpine build by @vanZeben in https://github.com/actions/runner/pull/2871
+* Bump node 16 to v16.20.2 by @vanZeben in https://github.com/actions/runner/pull/2872
+* Bump directly dotnet vulnerable packages by @nikola-jokic in https://github.com/actions/runner/pull/2870
+* Fix ArgumentOutOfRangeException in PowerShellPostAmpersandEscape. by @TingluoHuang in https://github.com/actions/runner/pull/2875
+* bump container hook version in runner image by @nikola-jokic in https://github.com/actions/runner/pull/2881
+* Use `Directory.EnumerateFiles` instead of `Directory.GetFiles` in WhichUtil. by @TingluoHuang in https://github.com/actions/runner/pull/2882
+* Add warning about node16 deprecation by @takost in https://github.com/actions/runner/pull/2887
+* Throw TimeoutException instead of OperationCanceledException on the final retry in DownloadRepositoryAction by @TingluoHuang in https://github.com/actions/runner/pull/2895
+* Update message when runners are deleted by @thboop in https://github.com/actions/runner/pull/2896
+* Do not give up if Results is powering logs by @yacaovsnc in https://github.com/actions/runner/pull/2893
+* Allow use action archive cache to speed up workflow jobs. by @TingluoHuang in https://github.com/actions/runner/pull/2857
+* Upgrade docker engine to 24.0.6 in the runner container image by @Link- in https://github.com/actions/runner/pull/2886
+* Collect telemetry to measure upload speed for different backend. by @TingluoHuang in https://github.com/actions/runner/pull/2912
+* Use RawHttpMessageHandler and VssHttpRetryMessageHandler in ResultsHttpClient by @yacaovsnc in https://github.com/actions/runner/pull/2908
+* Retries to lock Services database on Windows by @sugymt in https://github.com/actions/runner/pull/2880
+* Update default version to node20 by @takost in https://github.com/actions/runner/pull/2844
+* Fixed Attempt typo by @corycalahan in https://github.com/actions/runner/pull/2849
+* Fix typo by @rajbos in https://github.com/actions/runner/pull/2670
 
 ## New Contributors
-* @kyanny made their first contribution in https://github.com/actions/runner/pull/2741
-* @ajschmidt8 made their first contribution in https://github.com/actions/runner/pull/2694
+* @Pantelis-Santorinios made their first contribution in https://github.com/actions/runner/pull/2855
+* @github-actions made their first contribution in https://github.com/actions/runner/pull/2852
+* @sugymt made their first contribution in https://github.com/actions/runner/pull/2880
+* @corycalahan made their first contribution in https://github.com/actions/runner/pull/2849
 
-**Full Changelog**: https://github.com/actions/runner/compare/v2.308.0...v2.309.0
+**Full Changelog**: https://github.com/actions/runner/compare/v2.309.0...v2.310.0
 
 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. 
 To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. 

releaseVersion

@@ -1 +1 @@
-<Update to ./src/runnerversion when creating release>
+2.310.0

src/Misc/layoutbin/runsvc.sh

@@ -10,7 +10,7 @@ if [ -f ".path" ]; then
     echo ".path=${PATH}"
 fi
 
-nodever=${GITHUB_ACTIONS_RUNNER_FORCED_NODE_VERSION:-node16}
+nodever=${GITHUB_ACTIONS_RUNNER_FORCED_NODE_VERSION:-node20}
 
 # insert anything to setup env when running as a service
 # run the host process which keep the listener alive

src/Misc/layoutbin/update.sh.template

@@ -135,6 +135,10 @@ if [[ "$currentplatform" == 'darwin'  && restartinteractiverunner -eq 0 ]]; then
     then
         # inspect the open file handles to find the node process
         # we can't actually inspect the process using ps because it uses relative paths and doesn't follow symlinks
+        nodever="node20"
+        path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
+        if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node16
+        then
             nodever="node16"
             path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
             if [[ $? -ne 0 || -z "$path" ]] # Fallback if RunnerService.js was started with node12
@@ -142,6 +146,7 @@ if [[ "$currentplatform" == 'darwin'  && restartinteractiverunner -eq 0 ]]; then
                 nodever="node12"
                 path=$(lsof -a -g "$procgroup" -F n | grep $nodever/bin/node | grep externals | tail -1 | cut -c2-)
             fi
+        fi
         if [[ $? -eq 0 && -n "$path" ]]
         then
             # trim the last 5 characters of the path '/node'

src/Misc/layoutroot/run-helper.cmd.template

@@ -2,7 +2,7 @@
 SET UPDATEFILE=update.finished
 "%~dp0\bin\Runner.Listener.exe" run %*
 
-rem using `if %ERRORLEVEL% EQU N` insterad of `if ERRORLEVEL N`
+rem using `if %ERRORLEVEL% EQU N` instead of `if ERRORLEVEL N`
 rem `if ERRORLEVEL N` means: error level is N or MORE
   
 if %ERRORLEVEL% EQU 0 (

src/Runner.Common/Constants.cs

@@ -257,10 +257,11 @@ namespace GitHub.Runner.Common
             {
                 public static readonly string ToolsDirectory = "agent.ToolsDirectory";
 
-                // Set this env var to "node12" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions.
+                // Set this env var to "node16" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions.
                 public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION";
                 public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION";
                 public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT";
+                public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE";
             }
 
             public static class System

src/Runner.Common/JobServerQueue.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -14,10 +15,11 @@ namespace GitHub.Runner.Common
     [ServiceLocator(Default = typeof(JobServerQueue))]
     public interface IJobServerQueue : IRunnerService, IThrottlingReporter
     {
+        IList<JobTelemetry> JobTelemetries { get; }
         TaskCompletionSource<int> JobRecordUpdated { get; }
         event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
         Task ShutdownAsync();
-        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultServiceOnly = false);
+        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false);
         void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
         void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
         void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines);
@@ -69,13 +71,18 @@ namespace GitHub.Runner.Common
         private Task[] _allDequeueTasks;
         private readonly TaskCompletionSource<int> _jobCompletionSource = new();
         private readonly TaskCompletionSource<int> _jobRecordUpdated = new();
+        private readonly List<JobTelemetry> _jobTelemetries = new();
         private bool _queueInProcess = false;
         private bool _resultsServiceOnly = false;
+        private Stopwatch _resultsUploadTimer = new();
+        private Stopwatch _actionsUploadTimer = new();
 
         public TaskCompletionSource<int> JobRecordUpdated => _jobRecordUpdated;
 
         public event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
 
+        public IList<JobTelemetry> JobTelemetries => _jobTelemetries;
+
         // Web console dequeue will start with process queue every 250ms for the first 60*4 times (~60 seconds).
         // Then the dequeue will happen every 500ms.
         // In this way, customer still can get instance live console output on job start,
@@ -87,6 +94,7 @@ namespace GitHub.Runner.Common
         private bool _firstConsoleOutputs = true;
 
         private bool _resultsClientInitiated = false;
+        private bool _enableTelemetry = false;
         private delegate Task ResultsFileUploadHandler(ResultsUploadFileInfo file);
 
         public override void Initialize(IHostContext hostContext)
@@ -96,14 +104,15 @@ namespace GitHub.Runner.Common
             _resultsServer = hostContext.GetService<IResultsServer>();
         }
 
-        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultServiceOnly = false)
+        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false)
         {
             Trace.Entering();
-            _resultsServiceOnly = resultServiceOnly;
+            _resultsServiceOnly = resultsServiceOnly;
+            _enableTelemetry = enableTelemetry;
 
             var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
 
-            if (!resultServiceOnly)
+            if (!resultsServiceOnly)
             {
                 _jobServer.InitializeWebsocketClient(serviceEndPoint);
             }
@@ -119,7 +128,7 @@ namespace GitHub.Runner.Common
             {
                 string liveConsoleFeedUrl = null;
                 Trace.Info("Initializing results client");
-                if (resultServiceOnly
+                if (resultsServiceOnly
                     && serviceEndPoint.Data.TryGetValue("FeedStreamUrl", out var feedStreamUrl)
                     && !string.IsNullOrEmpty(feedStreamUrl))
                 {
@@ -211,6 +220,12 @@ namespace GitHub.Runner.Common
             await _resultsServer.DisposeAsync();
 
             Trace.Info("All queue process tasks have been stopped, and all queues are drained.");
+            if (_enableTelemetry)
+            {
+                var uploadTimeComparison = $"Actions upload time: {_actionsUploadTimer.ElapsedMilliseconds} ms, Result upload time: {_resultsUploadTimer.ElapsedMilliseconds} ms";
+                Trace.Info(uploadTimeComparison);
+                _jobTelemetries.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = uploadTimeComparison });
+            }
         }
 
         public void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber)
@@ -456,6 +471,10 @@ namespace GitHub.Runner.Common
                     {
                         try
                         {
+                            if (_enableTelemetry)
+                            {
+                                _actionsUploadTimer.Start();
+                            }
                             await UploadFile(file);
                         }
                         catch (Exception ex)
@@ -471,6 +490,13 @@ namespace GitHub.Runner.Common
                             //    _fileUploadQueue.Enqueue(file);
                             //}
                         }
+                        finally
+                        {
+                            if (_enableTelemetry)
+                            {
+                                _actionsUploadTimer.Stop();
+                            }
+                        }
                     }
 
                     Trace.Info("Try to upload {0} log files or attachments, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount);
@@ -517,6 +543,10 @@ namespace GitHub.Runner.Common
                     {
                         try
                         {
+                            if (_enableTelemetry)
+                            {
+                                _resultsUploadTimer.Start();
+                            }
                             if (String.Equals(file.Type, ChecksAttachmentType.StepSummary, StringComparison.OrdinalIgnoreCase))
                             {
                                 await UploadSummaryFile(file);
@@ -541,12 +571,21 @@ namespace GitHub.Runner.Common
                             Trace.Error(ex);
                             errorCount++;
 
-                            // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results
+                            // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs
+                            if (!_resultsServiceOnly)
+                            {
                                 _resultsClientInitiated = false;
-
                                 SendResultsTelemetry(ex);
                             }
                         }
+                        finally
+                        {
+                            if (_enableTelemetry)
+                            {
+                                _resultsUploadTimer.Stop();
+                            }
+                        }
+                    }
 
                     Trace.Info("Tried to upload {0} file(s) to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount);
                 }
@@ -660,10 +699,12 @@ namespace GitHub.Runner.Common
                             {
                                 Trace.Info("Catch exception during update steps, skip update Results.");
                                 Trace.Error(e);
+                                if (!_resultsServiceOnly)
+                                {
                                     _resultsClientInitiated = false;
-
                                     SendResultsTelemetry(e);
                                 }
+                            }
 
                             if (_bufferedRetryRecords.Remove(update.TimelineId))
                             {

src/Runner.Common/ResultsServer.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Net.WebSockets;
 using System.Security;
@@ -52,8 +53,8 @@ namespace GitHub.Runner.Common
 
         public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token)
         {
-            var httpMessageHandler = HostContext.CreateHttpClientHandler();
-            this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true);
+            this._resultsClient = CreateHttpClient(uri, token);
+
             _token = token;
             if (!string.IsNullOrEmpty(liveConsoleFeedUrl))
             {
@@ -62,6 +63,26 @@ namespace GitHub.Runner.Common
             }
         }
 
+        public ResultsHttpClient CreateHttpClient(Uri uri, string token)
+        {
+            // Using default 100 timeout
+            RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null);
+
+            // Create retry handler
+            IEnumerable<DelegatingHandler> delegatingHandlers = new List<DelegatingHandler>();
+            if (settings.MaxRetryRequest > 0)
+            {
+                delegatingHandlers = new DelegatingHandler[] { new VssHttpRetryMessageHandler(settings.MaxRetryRequest) };
+            }
+
+            // Setup RawHttpMessageHandler without credentials
+            var httpMessageHandler = new RawHttpMessageHandler(new NoOpCredentials(null), settings);
+
+            var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers);
+
+            return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true);
+        }
+
         public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file,
             CancellationToken cancellationToken)
         {

src/Runner.Common/RunnerDotcomServer.cs

@@ -224,7 +224,7 @@ namespace GitHub.Runner.Common
                     }
                     catch (Exception ex) when (retry < maxRetryAttemptsCount && responseStatus != System.Net.HttpStatusCode.NotFound)
                     {
-                        Trace.Error($"{errorMessage} -- Atempt: {retry}");
+                        Trace.Error($"{errorMessage} -- Attempt: {retry}");
                         Trace.Error(ex);
                     }
                 }

src/Runner.Common/Util/NodeUtil.cs

@@ -5,7 +5,7 @@ namespace GitHub.Runner.Common.Util
 {
     public static class NodeUtil
     {
-        private const string _defaultNodeVersion = "node16";
+        private const string _defaultNodeVersion = "node20";
         public static readonly ReadOnlyCollection<string> BuiltInNodeVersions = new(new[] { "node16", "node20" });
         public static string GetInternalNodeVersion()
         {

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -744,7 +744,7 @@ namespace GitHub.Runner.Listener.Configuration
                     catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
                     {
                         retryCount++;
-                        Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}");
+                        Trace.Error($"Failed to get JIT runner token -- Attempt: {retryCount}");
                         Trace.Error(ex);
                     }
                 }
@@ -807,7 +807,7 @@ namespace GitHub.Runner.Listener.Configuration
                     catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
                     {
                         retryCount++;
-                        Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}");
+                        Trace.Error($"Failed to get tenant credentials -- Attempt: {retryCount}");
                         Trace.Error(ex);
                     }
                 }

src/Runner.Listener/Configuration/NativeWindowsServiceHelper.cs

@@ -514,9 +514,25 @@ namespace GitHub.Runner.Listener.Configuration
                 failureActions.Add(new FailureAction(RecoverAction.Restart, 60000));
 
                 // Lock the Service Database
-                svcLock = LockServiceDatabase(scmHndl);
-                if (svcLock.ToInt64() <= 0)
+                int svcLockRetries = 10;
+                int svcLockRetryTimeout = 5000;
+                while (true)
                 {
+                    svcLock = LockServiceDatabase(scmHndl);
+                    if (svcLock.ToInt64() > 0)
+                    {
+                        break;
+                    }
+
+                    _term.WriteLine("Retrying Lock Service Database...");
+
+                    svcLockRetries--;
+                    if (svcLockRetries > 0)
+                    {
+                        Thread.Sleep(svcLockRetryTimeout);
+                        continue;
+                    }
+
                     throw new Exception("Failed to Lock Service Database for Write");
                 }
 

src/Runner.Listener/MessageListener.cs

@@ -128,7 +128,7 @@ namespace GitHub.Runner.Listener
                         // "invalid_client" means the runner registration has been deleted from the server.
                         if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
-                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure.");
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
                             return false;
                         }
 
@@ -139,7 +139,7 @@ namespace GitHub.Runner.Listener
                         var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
                         if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
                         {
-                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure.");
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently.");
                             return false;
                         }
                     }

src/Runner.Sdk/Util/VssUtil.cs

@@ -85,6 +85,35 @@ namespace GitHub.Runner.Sdk
             VssCredentials credentials,
             IEnumerable<DelegatingHandler> additionalDelegatingHandler = null,
             TimeSpan? timeout = null)
+        {
+            RawClientHttpRequestSettings settings = GetHttpRequestSettings(timeout);
+            RawConnection connection = new(serverUri, new RawHttpMessageHandler(credentials.Federated, settings), additionalDelegatingHandler);
+            return connection;
+        }
+
+        public static VssCredentials GetVssCredential(ServiceEndpoint serviceEndpoint)
+        {
+            ArgUtil.NotNull(serviceEndpoint, nameof(serviceEndpoint));
+            ArgUtil.NotNull(serviceEndpoint.Authorization, nameof(serviceEndpoint.Authorization));
+            ArgUtil.NotNullOrEmpty(serviceEndpoint.Authorization.Scheme, nameof(serviceEndpoint.Authorization.Scheme));
+
+            if (serviceEndpoint.Authorization.Parameters.Count == 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(serviceEndpoint));
+            }
+
+            VssCredentials credentials = null;
+            string accessToken;
+            if (serviceEndpoint.Authorization.Scheme == EndpointAuthorizationSchemes.OAuth &&
+                serviceEndpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out accessToken))
+            {
+                credentials = new VssCredentials(new VssOAuthAccessTokenCredential(accessToken), CredentialPromptType.DoNotPrompt);
+            }
+
+            return credentials;
+        }
+
+        public static RawClientHttpRequestSettings GetHttpRequestSettings(TimeSpan? timeout = null)
         {
             RawClientHttpRequestSettings settings = RawClientHttpRequestSettings.Default.Clone();
 
@@ -116,30 +145,7 @@ namespace GitHub.Runner.Sdk
             // settings are applied to an HttpRequestMessage.
             settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture);
 
-            RawConnection connection = new(serverUri, new RawHttpMessageHandler(credentials.Federated, settings), additionalDelegatingHandler);
-            return connection;
-        }
-
-        public static VssCredentials GetVssCredential(ServiceEndpoint serviceEndpoint)
-        {
-            ArgUtil.NotNull(serviceEndpoint, nameof(serviceEndpoint));
-            ArgUtil.NotNull(serviceEndpoint.Authorization, nameof(serviceEndpoint.Authorization));
-            ArgUtil.NotNullOrEmpty(serviceEndpoint.Authorization.Scheme, nameof(serviceEndpoint.Authorization.Scheme));
-
-            if (serviceEndpoint.Authorization.Parameters.Count == 0)
-            {
-                throw new ArgumentOutOfRangeException(nameof(serviceEndpoint));
-            }
-
-            VssCredentials credentials = null;
-            string accessToken;
-            if (serviceEndpoint.Authorization.Scheme == EndpointAuthorizationSchemes.OAuth &&
-                serviceEndpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out accessToken))
-            {
-                credentials = new VssCredentials(new VssOAuthAccessTokenCredential(accessToken), CredentialPromptType.DoNotPrompt);
-            }
-
-            return credentials;
+            return settings;
         }
     }
 }

src/Runner.Worker/ActionManager.cs

@@ -762,6 +762,8 @@ namespace GitHub.Runner.Worker
             ArgUtil.NotNull(downloadInfo, nameof(downloadInfo));
             ArgUtil.NotNullOrEmpty(downloadInfo.NameWithOwner, nameof(downloadInfo.NameWithOwner));
             ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.Ref));
+            ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.ResolvedNameWithOwner));
+            ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.ResolvedSha));
 
             string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), downloadInfo.NameWithOwner.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), downloadInfo.Ref);
             string watermarkFile = GetWatermarkFilePath(destDirectory);
@@ -778,11 +780,6 @@ namespace GitHub.Runner.Worker
                 executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' (SHA:{downloadInfo.ResolvedSha})");
             }
 
-            await DownloadRepositoryActionAsync(executionContext, downloadInfo, destDirectory);
-        }
-
-        private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo downloadInfo, string destDirectory)
-        {
             //download and extract action in a temp folder and rename it on success
             string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
             Directory.CreateDirectory(tempDirectory);
@@ -795,102 +792,50 @@ namespace GitHub.Runner.Worker
             string link = downloadInfo?.TarballUrl;
 #endif
 
-            Trace.Info($"Save archive '{link}' into {archiveFile}.");
             try
             {
-                int retryCount = 0;
-
-                // Allow up to 20 * 60s for any action to be downloaded from github graph.
-                int timeoutSeconds = 20 * 60;
-                while (retryCount < 3)
+                var useActionArchiveCache = false;
+                if (executionContext.Global.Variables.GetBoolean("DistributedTask.UseActionArchiveCache") == true)
                 {
-                    using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
-                    using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
+                    var hasActionArchiveCache = false;
+                    var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory);
+                    if (!string.IsNullOrEmpty(actionArchiveCacheDir) &&
+                        Directory.Exists(actionArchiveCacheDir))
+                    {
+                        hasActionArchiveCache = true;
+                        Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'");
+#if OS_WINDOWS
+                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip");
+#else
+                        var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz");
+#endif
+                        if (File.Exists(cacheArchiveFile))
                         {
                             try
                             {
-                            //open zip stream in async mode
-                            using (FileStream fs = new(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
-                            using (var httpClientHandler = HostContext.CreateHttpClientHandler())
-                            using (var httpClient = new HttpClient(httpClientHandler))
-                            {
-                                httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadInfo.Authentication?.Token);
-
-                                httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
-                                using (var response = await httpClient.GetAsync(link))
-                                {
-                                    var requestId = UrlUtil.GetGitHubRequestId(response.Headers);
-                                    if (!string.IsNullOrEmpty(requestId))
-                                    {
-                                        Trace.Info($"Request URL: {link} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}");
+                                Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'");
+                                File.Copy(cacheArchiveFile, archiveFile);
+                                useActionArchiveCache = true;
+                                executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'");
                             }
-
-                                    if (response.IsSuccessStatusCode)
+                            catch (Exception ex)
                             {
-                                        using (var result = await response.Content.ReadAsStreamAsync())
-                                        {
-                                            await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
-                                            await fs.FlushAsync(actionDownloadCancellation.Token);
-
-                                            // download succeed, break out the retry loop.
-                                            break;
-                                        }
-                                    }
-                                    else if (response.StatusCode == HttpStatusCode.NotFound)
-                                    {
-                                        // It doesn't make sense to retry in this case, so just stop
-                                        throw new ActionNotFoundException(new Uri(link), requestId);
-                                    }
-                                    else
-                                    {
-                                        // Something else bad happened, let's go to our retry logic
-                                        response.EnsureSuccessStatusCode();
-                                    }
-                                }
-                            }
-                        }
-                        catch (OperationCanceledException) when (executionContext.CancellationToken.IsCancellationRequested)
-                        {
-                            Trace.Info("Action download has been cancelled.");
-                            throw;
-                        }
-                        catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2)
-                        {
-                            Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds.");
-                            throw new TimeoutException($"Action '{link}' download has timed out. Error: {ex.Message}");
-                        }
-                        catch (ActionNotFoundException)
-                        {
-                            Trace.Info($"The action at '{link}' does not exist");
-                            throw;
-                        }
-                        catch (Exception ex) when (retryCount < 2)
-                        {
-                            retryCount++;
-                            Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}");
-                            Trace.Error(ex);
-                            if (actionDownloadTimeout.Token.IsCancellationRequested)
-                            {
-                                // action download didn't finish within timeout
-                                executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds.");
-                            }
-                            else
-                            {
-                                executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}");
+                                Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}");
                             }
                         }
                     }
 
-                    if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
+                    executionContext.Global.JobTelemetry.Add(new JobTelemetry()
                     {
-                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
-                        executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
-                        await Task.Delay(backOff);
-                    }
+                        Type = JobTelemetryType.General,
+                        Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
+                    });
                 }
 
-                ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
-                executionContext.Debug($"Download '{link}' to '{archiveFile}'");
+                if (!useActionArchiveCache)
+                {
+                    await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile);
+                }
 
                 var stagingDirectory = Path.Combine(tempDirectory, "_staging");
                 Directory.CreateDirectory(stagingDirectory);
@@ -947,7 +892,6 @@ namespace GitHub.Runner.Worker
                 }
 
                 Trace.Verbose("Create watermark file indicate action download succeed.");
-                string watermarkFile = GetWatermarkFilePath(destDirectory);
                 File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
 
                 executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
@@ -1155,6 +1099,104 @@ namespace GitHub.Runner.Worker
             HostContext.SecretMasker.AddValue(base64EncodingToken);
             return new AuthenticationHeaderValue("Basic", base64EncodingToken);
         }
+
+        private async Task DownloadRepositoryArchive(IExecutionContext executionContext, string downloadUrl, string downloadAuthToken, string archiveFile)
+        {
+            Trace.Info($"Save archive '{downloadUrl}' into {archiveFile}.");
+            int retryCount = 0;
+
+            // Allow up to 20 * 60s for any action to be downloaded from github graph.
+            int timeoutSeconds = 20 * 60;
+            while (retryCount < 3)
+            {
+                using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
+                using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
+                {
+                    try
+                    {
+                        //open zip stream in async mode
+                        using (FileStream fs = new(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
+                        using (var httpClientHandler = HostContext.CreateHttpClientHandler())
+                        using (var httpClient = new HttpClient(httpClientHandler))
+                        {
+                            httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadAuthToken);
+
+                            httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
+                            using (var response = await httpClient.GetAsync(downloadUrl))
+                            {
+                                var requestId = UrlUtil.GetGitHubRequestId(response.Headers);
+                                if (!string.IsNullOrEmpty(requestId))
+                                {
+                                    Trace.Info($"Request URL: {downloadUrl} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}");
+                                }
+
+                                if (response.IsSuccessStatusCode)
+                                {
+                                    using (var result = await response.Content.ReadAsStreamAsync())
+                                    {
+                                        await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
+                                        await fs.FlushAsync(actionDownloadCancellation.Token);
+
+                                        // download succeed, break out the retry loop.
+                                        break;
+                                    }
+                                }
+                                else if (response.StatusCode == HttpStatusCode.NotFound)
+                                {
+                                    // It doesn't make sense to retry in this case, so just stop
+                                    throw new ActionNotFoundException(new Uri(downloadUrl), requestId);
+                                }
+                                else
+                                {
+                                    // Something else bad happened, let's go to our retry logic
+                                    response.EnsureSuccessStatusCode();
+                                }
+                            }
+                        }
+                    }
+                    catch (OperationCanceledException) when (executionContext.CancellationToken.IsCancellationRequested)
+                    {
+                        Trace.Info("Action download has been cancelled.");
+                        throw;
+                    }
+                    catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2)
+                    {
+                        Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds.");
+                        throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message}");
+                    }
+                    catch (ActionNotFoundException)
+                    {
+                        Trace.Info($"The action at '{downloadUrl}' does not exist");
+                        throw;
+                    }
+                    catch (Exception ex) when (retryCount < 2)
+                    {
+                        retryCount++;
+                        Trace.Error($"Fail to download archive '{downloadUrl}' -- Attempt: {retryCount}");
+                        Trace.Error(ex);
+                        if (actionDownloadTimeout.Token.IsCancellationRequested)
+                        {
+                            // action download didn't finish within timeout
+                            executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds.");
+                        }
+                        else
+                        {
+                            executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message}");
+                        }
+                    }
+                }
+
+                if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
+                {
+                    var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
+                    executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
+                    await Task.Delay(backOff);
+                }
+            }
+
+            ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
+            executionContext.Debug($"Download '{downloadUrl}' to '{archiveFile}'");
+        }
     }
 
     public sealed class Definition

src/Runner.Worker/ExecutionContext.cs

@@ -836,7 +836,6 @@ namespace GitHub.Runner.Worker
             // Actions environment
             ActionsEnvironment = message.ActionsEnvironment;
 
-
             // Service container info
             Global.ServiceContainers = new List<ContainerInfo>();
 

src/Runner.Worker/JobRunner.cs

@@ -51,6 +51,13 @@ namespace GitHub.Runner.Worker
                 HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
             }
 
+            var jobServerQueueTelemetry = false;
+            if (message.Variables.TryGetValue("DistributedTask.EnableJobServerQueueTelemetry", out VariableValue enableJobServerQueueTelemetry) &&
+                !string.IsNullOrEmpty(enableJobServerQueueTelemetry?.Value))
+            {
+                jobServerQueueTelemetry = StringUtil.ConvertToBoolean(enableJobServerQueueTelemetry.Value);
+            }
+
             ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
             if (MessageUtil.IsRunServiceJob(message.MessageType))
             {
@@ -72,7 +79,7 @@ namespace GitHub.Runner.Worker
                     launchServer.InitializeLaunchClient(new Uri(launchReceiverEndpoint), accessToken);
                 }
                 _jobServerQueue = HostContext.GetService<IJobServerQueue>();
-                _jobServerQueue.Start(message, resultServiceOnly: true);
+                _jobServerQueue.Start(message, resultsServiceOnly: true, enableTelemetry: jobServerQueueTelemetry);
             }
             else
             {
@@ -94,7 +101,7 @@ namespace GitHub.Runner.Worker
                 VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, delegatingHandlers);
                 await jobServer.ConnectAsync(jobConnection);
 
-                _jobServerQueue.Start(message);
+                _jobServerQueue.Start(message, enableTelemetry: jobServerQueueTelemetry);
                 server = jobServer;
             }
 
@@ -405,6 +412,13 @@ namespace GitHub.Runner.Worker
                 result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
             }
 
+            // include any job telemetry from the background upload process.
+            if (_jobServerQueue != null &&
+                _jobServerQueue.JobTelemetries.Count > 0)
+            {
+                jobContext.Global.JobTelemetry.AddRange(_jobServerQueue.JobTelemetries);
+            }
+
             // Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
             _tempDirectoryManager?.CleanupTempDirectory();
 

src/Sdk/Common/Common/Authentication/NoOpCredentials.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace GitHub.Services.Common
+{
+    // Set of classes used to bypass token operations
+    // Results Service and External services follow a different auth model but
+    // we are required to pass in a credentials object to create a RawHttpMessageHandler
+    public class NoOpCredentials : FederatedCredential
+    {
+        public NoOpCredentials(IssuedToken initialToken) : base(initialToken)
+        {
+        }
+
+        public override VssCredentialsType CredentialType { get; }
+        protected override IssuedTokenProvider OnCreateTokenProvider(Uri serverUrl, IHttpResponse response)
+        {
+            return null;
+        }
+    }
+}

src/Sdk/Common/Common/RawHttpMessageHandler.cs

@@ -109,7 +109,7 @@ namespace GitHub.Services.Common
             lock (m_thisLock)
             {
                 // Ensure that we attempt to use the most appropriate authentication mechanism by default.
-                if (m_tokenProvider == null)
+                if (m_tokenProvider == null && !(this.Credentials is NoOpCredentials))
                 {
                     m_tokenProvider = this.Credentials.CreateTokenProvider(request.RequestUri, null, null);
                 }
@@ -121,7 +121,8 @@ namespace GitHub.Services.Common
             HttpResponseMessageWrapper responseWrapper;
 
             Boolean lastResponseDemandedProxyAuth = false;
-            Int32 retries = m_maxAuthRetries;
+            // do not retry if we cannot recreate tokens
+            Int32 retries = this.Credentials is NoOpCredentials ? 0 : m_maxAuthRetries;
             try
             {
                 tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
@@ -138,8 +139,12 @@ namespace GitHub.Services.Common
                     }
 
                     // Let's start with sending a token
-                    IssuedToken token = await m_tokenProvider.GetTokenAsync(null, tokenSource.Token).ConfigureAwait(false);
+                    IssuedToken token = null;
+                    if (m_tokenProvider != null)
+                    {
+                        token = await m_tokenProvider.GetTokenAsync(null, tokenSource.Token).ConfigureAwait(false);
                         ApplyToken(request, token, applyICredentialsToWebProxy: lastResponseDemandedProxyAuth);
+                    }
 
                     // The WinHttpHandler will chunk any content that does not have a computed length which is
                     // not what we want. By loading into a buffer up-front we bypass this behavior and there is

src/Sdk/DTWebApi/WebApi/JobTelemetryType.cs

@@ -2,6 +2,7 @@
 
 namespace GitHub.DistributedTask.WebApi
 {
+    // do NOT add new enum since it will break backward compatibility with GHES
     public enum JobTelemetryType
     {
         [EnumMember]

src/Test/L0/Worker/ActionManagerL0.cs

@@ -293,6 +293,118 @@ namespace GitHub.Runner.Common.Tests.Worker
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public async void PrepareActions_DownloadActionFromGraph_UseCache()
+        {
+            try
+            {
+                //Arrange
+                Setup();
+                Directory.CreateDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache"));
+                Directory.CreateDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache", "actions_download-artifact"));
+                Directory.CreateDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact"));
+                Environment.SetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory, Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache"));
+
+                const string Content = @"
+# Container action
+name: '1ae80bcb-c1df-4362-bdaa-54f729c60281'
+description: 'Greet the world and record the time'
+author: 'GitHub'
+inputs:
+  greeting: # id of input
+    description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
+    required: true
+    default: 'Hello'
+  entryPoint: # id of input
+    description: 'optional docker entrypoint overwrite.'
+    required: false
+outputs:
+  time: # id of output
+    description: 'The time we did the greeting'
+icon: 'hello.svg' # vector art to display in the GitHub Marketplace
+color: 'green' # optional, decorates the entry in the GitHub Marketplace
+runs:
+  using: 'node12'
+  main: 'task.js'
+";
+                await File.WriteAllTextAsync(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact", "action.yml"), Content);
+
+#if OS_WINDOWS
+                ZipFile.CreateFromDirectory(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact"), Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache", "actions_download-artifact", "master-sha.zip"), CompressionLevel.Fastest, true);
+#else
+                string tar = WhichUtil.Which("tar", require: true, trace: _hc.GetTrace());
+
+                // tar -xzf
+                using (var processInvoker = new ProcessInvokerWrapper())
+                {
+                    processInvoker.Initialize(_hc);
+                    processInvoker.OutputDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
+                    {
+                        if (!string.IsNullOrEmpty(args.Data))
+                        {
+                            _hc.GetTrace().Info(args.Data);
+                        }
+                    });
+
+                    processInvoker.ErrorDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
+                    {
+                        if (!string.IsNullOrEmpty(args.Data))
+                        {
+                            _hc.GetTrace().Error(args.Data);
+                        }
+                    });
+
+                    string cwd = Path.GetDirectoryName(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact"));
+                    string inputDirectory = Path.GetFileName(Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "actions-download-artifact"));
+                    string archiveFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Temp), "action_cache", "actions_download-artifact", "master-sha.tar.gz");
+                    int exitCode = await processInvoker.ExecuteAsync(_hc.GetDirectory(WellKnownDirectory.Bin), tar, $"-czf \"{archiveFile}\" -C \"{cwd}\" \"{inputDirectory}\"", null, CancellationToken.None);
+                    if (exitCode != 0)
+                    {
+                        throw new NotSupportedException($"Can't use 'tar -czf' to create archive file: {archiveFile}. return code: {exitCode}.");
+                    }
+                }
+#endif
+                var actionId = Guid.NewGuid();
+                var actions = new List<Pipelines.ActionStep>
+                {
+                    new Pipelines.ActionStep()
+                    {
+                        Name = "action",
+                        Id = actionId,
+                        Reference = new Pipelines.RepositoryPathReference()
+                        {
+                            Name = "actions/download-artifact",
+                            Ref = "master",
+                            RepositoryType = "GitHub"
+                        }
+                    }
+                };
+
+                _ec.Object.Global.Variables.Set("DistributedTask.UseActionArchiveCache", bool.TrueString);
+
+                //Act
+                await _actionManager.PrepareActionsAsync(_ec.Object, actions);
+
+                //Assert
+                var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/download-artifact", "master.completed");
+                Assert.True(File.Exists(watermarkFile));
+
+                var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/download-artifact", "master", "action.yml");
+                Assert.True(File.Exists(actionYamlFile));
+
+                _hc.GetTrace().Info(File.ReadAllText(actionYamlFile));
+
+                Assert.Contains("1ae80bcb-c1df-4362-bdaa-54f729c60281", File.ReadAllText(actionYamlFile));
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory, null);
+                Teardown();
+            }
+        }
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
@@ -2272,6 +2384,7 @@ runs:
             _ec.Setup(x => x.ExpressionFunctions).Returns(new List<IFunctionInfo>());
             _ec.Object.Global.FileTable = new List<String>();
             _ec.Object.Global.Plan = new TaskOrchestrationPlanReference();
+            _ec.Object.Global.JobTelemetry = new List<JobTelemetry>();
             _ec.Setup(x => x.Write(It.IsAny<string>(), It.IsAny<string>())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); });
             _ec.Setup(x => x.AddIssue(It.IsAny<Issue>(), It.IsAny<ExecutionContextLogOptions>())).Callback((Issue issue, ExecutionContextLogOptions logOptions) => { _hc.GetTrace().Info($"[{issue.Type}]{logOptions.LogMessageOverride ?? issue.Message}"); });
             _ec.Setup(x => x.GetGitHubContext("workspace")).Returns(Path.Combine(_workFolder, "actions", "actions"));
@@ -2294,6 +2407,8 @@ runs:
                         {
                             NameWithOwner = action.NameWithOwner,
                             Ref = action.Ref,
+                            ResolvedNameWithOwner = action.NameWithOwner,
+                            ResolvedSha = $"{action.Ref}-sha",
                             TarballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/tarball/{action.Ref}",
                             ZipballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/zipball/{action.Ref}",
                         };
@@ -2313,6 +2428,8 @@ runs:
                         {
                             NameWithOwner = action.NameWithOwner,
                             Ref = action.Ref,
+                            ResolvedNameWithOwner = action.NameWithOwner,
+                            ResolvedSha = $"{action.Ref}-sha",
                             TarballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/tarball/{action.Ref}",
                             ZipballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/zipball/{action.Ref}",
                         };

src/runnerversion

@@ -1 +1 @@
-2.309.0
+2.310.0