Apply error throttler for unexpected exceptions as well

Bump hook version to 0.6.1 (#3350 )
Backoff to avoid excessive retries to Run Service in a duration (#3354 )
2025-12-10 04:06:57 +00:00 · 2024-06-27 11:04:09 -07:00 · 2024-06-26 14:56:33 +02:00 · 2024-06-24 16:33:22 -05:00 · 2024-06-19 16:38:32 +00:00 · 2024-06-18 12:12:10 -04:00
31 changed files with 596 additions and 74 deletions
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -4,7 +4,7 @@
  "features": {
    "ghcr.io/devcontainers/features/docker-in-docker:1": {},
    "ghcr.io/devcontainers/features/dotnet": {
-      "version": "6.0.421"
+      "version": "8.0.302"
    },
    "ghcr.io/devcontainers/features/node:1": {
      "version": "16"
--- a/.github/workflows/publish-image.yml
+++ b/.github/workflows/publish-image.yml
@@ -25,10 +25,12 @@ jobs:
      - name: Compute image version
        id: image
        uses: actions/github-script@v6
+        env:
+          RUNNER_VERSION: ${{ github.event.inputs.runnerVersion }}
        with:
          script: |
            const fs = require('fs');
-            const inputRunnerVersion = "${{ github.event.inputs.runnerVersion }}"
+            const inputRunnerVersion = process.env.RUNNER_VERSION;
            if (inputRunnerVersion) {
              console.log(`Using input runner version ${inputRunnerVersion}`)
              core.setOutput('version', inputRunnerVersion);
--- a/.gitignore
+++ b/.gitignore
@@ -26,4 +26,5 @@ _dotnetsdk
 TestResults
 TestLogs
 .DS_Store
+.mono
 **/*.DotSettings.user
--- a/images/Dockerfile
+++ b/images/Dockerfile
@@ -4,9 +4,9 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build
 ARG TARGETOS
 ARG TARGETARCH
 ARG RUNNER_VERSION
-ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.0
-ARG DOCKER_VERSION=25.0.4
-ARG BUILDX_VERSION=0.13.1
+ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.1
+ARG DOCKER_VERSION=25.0.5
+ARG BUILDX_VERSION=0.13.2

 RUN apt update -y && apt install curl unzip -y

--- a/src/Runner.Common/JobServerQueue.cs
+++ b/src/Runner.Common/JobServerQueue.cs
@@ -613,7 +613,7 @@ namespace GitHub.Runner.Common

        private void SendResultsTelemetry(Exception ex)
        {
-            var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {ex.Message}" };
+            var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {HostContext.SecretMasker.MaskSecrets(ex.Message)}" };
            issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure;

            var telemetryRecord = new TimelineRecord()
--- a/src/Runner.Common/RunServer.cs
+++ b/src/Runner.Common/RunServer.cs
@@ -62,7 +62,10 @@ namespace GitHub.Runner.Common
            CheckConnection();
            return RetryRequest<AgentJobRequestMessage>(
                async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, cancellationToken), cancellationToken,
-                shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException);
+                shouldRetry: ex =>
+                    ex is not TaskOrchestrationJobNotFoundException &&          // HTTP status 404
+                    ex is not TaskOrchestrationJobAlreadyAcquiredException &&   // HTTP status 409
+                    ex is not TaskOrchestrationJobUnprocessableException);      // HTTP status 422
        }

        public Task CompleteJobAsync(
--- a/src/Runner.Common/Runner.Common.csproj
+++ b/src/Runner.Common/Runner.Common.csproj
@@ -1,11 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <OutputType>Library</OutputType>
    <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
    <Version>$(Version)</Version>
  </PropertyGroup>

@@ -15,11 +15,11 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
  </ItemGroup>

  <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
--- a/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs
+++ b/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs
@@ -1,4 +1,5 @@
 #if OS_WINDOWS
+#pragma warning disable CA1416
 using System.IO;
 using System.Security.Cryptography;
 using System.Text;
@@ -84,4 +85,5 @@ namespace GitHub.Runner.Listener.Configuration
        }
    }
 }
+#pragma warning restore CA1416
 #endif
--- a/src/Runner.Listener/ErrorThrottler.cs
+++ b/src/Runner.Listener/ErrorThrottler.cs
@@ -0,0 +1,44 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common;
+using GitHub.Services.Common;
+
+namespace GitHub.Runner.Listener
+{
+    [ServiceLocator(Default = typeof(ErrorThrottler))]
+    public interface IErrorThrottler : IRunnerService
+    {
+        void Reset();
+        Task IncrementAndWaitAsync(CancellationToken token);
+    }
+
+    public sealed class ErrorThrottler : RunnerService, IErrorThrottler
+    {
+        internal static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(1);
+        internal static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1);
+        internal static readonly TimeSpan BackoffCoefficient = TimeSpan.FromSeconds(1);
+        private int _count = 0;
+
+        public void Reset()
+        {
+            _count = 0;
+        }
+
+        public async Task IncrementAndWaitAsync(CancellationToken token)
+        {
+            if (++_count <= 1)
+            {
+                return;
+            }
+
+            TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(
+                attempt: _count - 2, // 0-based attempt
+                minBackoff: MinBackoff,
+                maxBackoff: MaxBackoff,
+                deltaBackoff: BackoffCoefficient);
+            Trace.Warning($"Back off {backoff.TotalSeconds} seconds before next attempt. Current consecutive error count: {_count}");
+            await HostContext.Delay(backoff, token);
+        }
+    }
+}
--- a/src/Runner.Listener/Runner.Listener.csproj
+++ b/src/Runner.Listener/Runner.Listener.csproj
@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <OutputType>Exe</OutputType>
    <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
    <Version>$(Version)</Version>
    <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
    <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
@@ -18,11 +19,11 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="System.IO.FileSystem.AccessControl" Version="4.4.0" />
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
+    <PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="8.0.0" />
  </ItemGroup>

  <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
--- a/src/Runner.Listener/Runner.cs
+++ b/src/Runner.Listener/Runner.cs
@@ -32,10 +32,25 @@ namespace GitHub.Runner.Listener
        private bool _inConfigStage;
        private ManualResetEvent _completedCommand = new(false);

+        // <summary>
+        // Helps avoid excessive calls to Run Service when encountering non-retriable errors from /acquirejob.
+        // Normally we rely on the HTTP clients to back off between retry attempts. However, acquiring a job
+        // involves calls to both Run Serivce and Broker. And Run Service and Broker communicate with each other
+        // in an async fashion.
+        //
+        // When Run Service encounters a non-retriable error, it sends an async message to Broker. The runner will,
+        // however, immediately call Broker to get the next message. If the async event from Run Service to Broker
+        // has not yet been processed, the next message from Broker may be the same job message.
+        //
+        // The error throttler helps us back off when encountering successive, non-retriable errors from /acquirejob.
+        // </summary>
+        private IErrorThrottler _acquireJobThrottler;
+
        public override void Initialize(IHostContext hostContext)
        {
            base.Initialize(hostContext);
            _term = HostContext.GetService<ITerminal>();
+            _acquireJobThrottler = HostContext.CreateService<IErrorThrottler>();
        }

        public async Task<int> ExecuteCommand(CommandSettings command)
@@ -213,10 +228,12 @@ namespace GitHub.Runner.Listener
                            var configFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), config.Key);
                            var configContent = Convert.FromBase64String(config.Value);
 #if OS_WINDOWS
+#pragma warning disable CA1416
                            if (configFile == HostContext.GetConfigFile(WellKnownConfigFile.RSACredentials))
                            {
                                configContent = ProtectedData.Protect(configContent, null, DataProtectionScope.LocalMachine);
                            }
+#pragma warning restore CA1416
 #endif
                            File.WriteAllBytes(configFile, configContent);
                            File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden);
@@ -563,18 +580,22 @@ namespace GitHub.Runner.Listener
                                        await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), creds);
                                        try
                                        {
-                                            jobRequestMessage =
-                                            await runServer.GetJobMessageAsync(messageRef.RunnerRequestId,
-                                            messageQueueLoopTokenSource.Token);
+                                            jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token);
+                                            _acquireJobThrottler.Reset();
                                        }
-                                        catch (TaskOrchestrationJobAlreadyAcquiredException)
+                                        catch (Exception ex) when (
+                                            ex is TaskOrchestrationJobNotFoundException ||          // HTTP status 404
+                                            ex is TaskOrchestrationJobAlreadyAcquiredException ||   // HTTP status 409
+                                            ex is TaskOrchestrationJobUnprocessableException)       // HTTP status 422
                                        {
-                                            Trace.Info("Job is already acquired, skip this message.");
+                                            Trace.Info($"Skipping message Job. {ex.Message}");
+                                            await _acquireJobThrottler.IncrementAndWaitAsync(messageQueueLoopTokenSource.Token);
                                            continue;
                                        }
                                        catch (Exception ex)
                                        {
                                            Trace.Error($"Caught exception from acquiring job message: {ex}");
+                                            await _acquireJobThrottler.IncrementAndWaitAsync(messageQueueLoopTokenSource.Token);
                                            continue;
                                        }
                                    }
--- a/src/Runner.PluginHost/Runner.PluginHost.csproj
+++ b/src/Runner.PluginHost/Runner.PluginHost.csproj
@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <OutputType>Exe</OutputType>
    <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
    <Version>$(Version)</Version>
    <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
    <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
--- a/src/Runner.Plugins/Runner.Plugins.csproj
+++ b/src/Runner.Plugins/Runner.Plugins.csproj
@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <OutputType>Library</OutputType>
    <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
    <Version>$(Version)</Version>
  </PropertyGroup>

--- a/src/Runner.Sdk/Runner.Sdk.csproj
+++ b/src/Runner.Sdk/Runner.Sdk.csproj
@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <OutputType>Library</OutputType>
    <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
    <Version>$(Version)</Version>
  </PropertyGroup>

@@ -14,9 +15,9 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
-    <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
  </ItemGroup>

  <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
--- a/src/Runner.Worker/Runner.Worker.csproj
+++ b/src/Runner.Worker/Runner.Worker.csproj
@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <OutputType>Exe</OutputType>
    <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+    <SelfContained>true</SelfContained>
    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-    <NoWarn>NU1701;NU1603</NoWarn>
+    <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
    <Version>$(Version)</Version>
    <PredefinedCulturesOnly>false</PredefinedCulturesOnly>
    <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
@@ -18,9 +19,9 @@
  </ItemGroup>

  <ItemGroup>
-    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
-    <PackageReference Include="System.Threading.Channels" Version="4.4.0" />
+    <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="8.0.0" />
+    <PackageReference Include="System.Threading.Channels" Version="8.0.0" />
    <PackageReference Include="YamlDotNet.Signed" Version="5.3.0" />
  </ItemGroup>

--- a/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs
+++ b/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs
@@ -34,6 +34,7 @@ namespace GitHub.Services.Common

        public String PropertyName { get; set; }

+        [Obsolete]
        [SecurityCritical]
        public override void GetObjectData(SerializationInfo info, StreamingContext context)
        {
--- a/src/Sdk/Common/Common/VssException.cs
+++ b/src/Sdk/Common/Common/VssException.cs
@@ -127,6 +127,7 @@ namespace GitHub.Services.Common
            EventId = (int)info.GetValue("m_eventId", typeof(int));
        }

+        [Obsolete]
        [SecurityCritical]
        public override void GetObjectData(SerializationInfo info, StreamingContext context)
        {
--- a/src/Sdk/DTWebApi/WebApi/Exceptions.cs
+++ b/src/Sdk/DTWebApi/WebApi/Exceptions.cs
@@ -1539,6 +1539,26 @@ namespace GitHub.DistributedTask.WebApi
        }
    }

+    [Serializable]
+    [ExceptionMapping("0.0", "3.0", "TaskOrchestrationJobUnprocessableException", "GitHub.DistributedTask.WebApi.TaskOrchestrationJobUnprocessableException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")]
+    public sealed class TaskOrchestrationJobUnprocessableException : DistributedTaskException
+    {
+        public TaskOrchestrationJobUnprocessableException(String message)
+            : base(message)
+        {
+        }
+
+        public TaskOrchestrationJobUnprocessableException(String message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+
+        private TaskOrchestrationJobUnprocessableException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+        }
+    }
+
    [Serializable]
    [ExceptionMapping("0.0", "3.0", "TaskOrchestrationPlanSecurityException", "GitHub.DistributedTask.WebApi.TaskOrchestrationPlanSecurityException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")]
    public sealed class TaskOrchestrationPlanSecurityException : DistributedTaskException
--- a/src/Sdk/RSWebApi/Contracts/RunServiceError.cs
+++ b/src/Sdk/RSWebApi/Contracts/RunServiceError.cs
@@ -0,0 +1,17 @@
+using System.Runtime.Serialization;
+
+namespace GitHub.Actions.RunService.WebApi
+{
+    [DataContract]
+    public class RunServiceError
+    {
+        [DataMember(Name = "source", EmitDefaultValue = false)]
+        public string Source { get; set; }
+
+        [DataMember(Name = "statusCode", EmitDefaultValue = false)]
+        public int Code { get; set; }
+
+        [DataMember(Name = "errorMessage", EmitDefaultValue = false)]
+        public string Message { get; set; }
+    }
+}
--- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs
+++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs
@@ -86,6 +86,7 @@ namespace GitHub.Actions.RunService.WebApi
                httpMethod,
                requestUri: requestUri,
                content: requestContent,
+                readErrorBody: true,
                cancellationToken: cancellationToken);

            if (result.IsSuccess)
@@ -93,14 +94,35 @@ namespace GitHub.Actions.RunService.WebApi
                return result.Value;
            }

+            if (TryParseErrorBody(result.ErrorBody, out RunServiceError error))
+            {
+                switch ((HttpStatusCode)error.Code)
+                {
+                    case HttpStatusCode.NotFound:
+                        throw new TaskOrchestrationJobNotFoundException($"Job message not found '{messageId}'. {error.Message}");
+                    case HttpStatusCode.Conflict:
+                        throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired '{messageId}'. {error.Message}");
+                    case HttpStatusCode.UnprocessableEntity:
+                        throw new TaskOrchestrationJobUnprocessableException($"Unprocessable job '{messageId}'. {error.Message}");
+                }
+            }
+
+            // Temporary back compat
            switch (result.StatusCode)
            {
                case HttpStatusCode.NotFound:
                    throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}");
                case HttpStatusCode.Conflict:
                    throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired: {messageId}");
-                default:
-                    throw new Exception($"Failed to get job message: {result.Error}");
+            }
+
+            if (!string.IsNullOrEmpty(result.ErrorBody))
+            {
+                throw new Exception($"Failed to get job message: {result.Error}. {Truncate(result.ErrorBody)}");
+            }
+            else
+            {
+                throw new Exception($"Failed to get job message: {result.Error}");
            }
        }

@@ -108,7 +130,7 @@ namespace GitHub.Actions.RunService.WebApi
            Uri requestUri,
            Guid planId,
            Guid jobId,
-            TaskResult result,
+            TaskResult conclusion,
            Dictionary<String, VariableValue> outputs,
            IList<StepResult> stepResults,
            IList<Annotation> jobAnnotations,
@@ -120,7 +142,7 @@ namespace GitHub.Actions.RunService.WebApi
            {
                PlanID = planId,
                JobID = jobId,
-                Conclusion = result,
+                Conclusion = conclusion,
                Outputs = outputs,
                StepResults = stepResults,
                Annotations = jobAnnotations,
@@ -130,22 +152,39 @@ namespace GitHub.Actions.RunService.WebApi
            requestUri = new Uri(requestUri, "completejob");

            var requestContent = new ObjectContent<CompleteJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
-            var response = await SendAsync(
+            var result = await Send2Async(
                    httpMethod,
                    requestUri,
                    content: requestContent,
                    cancellationToken: cancellationToken);
-            if (response.IsSuccessStatusCode)
+            if (result.IsSuccess)
            {
                return;
            }

-            switch (response.StatusCode)
+            if (TryParseErrorBody(result.ErrorBody, out RunServiceError error))
+            {
+                switch ((HttpStatusCode)error.Code)
+                {
+                    case HttpStatusCode.NotFound:
+                        throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}. {error.Message}");
+                }
+            }
+
+            // Temporary back compat
+            switch (result.StatusCode)
            {
                case HttpStatusCode.NotFound:
                    throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}");
-                default:
-                    throw new Exception($"Failed to complete job: {response.ReasonPhrase}");
+            }
+
+            if (!string.IsNullOrEmpty(result.ErrorBody))
+            {
+                throw new Exception($"Failed to complete job: {result.Error}. {Truncate(result.ErrorBody)}");
+            }
+            else
+            {
+                throw new Exception($"Failed to complete job: {result.Error}");
            }
        }

@@ -169,6 +208,7 @@ namespace GitHub.Actions.RunService.WebApi
                httpMethod,
                requestUri,
                content: requestContent,
+                readErrorBody: true,
                cancellationToken: cancellationToken);

            if (result.IsSuccess)
@@ -176,12 +216,29 @@ namespace GitHub.Actions.RunService.WebApi
                return result.Value;
            }

+            if (TryParseErrorBody(result.ErrorBody, out RunServiceError error))
+            {
+                switch ((HttpStatusCode)error.Code)
+                {
+                    case HttpStatusCode.NotFound:
+                        throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}. {error.Message}");
+                }
+            }
+
+            // Temporary back compat
            switch (result.StatusCode)
            {
                case HttpStatusCode.NotFound:
                    throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}");
-                default:
-                    throw new Exception($"Failed to renew job: {result.Error}");
+            }
+
+            if (!string.IsNullOrEmpty(result.ErrorBody))
+            {
+                throw new Exception($"Failed to renew job: {result.Error}. {Truncate(result.ErrorBody)}");
+            }
+            else
+            {
+                throw new Exception($"Failed to renew job: {result.Error}");
            }
        }

@@ -190,5 +247,36 @@ namespace GitHub.Actions.RunService.WebApi
            var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
            return JsonConvert.DeserializeObject<T>(json, s_serializerSettings);
        }
+
+        private static bool TryParseErrorBody(string errorBody, out RunServiceError error)
+        {
+            if (!string.IsNullOrEmpty(errorBody))
+            {
+                try
+                {
+                    error = JsonUtility.FromString<RunServiceError>(errorBody);
+                    if (error?.Source == "actions-run-service")
+                    {
+                        return true;
+                    }
+                }
+                catch (Exception)
+                {
+                }
+            }
+
+            error = null;
+            return false;
+        }
+
+        private static string Truncate(string errorBody)
+        {
+            if (errorBody.Length > 100)
+            {
+                return errorBody.Substring(0, 100) + "[truncated]";
+            }
+
+            return errorBody;
+        }
    }
 }
--- a/src/Sdk/Sdk.csproj
+++ b/src/Sdk/Sdk.csproj
@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">

    <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
        <OutputType>Library</OutputType>
        <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
+        <!-- <SelfContained>true</SelfContained> -->
        <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-        <NoWarn>NU1701;NU1603</NoWarn>
+        <NoWarn>NU1701;NU1603;SYSLIB0050;SYSLIB0051</NoWarn>
        <Version>$(Version)</Version>
        <DefineConstants>TRACE</DefineConstants>
        <LangVersion>8.0</LangVersion>
@@ -14,13 +15,13 @@

    <ItemGroup>
        <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
-        <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
+        <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
        <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-        <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.9" />
-        <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="5.2.1" />
-        <PackageReference Include="System.Security.Cryptography.Cng" Version="4.4.0" />
-        <PackageReference Include="System.Security.Cryptography.Pkcs" Version="4.4.0" />
-        <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
+        <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="6.0.0" />
+        <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="7.5.1" />
+        <PackageReference Include="System.Security.Cryptography.Cng" Version="5.0.0" />
+        <PackageReference Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
+        <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="8.0.0" />
        <PackageReference Include="Minimatch" Version="2.0.0" />
        <PackageReference Include="YamlDotNet.Signed" Version="5.3.0" />
        <PackageReference Include="System.Net.Http" Version="4.3.4" />
--- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs
+++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs
@@ -85,6 +85,7 @@ namespace GitHub.Services.OAuth
            set;
        }

+        [Obsolete]
        public override void GetObjectData(SerializationInfo info, StreamingContext context)
        {
            base.GetObjectData(info, context);
--- a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs
+++ b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs
@@ -101,7 +101,7 @@ namespace Sdk.WebApi.WebApi
            }
        }

-        protected Task<RawHttpClientResult<T>> SendAsync<T>(
+        protected async Task<RawHttpClientResult> Send2Async(
            HttpMethod method,
            Uri requestUri,
            HttpContent content = null,
@@ -109,7 +109,47 @@ namespace Sdk.WebApi.WebApi
            Object userState = null,
            CancellationToken cancellationToken = default(CancellationToken))
        {
-            return SendAsync<T>(method, null, requestUri, content, queryParameters, userState, cancellationToken);
+            using (var response = await SendAsync(method, requestUri, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
+            {
+                if (response.IsSuccessStatusCode)
+                {
+                    return new RawHttpClientResult(
+                        isSuccess: true,
+                        error: string.Empty,
+                        statusCode: response.StatusCode);
+                }
+                else
+                {
+                    var errorBody = default(string);
+                    try
+                    {
+                        errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+                    }
+                    catch (Exception ex)
+                    {
+                        errorBody = $"Error reading HTTP response body: {ex.Message}";
+                    }
+
+                    string errorMessage = $"Error: {response.ReasonPhrase}";
+                    return new RawHttpClientResult(
+                        isSuccess: false,
+                        error: errorMessage,
+                        statusCode: response.StatusCode,
+                        errorBody: errorBody);
+                }
+            }
+        }
+
+        protected Task<RawHttpClientResult<T>> SendAsync<T>(
+            HttpMethod method,
+            Uri requestUri,
+            HttpContent content = null,
+            IEnumerable<KeyValuePair<String, String>> queryParameters = null,
+            Boolean readErrorBody = false,
+            Object userState = null,
+            CancellationToken cancellationToken = default(CancellationToken))
+        {
+            return SendAsync<T>(method, null, requestUri, content, queryParameters, readErrorBody, userState, cancellationToken);
        }

        protected async Task<RawHttpClientResult<T>> SendAsync<T>(
@@ -118,18 +158,20 @@ namespace Sdk.WebApi.WebApi
            Uri requestUri,
            HttpContent content = null,
            IEnumerable<KeyValuePair<String, String>> queryParameters = null,
+            Boolean readErrorBody = false,
            Object userState = null,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
            using (HttpRequestMessage requestMessage = CreateRequestMessage(method, additionalHeaders, requestUri, content, queryParameters))
            {
-                return await SendAsync<T>(requestMessage, userState, cancellationToken).ConfigureAwait(false);
+                return await SendAsync<T>(requestMessage, readErrorBody, userState, cancellationToken).ConfigureAwait(false);
            }
        }

        protected async Task<RawHttpClientResult<T>> SendAsync<T>(
            HttpRequestMessage message,
+            Boolean readErrorBody = false,
            Object userState = null,
            CancellationToken cancellationToken = default(CancellationToken))
        {
@@ -145,8 +187,21 @@ namespace Sdk.WebApi.WebApi
                }
                else
                {
+                    var errorBody = default(string);
+                    if (readErrorBody)
+                    {
+                        try
+                        {
+                            errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+                        }
+                        catch (Exception ex)
+                        {
+                            errorBody = $"Error reading HTTP response body: {ex.Message}";
+                        }
+                    }
+
                    string errorMessage = $"Error: {response.ReasonPhrase}";
-                    return RawHttpClientResult<T>.Fail(errorMessage, response.StatusCode);
+                    return RawHttpClientResult<T>.Fail(errorMessage, response.StatusCode, errorBody);
                }
            }
        }
--- a/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs
+++ b/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs
@@ -5,15 +5,27 @@ namespace Sdk.WebApi.WebApi
    public class RawHttpClientResult
    {
        public bool IsSuccess { get; protected set; }
+
+        /// <summary>
+        /// A description of the HTTP status code, like "Error: Unprocessable Entity"
+        /// </summary>
        public string Error { get; protected set; }
+
+        /// <summary>
+        /// The HTTP response body for unsuccessful HTTP status codes, or an error message when reading the response body fails.
+        /// </summary>
+        public string ErrorBody { get; protected set; }
+
        public HttpStatusCode StatusCode { get; protected set; }
+
        public bool IsFailure => !IsSuccess;

-        protected RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode)
+        public RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode, string errorBody = null)
        {
            IsSuccess = isSuccess;
            Error = error;
            StatusCode = statusCode;
+            ErrorBody = errorBody;
        }
    }

@@ -21,13 +33,13 @@ namespace Sdk.WebApi.WebApi
    {
        public T Value { get; private set; }

-        protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode)
-            : base(isSuccess, error, statusCode)
+        protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode, string errorBody)
+            : base(isSuccess, error, statusCode, errorBody)
        {
            Value = value;
        }

-        public static RawHttpClientResult<T> Fail(string message, HttpStatusCode statusCode) => new RawHttpClientResult<T>(default(T), false, message, statusCode);
-        public static RawHttpClientResult<T> Ok(T value) => new RawHttpClientResult<T>(value, true, string.Empty, HttpStatusCode.OK);
+        public static RawHttpClientResult<T> Fail(string message, HttpStatusCode statusCode, string errorBody) => new RawHttpClientResult<T>(default(T), false, message, statusCode, errorBody);
+        public static RawHttpClientResult<T> Ok(T value) => new RawHttpClientResult<T>(value, true, string.Empty, HttpStatusCode.OK, null);
    }
 }
--- a/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs
+++ b/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs
@@ -24,6 +24,7 @@ namespace GitHub.Services.WebApi
            HttpStatusCode = (HttpStatusCode)info.GetInt32("HttpStatusCode");
        }

+        [Obsolete]
        [SecurityCritical]
        public override void GetObjectData(SerializationInfo info, StreamingContext context)
        {
--- a/src/Test/L0/Listener/ErrorThrottlerL0.cs
+++ b/src/Test/L0/Listener/ErrorThrottlerL0.cs
@@ -0,0 +1,213 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Listener;
+using GitHub.Runner.Listener.Configuration;
+using GitHub.Runner.Common.Tests;
+using System.Runtime.CompilerServices;
+using GitHub.Services.WebApi;
+using Moq;
+using Xunit;
+
+namespace GitHub.Runner.Common.Tests.Listener
+{
+    public sealed class ErrorThrottlerL0
+    {
+        [Theory]
+        [InlineData(1)]
+        [InlineData(2)]
+        [InlineData(3)]
+        [InlineData(4)]
+        [InlineData(5)]
+        [InlineData(6)]
+        [InlineData(7)]
+        [InlineData(8)]
+        public async void TestIncrementAndWait(int totalAttempts)
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                // Arrange
+                var errorThrottler = new ErrorThrottler();
+                errorThrottler.Initialize(hc);
+                var eventArgs = new List<DelayEventArgs>();
+                hc.Delaying += (sender, args) =>
+                {
+                    eventArgs.Add(args);
+                };
+
+                // Act
+                for (int attempt = 1; attempt <= totalAttempts; attempt++)
+                {
+                    await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                }
+
+                // Assert
+                Assert.Equal(totalAttempts - 1, eventArgs.Count);
+                for (int i = 0; i < eventArgs.Count; i++)
+                {
+                    // Expected milliseconds
+                    int expectedMin;
+                    int expectedMax;
+
+                    switch (i)
+                    {
+                        case 0:
+                            expectedMin = 1000; // Min backoff
+                            expectedMax = 1000;
+                            break;
+                        case 1:
+                            expectedMin = 1800; // Min + 0.8 * Coefficient
+                            expectedMax = 2200; // Min + 1.2 * Coefficient
+                            break;
+                        case 2:
+                            expectedMin = 3400; // Min + 0.8 * Coefficient * 3
+                            expectedMax = 4600; // Min + 1.2 * Coefficient * 3
+                            break;
+                        case 3:
+                            expectedMin = 6600; // Min + 0.8 * Coefficient * 7
+                            expectedMax = 9400; // Min + 1.2 * Coefficient * 7
+                            break;
+                        case 4:
+                            expectedMin = 13000; // Min + 0.8 * Coefficient * 15
+                            expectedMax = 19000; // Min + 1.2 * Coefficient * 15
+                            break;
+                        case 5:
+                            expectedMin = 25800; // Min + 0.8 * Coefficient * 31
+                            expectedMax = 38200; // Min + 1.2 * Coefficient * 31
+                            break;
+                        case 6:
+                            expectedMin = 51400; // Min + 0.8 * Coefficient * 63
+                            expectedMax = 60000; // Max backoff
+                            break;
+                        case 7:
+                            expectedMin = 60000;
+                            expectedMax = 60000;
+                            break;
+                        default:
+                            throw new NotSupportedException("Unexpected eventArgs count");
+                    }
+
+                    var actualMilliseconds = eventArgs[i].Delay.TotalMilliseconds;
+                    Assert.True(expectedMin <= actualMilliseconds, $"Unexpected min delay for eventArgs[{i}]. Expected min {expectedMin}, actual {actualMilliseconds}");
+                    Assert.True(expectedMax >= actualMilliseconds, $"Unexpected max delay for eventArgs[{i}]. Expected max {expectedMax}, actual {actualMilliseconds}");
+                }
+            }
+        }
+
+        [Fact]
+        public async void TestReset()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                // Arrange
+                var errorThrottler = new ErrorThrottler();
+                errorThrottler.Initialize(hc);
+                var eventArgs = new List<DelayEventArgs>();
+                hc.Delaying += (sender, args) =>
+                {
+                    eventArgs.Add(args);
+                };
+
+                // Act
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                errorThrottler.Reset();
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+
+                // Assert
+                Assert.Equal(4, eventArgs.Count);
+                for (int i = 0; i < eventArgs.Count; i++)
+                {
+                    // Expected milliseconds
+                    int expectedMin;
+                    int expectedMax;
+
+                    switch (i)
+                    {
+                        case 0:
+                        case 2:
+                            expectedMin = 1000; // Min backoff
+                            expectedMax = 1000;
+                            break;
+                        case 1:
+                        case 3:
+                            expectedMin = 1800; // Min + 0.8 * Coefficient
+                            expectedMax = 2200; // Min + 1.2 * Coefficient
+                            break;
+                        default:
+                            throw new NotSupportedException("Unexpected eventArgs count");
+                    }
+
+                    var actualMilliseconds = eventArgs[i].Delay.TotalMilliseconds;
+                    Assert.True(expectedMin <= actualMilliseconds, $"Unexpected min delay for eventArgs[{i}]. Expected min {expectedMin}, actual {actualMilliseconds}");
+                    Assert.True(expectedMax >= actualMilliseconds, $"Unexpected max delay for eventArgs[{i}]. Expected max {expectedMax}, actual {actualMilliseconds}");
+                }
+            }
+        }
+
+        [Fact]
+        public async void TestReceivesCancellationToken()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                // Arrange
+                var errorThrottler = new ErrorThrottler();
+                errorThrottler.Initialize(hc);
+                var eventArgs = new List<DelayEventArgs>();
+                hc.Delaying += (sender, args) =>
+                {
+                    eventArgs.Add(args);
+                };
+                var cancellationTokenSource1 = new CancellationTokenSource();
+                var cancellationTokenSource2 = new CancellationTokenSource();
+                var cancellationTokenSource3 = new CancellationTokenSource();
+
+                // Act
+                await errorThrottler.IncrementAndWaitAsync(cancellationTokenSource1.Token);
+                await errorThrottler.IncrementAndWaitAsync(cancellationTokenSource2.Token);
+                await errorThrottler.IncrementAndWaitAsync(cancellationTokenSource3.Token);
+
+                // Assert
+                Assert.Equal(2, eventArgs.Count);
+                Assert.Equal(cancellationTokenSource2.Token, eventArgs[0].Token);
+                Assert.Equal(cancellationTokenSource3.Token, eventArgs[1].Token);
+            }
+        }
+
+        [Fact]
+        public async void TestReceivesSender()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                // Arrange
+                var errorThrottler = new ErrorThrottler();
+                errorThrottler.Initialize(hc);
+                var senders = new List<object>();
+                hc.Delaying += (sender, args) =>
+                {
+                    senders.Add(sender);
+                };
+
+                // Act
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+                await errorThrottler.IncrementAndWaitAsync(CancellationToken.None);
+
+                // Assert
+                Assert.Equal(2, senders.Count);
+                Assert.Equal(hc, senders[0]);
+                Assert.Equal(hc, senders[1]);
+            }
+        }
+
+        private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
+        {
+            return new TestHostContext(this, testName);
+        }
+    }
+}
--- a/src/Test/L0/Listener/RunnerL0.cs
+++ b/src/Test/L0/Listener/RunnerL0.cs
@@ -23,6 +23,7 @@ namespace GitHub.Runner.Common.Tests.Listener
        private Mock<ITerminal> _term;
        private Mock<IConfigurationStore> _configStore;
        private Mock<ISelfUpdater> _updater;
+        private Mock<IErrorThrottler> _acquireJobThrottler;

        public RunnerL0()
        {
@@ -35,6 +36,7 @@ namespace GitHub.Runner.Common.Tests.Listener
            _term = new Mock<ITerminal>();
            _configStore = new Mock<IConfigurationStore>();
            _updater = new Mock<ISelfUpdater>();
+            _acquireJobThrottler = new Mock<IErrorThrottler>();
        }

        private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName)
@@ -67,6 +69,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                hc.SetSingleton<IPromptManager>(_promptManager.Object);
                hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);
                runner.Initialize(hc);
                var settings = new RunnerSettings
                {
@@ -174,6 +177,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                hc.SetSingleton<IPromptManager>(_promptManager.Object);
                hc.SetSingleton<IMessageListener>(_messageListener.Object);
                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);

                var command = new CommandSettings(hc, args);

@@ -205,6 +209,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                hc.SetSingleton<IPromptManager>(_promptManager.Object);
                hc.SetSingleton<IMessageListener>(_messageListener.Object);
                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);

                var command = new CommandSettings(hc, new[] { "run" });

@@ -242,6 +247,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                hc.SetSingleton<IPromptManager>(_promptManager.Object);
                hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);
                runner.Initialize(hc);
                var settings = new RunnerSettings
                {
@@ -338,6 +344,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                hc.SetSingleton<IPromptManager>(_promptManager.Object);
                hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);
                runner.Initialize(hc);
                var settings = new RunnerSettings
                {
@@ -439,6 +446,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
                hc.SetSingleton<ISelfUpdater>(_updater.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);

                runner.Initialize(hc);
                var settings = new RunnerSettings
@@ -522,6 +530,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                hc.SetSingleton<IConfigurationManager>(_configurationManager.Object);
                hc.SetSingleton<IConfigurationStore>(_configStore.Object);
                hc.SetSingleton<IPromptManager>(_promptManager.Object);
+                hc.EnqueueInstance<IErrorThrottler>(_acquireJobThrottler.Object);

                var command = new CommandSettings(hc, new[] { "remove", "--local" });

--- a/src/Test/L0/TestHostContext.cs
+++ b/src/Test/L0/TestHostContext.cs
@@ -30,9 +30,11 @@ namespace GitHub.Runner.Common.Tests
        private string _tempDirectoryRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("D"));
        private StartupType _startupType;
        public event EventHandler Unloading;
+        public event EventHandler<DelayEventArgs> Delaying;
        public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token;
        public ShutdownReason RunnerShutdownReason { get; private set; }
        public ISecretMasker SecretMasker => _secretMasker;
+
        public TestHostContext(object testClass, [CallerMemberName] string testName = "")
        {
            ArgUtil.NotNull(testClass, nameof(testClass));
@@ -92,6 +94,14 @@ namespace GitHub.Runner.Common.Tests

        public async Task Delay(TimeSpan delay, CancellationToken token)
        {
+            // Event callback
+            EventHandler<DelayEventArgs> handler = Delaying;
+            if (handler != null)
+            {
+                handler(this, new DelayEventArgs(delay, token));
+            }
+
+            // Delay zero
            await Task.Delay(TimeSpan.Zero);
        }

@@ -361,4 +371,19 @@ namespace GitHub.Runner.Common.Tests
            }
        }
    }
+
+    public class DelayEventArgs : EventArgs
+    {
+        public DelayEventArgs(
+            TimeSpan delay,
+            CancellationToken token)
+        {
+            Delay = delay;
+            Token = token;
+        }
+
+        public TimeSpan Delay { get; }
+
+        public CancellationToken Token { get; }
+    }
 }
--- a/src/Test/Test.csproj
+++ b/src/Test/Test.csproj
@@ -1,9 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
    <PropertyGroup>
-        <TargetFramework>net6.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
        <RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64</RuntimeIdentifiers>
        <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
-        <NoWarn>NU1701;NU1603;NU1603;xUnit2013;</NoWarn>
+        <NoWarn>NU1701;NU1603;NU1603;xUnit2013;SYSLIB0050;SYSLIB0051</NoWarn>
    </PropertyGroup>

    <ItemGroup>
@@ -15,13 +15,13 @@
    </ItemGroup>

    <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.2.0" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
        <PackageReference Include="xunit" Version="2.7.1" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.5.8" />
        <PackageReference Include="System.Buffers" Version="4.5.1" />
-        <PackageReference Include="System.Reflection.TypeExtensions" Version="4.4.0" />
+        <PackageReference Include="System.Reflection.TypeExtensions" Version="4.7.0" />
        <PackageReference Include="System.Threading.ThreadPool" Version="4.3.0" />
-        <PackageReference Include="Moq" Version="4.11.0" />
+        <PackageReference Include="Moq" Version="4.20.70" />
    </ItemGroup>

    <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
--- a/src/dev.sh
+++ b/src/dev.sh
@@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout"
 DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x"
 PACKAGE_DIR="$SCRIPT_DIR/../_package"
 DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk"
-DOTNETSDK_VERSION="6.0.421"
+DOTNETSDK_VERSION="8.0.302"
 DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION"
 RUNNER_VERSION=$(cat runnerversion)

--- a/src/global.json
+++ b/src/global.json
@@ -1,5 +1,5 @@
 {
  "sdk": {
-    "version": "6.0.421"
+    "version": "8.0.302"
  }
 }
Author	SHA1	Message	Date
eric sciple	5e3f8b2fe5	Apply error throttler for unexpected exceptions as well	2024-06-27 11:04:09 -07:00
Nikola Jokic	70746ff593	Bump hook version to 0.6.1 (#3350 )	2024-06-26 14:56:33 +02:00
eric sciple	054fc2e046	Backoff to avoid excessive retries to Run Service in a duration (#3354 )	2024-06-24 16:33:22 -05:00
eric sciple	ecb732eaf4	Receive error body from Run Service (#3342 )	2024-06-19 16:38:32 +00:00
dependabot[bot]	3dab1f1fb0	Bump System.Security.Cryptography.Pkcs from 5.0.0 to 8.0.0 in /src (#3347 ) Bumps [System.Security.Cryptography.Pkcs](https://github.com/dotnet/runtime) from 5.0.0 to 8.0.0. - [Release notes](https://github.com/dotnet/runtime/releases) - [Commits](https://github.com/dotnet/runtime/compare/v5.0.0...v8.0.0) --- updated-dependencies: - dependency-name: System.Security.Cryptography.Pkcs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2024-06-18 12:12:10 -04:00
github-actions[bot]	8f1c723ba0	Upgrade dotnet sdk to v8.0.302 (#3346 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2024-06-18 12:03:44 -04:00
Tingluo Huang	1e74a8137b	Bump runner to dotnet 8 (#3345 ) * Bump runner to dotnet 8 * .	2024-06-18 11:28:53 -04:00
Josh Gross	3f28dd845f	Pass runner version as environment variable in workflow (#3318 )	2024-06-10 18:13:17 -04:00
Tingluo Huang	edfdbb9661	Make sure we mask secrets when reporting telemetry. (#3315 )	2024-06-04 09:57:15 -04:00
Hidetake Iwata	00888c10f9	Bump docker version and docker buildx version (#3277 )	2024-05-31 16:22:54 +00:00