diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 8455ebef2..243e0e741 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -13,6 +13,11 @@ using GitHub.Runner.Listener.Configuration; using GitHub.Runner.Sdk; using GitHub.Services.WebApi; using Pipelines = GitHub.DistributedTask.Pipelines; +using System.Runtime.Serialization; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using GitHub.Services.Common; namespace GitHub.Runner.Listener { @@ -477,7 +482,12 @@ namespace GitHub.Runner.Listener // todo: add retries https://github.com/github/actions-broker/issues/49 var runServer = HostContext.CreateService(); await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds); - var jobMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId); + + Func> getJobRequestMessageAsync = async() => { + return await runServer.GetJobMessageAsync(messageRef.RunnerRequestId); + }; + + var jobMessage = await RetryHelper.RetryWithTimeoutAsync(getJobRequestMessageAsync, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10)); jobDispatcher.Run(jobMessage, runOnce); if (runOnce) diff --git a/src/Sdk/Common/Common/Utility/RetryHelper.cs b/src/Sdk/Common/Common/Utility/RetryHelper.cs new file mode 100644 index 000000000..4cabcb7d4 --- /dev/null +++ b/src/Sdk/Common/Common/Utility/RetryHelper.cs @@ -0,0 +1,41 @@ +using System; +using System.ComponentModel; +using System.Threading.Tasks; + +namespace GitHub.Services.Common +{ + [EditorBrowsable(EditorBrowsableState.Never)] + + public static class RetryHelper + { + public static async Task RetryWithTimeoutAsync( + Func> retriableAction, + TimeSpan minBackoff, + TimeSpan maxBackoff, + int maxTimeout = 5 + ) + { + var remainingTime = TimeSpan.FromMinutes(maxTimeout); + while (true) + { + try + { + return await retriableAction(); + } + catch + { + if (remainingTime > TimeSpan.Zero) + { + var backOff = BackoffTimerHelper.GetRandomBackoff(minBackoff, maxBackoff); + remainingTime -= backOff; + await Task.Delay(backOff); + } + else + { + throw; + } + } + } + } + } +}