Add retry logic around getting job messages from broker (#1939)

* Jsut simple solution without additional funcs

* Delete old comment

* resolve

* Refactor retry function, make it more common

* Make retry function generic, get rid of extra params

* delete extra using

* Add cancellation token and limit of attempts

* Add some additional logging

* Rework condition

* replace to do..while

* return `while (true)` to simplify code structure

* Add other cancelling token, add TODO comment

src/Runner.Common/RunServer.cs

@@ -17,7 +17,7 @@ namespace GitHub.Runner.Common
     {
         Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
 
-        Task<AgentJobRequestMessage> GetJobMessageAsync(string id);
+        Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken token);
     }
 
     public sealed class RunServer : RunnerService, IRunServer
@@ -67,10 +67,40 @@ namespace GitHub.Runner.Common
             }
         }
 
-        public Task<AgentJobRequestMessage> GetJobMessageAsync(string id)
+        public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken cancellationToken)
         {
             CheckConnection();
-            return _taskAgentClient.GetJobMessageAsync(id);
+            var jobMessage = RetryRequest<AgentJobRequestMessage>(async () =>
+                                                    {
+                                                        return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken);
+                                                    }, cancellationToken);
+            return jobMessage;
+        }
+
+        private async Task<T> RetryRequest<T>(Func<Task<T>> func,
+                                              CancellationToken cancellationToken,
+                                              int maxRetryAttemptsCount = 5
+                                             )
+        {
+            var retryCount = 0;
+            while (true)
+            {
+                retryCount++;
+                cancellationToken.ThrowIfCancellationRequested();
+                try
+                {
+                    return await func();
+                }
+                // TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122
+                catch (Exception ex) when (retryCount < maxRetryAttemptsCount)
+                {
+                    Trace.Error("Catch exception during get full job message");
+                    Trace.Error(ex);
+                    var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
+                    Trace.Warning($"Back off {backOff.TotalSeconds} seconds before next retry. {maxRetryAttemptsCount - retryCount} attempt left.");
+                    await Task.Delay(backOff, cancellationToken);
+                }
+            }
         }
     }
 }

src/Runner.Listener/Runner.cs

@@ -474,10 +474,9 @@ namespace GitHub.Runner.Listener
                                     var credMgr = HostContext.GetService<ICredentialManager>();
                                     var creds = credMgr.LoadCredentials();
 
-                                    // todo: add retries https://github.com/github/actions-broker/issues/49
                                     var runServer = HostContext.CreateService<IRunServer>();
                                     await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
-                                    var jobMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId);
+                                    var jobMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token);
 
                                     jobDispatcher.Run(jobMessage, runOnce);
                                     if (runOnce)