Issue 1662: retry policy for methods GetTenantCredential and GetJITRunnerTokenAsync (#1691)

* Issue 1662: Adding retry policy for methods GetTenantCredential and GetJITRunnerTokenAsync

* Adding HttpClient creation to the retry

* Random backoff time

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -613,6 +613,9 @@ namespace GitHub.Runner.Listener.Configuration
                 throw new ArgumentException($"'{githubUrl}' should point to an org or repository.");
             }
 
+            int retryCount = 0;
+            while(retryCount < 3)
+            {
                 using (var httpClientHandler = HostContext.CreateHttpClientHandler())
                 using (var httpClient = new HttpClient(httpClientHandler))
                 {
@@ -621,7 +624,8 @@ namespace GitHub.Runner.Listener.Configuration
                     httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", base64EncodingToken);
                     httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
                     httpClient.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github.v3+json");
-
+                    try
+                    {
                         var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty));
 
                         if (response.IsSuccessStatusCode)
@@ -630,16 +634,32 @@ namespace GitHub.Runner.Listener.Configuration
                             var jsonResponse = await response.Content.ReadAsStringAsync();
                             return StringUtil.ConvertFromJson<GitHubRunnerRegisterToken>(jsonResponse);
                         }
+                        else if(response.StatusCode == System.Net.HttpStatusCode.NotFound)
+                        {
+                            // It doesn't make sense to retry in this case, so just stop
+                            break;
+                        }
                         else
                         {
                             _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
                             var errorResponse = await response.Content.ReadAsStringAsync();
                             _term.WriteError(errorResponse);
                             response.EnsureSuccessStatusCode();
+                        }
+                    }
+                    catch(Exception ex) when (retryCount < 2)
+                    {
+                        retryCount++;
+                        Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}");
+                        Trace.Error(ex);
+                        Trace.Info("Retrying in 5 seconds");
+                    }
+                }
+                var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5));
+                await Task.Delay(backOff);
+            }
             return null;
         }
-            }
-        }
 
         private async Task<GitHubAuthResult> GetTenantCredential(string githubUrl, string githubToken, string runnerEvent)
         {
@@ -654,6 +674,9 @@ namespace GitHub.Runner.Listener.Configuration
                 githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/actions/runner-registration";
             }
 
+            int retryCount = 0;
+            while (retryCount < 3)
+            {
                 using (var httpClientHandler = HostContext.CreateHttpClientHandler())
                 using (var httpClient = new HttpClient(httpClientHandler))
                 {
@@ -666,23 +689,42 @@ namespace GitHub.Runner.Listener.Configuration
                         {"runner_event", runnerEvent}
                     };
 
+                    try
+                    {
                         var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"));
 
-                if (response.IsSuccessStatusCode)
+                        if(response.IsSuccessStatusCode)
                         {
                             Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
                             var jsonResponse = await response.Content.ReadAsStringAsync();
                             return StringUtil.ConvertFromJson<GitHubAuthResult>(jsonResponse);
                         }
+                        else if(response.StatusCode == System.Net.HttpStatusCode.NotFound)
+                        {
+                            // It doesn't make sense to retry in this case, so just stop
+                            break;
+                        }
                         else
                         {
                             _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
                             var errorResponse = await response.Content.ReadAsStringAsync();
                             _term.WriteError(errorResponse);
+                            // Something else bad happened, let's go to our retry logic
                             response.EnsureSuccessStatusCode();
+                        }
+                    }
+                    catch(Exception ex) when (retryCount < 2)
+                    {
+                        retryCount++;
+                        Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}");
+                        Trace.Error(ex);
+                        Trace.Info("Retrying in 5 seconds");
+                    }
+                }
+                var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5));
+                await Task.Delay(backOff);
+            }
             return null;
         }
     }
-        }
-    }
 }