From 6603bfb74c2e040f9ec64df237fc0384130b84b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Enes=20=C3=87ak=C4=B1r?= <enes@cakir.web.tr>
Date: Wed, 21 Feb 2024 20:32:37 +0300
Subject: [PATCH] Add a retry logic to docker login operation (#3089)

While there's an existing retry mechanism for the `docker pull` command
[^1], it's missing for `docker login`.

Similar to the `docker pull` scenario, the container registry could
potentially be briefly unavailable or inaccessible, leading to failed
`docker login` attempt and subsequent workflow run failures.

Since it's container based workflow, there is not way to retry on
customer side. The runner should retry itself.

It also aligns with community feedback [^2].

[^1]: https://github.com/actions/runner/blob/8e0cd36cd8c74c3067ffe10189c1e42f7e753af2/src/Runner.Worker/ContainerOperationProvider.cs#L201
[^2]: https://github.com/orgs/community/discussions/73069

Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com>
---
 .../ContainerOperationProvider.cs             | 36 +++++++++++++++----
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs
index 6e5b12047..c5cccb77e 100644
--- a/src/Runner.Worker/ContainerOperationProvider.cs
+++ b/src/Runner.Worker/ContainerOperationProvider.cs
@@ -466,17 +466,39 @@ namespace GitHub.Runner.Worker
             {
                 throw new InvalidOperationException($"Failed to create directory to store registry client credentials: {e.Message}");
             }
-            var loginExitCode = await _dockerManager.DockerLogin(
-                executionContext,
-                configLocation,
-                container.RegistryServer,
-                container.RegistryAuthUsername,
-                container.RegistryAuthPassword);
 
-            if (loginExitCode != 0)
+            // Login docker with retry up to 3 times
+            int retryCount = 0;
+            int loginExitCode = 0;
+            while (retryCount < 3)
+            {
+                loginExitCode = await _dockerManager.DockerLogin(
+                    executionContext,
+                    configLocation,
+                    container.RegistryServer,
+                    container.RegistryAuthUsername,
+                    container.RegistryAuthPassword);
+                if (loginExitCode == 0)
+                {
+                    break;
+                }
+                else
+                {
+                    retryCount++;
+                    if (retryCount < 3)
+                    {
+                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
+                        executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
+                        await Task.Delay(backOff);
+                    }
+                }
+            }
+
+            if (retryCount == 3 && loginExitCode != 0)
             {
                 throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
             }
+
             return configLocation;
         }