diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs
index 6dfa72c05..ceda3dbb3 100644
--- a/src/Runner.Worker/ActionManager.cs
+++ b/src/Runner.Worker/ActionManager.cs
@@ -594,7 +594,7 @@ namespace GitHub.Runner.Worker
                     actionDownloadInfos = await jobServer.ResolveActionDownloadInfoAsync(executionContext.Global.Plan.ScopeIdentifier, executionContext.Global.Plan.PlanType, executionContext.Global.Plan.PlanId, new WebApi.ActionReferenceList { Actions = actionReferences }, executionContext.CancellationToken);
                     break;
                 }
-                catch (Exception ex)
+                catch (Exception ex) when (!executionContext.CancellationToken.IsCancellationRequested) // Do not retry if the run is canceled.
                 {
                     if (attempt < 3)
                     {
@@ -609,7 +609,18 @@ namespace GitHub.Runner.Worker
                     }
                     else
                     {
-                        throw new WebApi.FailedToResolveActionDownloadInfoException("Failed to resolve action download info.", ex);
+                        // Some possible cases are:
+                        // * Repo is rate limited
+                        // * Repo or tag doesn't exist, or isn't public
+                        if (ex is WebApi.UnresolvableActionDownloadInfoException)
+                        {
+                            throw;
+                        }
+                        else
+                        {
+                            // This exception will be traced as an infrastructure failure
+                            throw new WebApi.FailedToResolveActionDownloadInfoException("Failed to resolve action download info.", ex);
+                        }
                     }
                 }
             }
diff --git a/src/Sdk/DTWebApi/WebApi/Exceptions.cs b/src/Sdk/DTWebApi/WebApi/Exceptions.cs
index 9e2435f5c..3690bbe6b 100644
--- a/src/Sdk/DTWebApi/WebApi/Exceptions.cs
+++ b/src/Sdk/DTWebApi/WebApi/Exceptions.cs
@@ -2459,6 +2459,25 @@ namespace GitHub.DistributedTask.WebApi
         }
     }
 
+    [Serializable]
+    public class UnresolvableActionDownloadInfoException : DistributedTaskException
+    {
+        public UnresolvableActionDownloadInfoException(String message)
+            : base(message)
+        {
+        }
+
+        public UnresolvableActionDownloadInfoException(String message, Exception innerException)
+            : base(message, innerException)
+        {
+        }
+
+        protected UnresolvableActionDownloadInfoException(SerializationInfo info, StreamingContext context)
+            : base(info, context)
+        {
+        }
+    }
+
     [Serializable]
     public sealed class FailedToResolveActionDownloadInfoException : DistributedTaskException
     {