prototype support for step retries

* Delete script files before replacing during update

Signed-off-by: Tim Etchells <tetchel@gmail.com>

* Use IOUtil.DelteFile()

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

releaseNote.md

@@ -1,16 +1,11 @@
 ## Features
-  - Verify the Runner Hash during auto-upgrade before installing the new runner version (#967)
-  - Support download of runners from authenticated endpoints (#920)
-  - Enabled tty output in Docker Actions (#916)
-  - Added '--check' command to verify runner connectivity (#949)
+
 
 ## Bugs
-  - Fix usage of /dev/null and ping in run.sh (#968)
+  - Fixed an issue where docker containers failed to initialize (#977)
 
 ## Misc
-  - Updated the copy for various runner messages (#972)
-  - Added the runner's OS to telemetry (#939)
-  - Various other telemetry improvements (#935)
+
 
 ## Windows x64
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.

src/Runner.Common/HostContext.cs

@@ -84,6 +84,7 @@ namespace GitHub.Runner.Common
             this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscape);
             this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift1);
             this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift2);
+            this.SecretMasker.AddValueEncoder(ValueEncoders.CommandLineArgumentEscape);
             this.SecretMasker.AddValueEncoder(ValueEncoders.ExpressionStringEscape);
             this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape);
             this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape);

src/Runner.Listener/SelfUpdater.cs

@@ -225,7 +225,7 @@ namespace GitHub.Runner.Listener
                                 using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
                                 using (Stream result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
                                 {
-                                    //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k). 
+                                    //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
                                     await result.CopyToAsync(fs, 81920, downloadCts.Token);
                                     await fs.FlushAsync(downloadCts.Token);
                                 }
@@ -357,8 +357,13 @@ namespace GitHub.Runner.Listener
             Trace.Info($"Copy any remaining .sh/.cmd files into runner root.");
             foreach (FileInfo file in new DirectoryInfo(latestRunnerDirectory).GetFiles() ?? new FileInfo[0])
             {
-                // Copy and replace the file.
-                file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
+                string destination = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name);
+
+                // Removing the file instead of just trying to overwrite it works around permissions issues on linux.
+                // https://github.com/actions/runner/issues/981
+                Trace.Info($"Copy {file.FullName} to {destination}");
+                IOUtil.DeleteFile(destination);
+                file.CopyTo(destination, true);
             }
         }
 

src/Runner.Worker/ActionRunner.cs

@@ -64,6 +64,8 @@ namespace GitHub.Runner.Worker
 
         public Pipelines.ActionStep Action { get; set; }
 
+        public Int32 Retries => Action?.Retries ?? 0;
+
         public TemplateToken Timeout => Action?.TimeoutInMinutes;
 
         public async Task RunAsync()

src/Runner.Worker/Container/DockerCommandManager.cs

@@ -197,8 +197,7 @@ namespace GitHub.Runner.Worker.Container
 
             dockerOptions.Add($"--workdir {container.ContainerWorkDirectory}");
             dockerOptions.Add($"--rm");
-            dockerOptions.Add($"-t");
-            
+
             foreach (var env in container.ContainerEnvironmentVariables)
             {
                 // e.g. -e MY_SECRET maps the value into the exec'ed process without exposing

src/Runner.Worker/ContainerOperationProvider.cs

@@ -198,8 +198,7 @@ namespace GitHub.Runner.Worker
                 }
             }
 
-            // TODO: Add at a later date. This currently no local package registry to test with
-            // UpdateRegistryAuthForGitHubToken(executionContext, container);
+            UpdateRegistryAuthForGitHubToken(executionContext, container);
 
             // Before pulling, generate client authentication if required
             var configLocation = await ContainerRegistryLogin(executionContext, container);
@@ -494,31 +493,14 @@ namespace GitHub.Runner.Worker
 
         private void UpdateRegistryAuthForGitHubToken(IExecutionContext executionContext, ContainerInfo container)
         {
-            var registryIsTokenCompatible = container.RegistryServer.Equals("docker.pkg.github.com", StringComparison.OrdinalIgnoreCase);
+            var registryIsTokenCompatible = container.RegistryServer.Equals("ghcr.io", StringComparison.OrdinalIgnoreCase) || container.RegistryServer.Equals("containers.pkg.github.com", StringComparison.OrdinalIgnoreCase);
             if (!registryIsTokenCompatible)
             {
                 return;
             }
 
-            var registryMatchesWorkflow = false;
-
-            // REGISTRY/OWNER/REPO/IMAGE[:TAG]
-            var imageParts = container.ContainerImage.Split('/');
-            if (imageParts.Length != 4)
-            {
-                executionContext.Warning($"Could not identify owner and repo for container image {container.ContainerImage}. Skipping automatic token auth");
-                return;
-            }
-            var owner = imageParts[1];
-            var repo = imageParts[2];
-            var nwo = $"{owner}/{repo}";
-            if (nwo.Equals(executionContext.GetGitHubContext("repository"), StringComparison.OrdinalIgnoreCase))
-            {
-                registryMatchesWorkflow = true;
-            }
-
             var registryCredentialsNotSupplied = string.IsNullOrEmpty(container.RegistryAuthUsername) && string.IsNullOrEmpty(container.RegistryAuthPassword);
-            if (registryCredentialsNotSupplied && registryMatchesWorkflow)
+            if (registryCredentialsNotSupplied)
             {
                 container.RegistryAuthUsername = executionContext.GetGitHubContext("actor");
                 container.RegistryAuthPassword = executionContext.GetGitHubContext("token");

src/Runner.Worker/Handlers/StepHost.cs

@@ -182,7 +182,7 @@ namespace GitHub.Runner.Worker.Handlers
             dockerCommandArgs.Add($"exec");
 
             // [OPTIONS]
-            dockerCommandArgs.Add($"-it");
+            dockerCommandArgs.Add($"-i");
             dockerCommandArgs.Add($"--workdir {workingDirectory}");
             foreach (var env in environment)
             {

src/Runner.Worker/JobExtensionRunner.cs

@@ -26,6 +26,7 @@ namespace GitHub.Runner.Worker
         public TemplateToken ContinueOnError => new BooleanToken(null, null, null, false);
         public string DisplayName { get; set; }
         public IExecutionContext ExecutionContext { get; set; }
+        public Int32 Retries => 0;
         public TemplateToken Timeout => new NumberToken(null, null, null, 0);
         public object Data => _data;
 

src/Runner.Worker/StepsRunner.cs

@@ -24,6 +24,7 @@ namespace GitHub.Runner.Worker
         TemplateToken ContinueOnError { get; }
         string DisplayName { get; set; }
         IExecutionContext ExecutionContext { get; set; }
+        Int32 Retries { get; }
         TemplateToken Timeout { get; }
         Task RunAsync();
     }
@@ -280,73 +281,96 @@ namespace GitHub.Runner.Worker
                 step.ExecutionContext.Error("An error occurred when attempting to determine the step timeout.");
                 step.ExecutionContext.Error(ex);
             }
-            if (timeoutMinutes > 0)
-            {
-                var timeout = TimeSpan.FromMinutes(timeoutMinutes);
-                step.ExecutionContext.SetTimeout(timeout);
-            }
 
-            await EncodingUtil.SetEncoding(HostContext, Trace, step.ExecutionContext.CancellationToken);
-
-            try
+            int attempt = 1;
+            while (true)
             {
-                await step.RunAsync();
-            }
-            catch (OperationCanceledException ex)
-            {
-                if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
-                    !jobCancellationToken.IsCancellationRequested)
+                if (timeoutMinutes > 0)
                 {
-                    Trace.Error($"Caught timeout exception from step: {ex.Message}");
-                    step.ExecutionContext.Error("The action has timed out.");
-                    step.ExecutionContext.Result = TaskResult.Failed;
+                    var timeout = TimeSpan.FromMinutes(timeoutMinutes);
+                    step.ExecutionContext.SetTimeout(timeout);
                 }
-                else
-                {
-                    // Log the exception and cancel the step.
-                    Trace.Error($"Caught cancellation exception from step: {ex}");
-                    step.ExecutionContext.Error(ex);
-                    step.ExecutionContext.Result = TaskResult.Canceled;
-                }
-            }
-            catch (Exception ex)
-            {
-                // Log the error and fail the step.
-                Trace.Error($"Caught exception from step: {ex}");
-                step.ExecutionContext.Error(ex);
-                step.ExecutionContext.Result = TaskResult.Failed;
-            }
 
-            // Merge execution context result with command result
-            if (step.ExecutionContext.CommandResult != null)
-            {
-                step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
-            }
+                await EncodingUtil.SetEncoding(HostContext, Trace, step.ExecutionContext.CancellationToken);
 
-            // Fixup the step result if ContinueOnError.
-            if (step.ExecutionContext.Result == TaskResult.Failed)
-            {
-                var continueOnError = false;
                 try
                 {
-                    continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
+                    await step.RunAsync();
+                }
+                catch (OperationCanceledException ex)
+                {
+                    if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
+                        !jobCancellationToken.IsCancellationRequested)
+                    {
+                        Trace.Error($"Caught timeout exception from step: {ex.Message}");
+                        step.ExecutionContext.Error("The action has timed out.");
+                        step.ExecutionContext.Result = TaskResult.Failed;
+                    }
+                    else
+                    {
+                        // Log the exception and cancel the step.
+                        Trace.Error($"Caught cancellation exception from step: {ex}");
+                        step.ExecutionContext.Error(ex);
+                        step.ExecutionContext.Result = TaskResult.Canceled;
+                    }
                 }
                 catch (Exception ex)
                 {
-                    Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
-                    Trace.Error(ex);
-                    step.ExecutionContext.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                    // Log the error and fail the step.
+                    Trace.Error($"Caught exception from step: {ex}");
                     step.ExecutionContext.Error(ex);
+                    step.ExecutionContext.Result = TaskResult.Failed;
                 }
 
-                if (continueOnError)
+                // Merge execution context result with command result
+                if (step.ExecutionContext.CommandResult != null)
                 {
-                    step.ExecutionContext.Outcome = step.ExecutionContext.Result;
-                    step.ExecutionContext.Result = TaskResult.Succeeded;
-                    Trace.Info($"Updated step result (continue on error)");
+                    step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
                 }
+
+                // Fixup the step result if ContinueOnError.
+                if (step.ExecutionContext.Result == TaskResult.Failed)
+                {
+                    var continueOnError = false;
+                    try
+                    {
+                        continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
+                    }
+                    catch (Exception ex)
+                    {
+                        Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                        Trace.Error(ex);
+                        step.ExecutionContext.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                        step.ExecutionContext.Error(ex);
+                    }
+
+                    if (continueOnError)
+                    {
+                        step.ExecutionContext.Outcome = step.ExecutionContext.Result;
+                        step.ExecutionContext.Result = TaskResult.Succeeded;
+                        Trace.Info($"Updated step result (continue on error)");
+                    }
+                }
+                Trace.Info($"Step result: {step.ExecutionContext.Result}");
+
+                if (step.ExecutionContext.Result == TaskResult.Failed && attempt <= step.Retries)
+                {
+                    attempt++;
+                    step.ExecutionContext.Result = null;
+                    step.ExecutionContext.ResultCode = null;
+                    // todo: replace the step cancellation token source
+                    // todo: reset the step.ExecutionContext.CommandResult
+                    // todo: create a new timeline record, e.g. "My display name (#2)"
+                    // todo: clear outputs? What will we do on a job? probably clear outputs since merging from separate timeline attempts would otherwise be complex
+                    // todo: consider intrastate - i guess it makes sense this doesn't get cleared
+                    // todo: reconcile all of the above wrt composite steps
+                    // todo: reconcile all of the above wrt pre/post
+                    // todo: distinguish retryable vs non-retryable failures? e.g. if an exception bubbles from the handler
+                    continue;
+                }
+
+                break;
             }
-            Trace.Info($"Step result: {step.ExecutionContext.Result}");
 
             // Complete the step context.
             step.ExecutionContext.Debug($"Finishing: {step.DisplayName}");

src/Sdk/DTLogging/Logging/ValueEncoders.cs

@@ -37,6 +37,12 @@ namespace GitHub.DistributedTask.Logging
             return Base64StringEscapeShift(value, 2);
         }
 
+        // Used when we pass environment variables to docker to escape " with \"
+        public static String CommandLineArgumentEscape(String value)
+        {
+            return value.Replace("\"", "\\\"");
+        }
+
         public static String ExpressionStringEscape(String value)
         {
             return Expressions2.Sdk.ExpressionUtility.StringEscape(value);

src/Sdk/DTPipelines/Pipelines/ActionStep.cs

@@ -22,6 +22,7 @@ namespace GitHub.DistributedTask.Pipelines
             this.Reference = actionToClone.Reference?.Clone();
 
             Environment = actionToClone.Environment?.Clone();
+            Retries = actionToClone.Retries;
             Inputs = actionToClone.Inputs?.Clone();
             ContextName = actionToClone?.ContextName;
             DisplayNameToken = actionToClone.DisplayNameToken?.Clone();
@@ -46,6 +47,9 @@ namespace GitHub.DistributedTask.Pipelines
         [DataMember(EmitDefaultValue = false)]
         public TemplateToken Environment { get; set; }
 
+        [DataMember(EmitDefaultValue = false)]
+        public Int32 Retries { get; set; }
+
         [DataMember(EmitDefaultValue = false)]
         public TemplateToken Inputs { get; set; }
 

src/runnerversion

@@ -1 +1 @@
-2.277.0
+2.277.1