adding compile-time const whether to use the broker service

- github/c2c-actions-support#883

releaseNote.md

@@ -1,11 +1,7 @@
 ## Features
 
 ## Bugs
-- Fixed a crash on runner startup (#1770)
-
-## Misc
-
-- Clarified the type of step running when running job started or completed hooks (#1769)
+- Fixed an issue where websockets failed to successfully close when posting log lines (#1790)
 
 
 ## Windows x64

src/Directory.Build.props

@@ -47,6 +47,11 @@
     <DefineConstants>$(DefineConstants);DEBUG</DefineConstants>
   </PropertyGroup>
 
+  <!-- Set USE_BROKER vars -->
+  <PropertyGroup Condition="'$(USE_BROKER)' == 'true'">
+    <DefineConstants>$(DefineConstants);USE_BROKER</DefineConstants>
+  </PropertyGroup>
+
   <!-- Set Treat tarnings as errors -->
   <PropertyGroup>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>

src/Misc/expressionFunc/hashFiles/package-lock.json

@@ -1823,9 +1823,9 @@
       }
     },
     "node_modules/minimist": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
       "dev": true
     },
     "node_modules/ms": {
@@ -3808,9 +3808,9 @@
       }
     },
     "minimist": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==",
       "dev": true
     },
     "ms": {

src/Runner.Common/BrokerServer.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common.Util;
+using GitHub.Services.WebApi;
+using GitHub.Services.Common;
+using GitHub.Runner.Sdk;
+using System.Net.Http;
+
+namespace GitHub.Runner.Common
+{
+    [ServiceLocator(Default = typeof(BrokerServer))]
+    public interface IBrokerServer : IRunnerService
+    {
+        Task ConnectAsync(Uri serverUrl);
+        Task<GitHub.DistributedTask.WebApi.TaskAgentMessage> GetMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken);
+    }
+
+    public sealed class BrokerServer : RunnerService, IBrokerServer
+    {
+        private HttpClient _httpClient;
+
+        public async Task ConnectAsync(Uri serverUrl)
+        {
+            _httpClient = new HttpClient();
+            _httpClient.BaseAddress = serverUrl;
+            _httpClient.Timeout = TimeSpan.FromSeconds(50);
+            await _httpClient.GetAsync("health");
+        }
+
+        public async Task<GitHub.DistributedTask.WebApi.TaskAgentMessage> GetMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken)
+        {
+            await _httpClient.GetAsync("message");
+            return null;
+        }
+    }
+}

src/Runner.Common/Constants.cs

@@ -149,6 +149,7 @@ namespace GitHub.Runner.Common
             public static class Features
             {
                 public static readonly string DiskSpaceWarning = "runner.diskspace.warning";
+                public static readonly string Node12Warning = "DistributedTask.AddWarningToNode12Action";
             }
 
             public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry";
@@ -158,6 +159,7 @@ namespace GitHub.Runner.Common
             public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/";
             public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`.";
             public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
+            public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: {0}";
         }
 
         public static class RunnerEvent

src/Runner.Common/JobServer.cs

@@ -2,7 +2,9 @@ using GitHub.DistributedTask.WebApi;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Net.WebSockets;
 using System.Text;
 using System.Threading;
@@ -10,6 +12,7 @@ using System.Threading.Tasks;
 using GitHub.Runner.Sdk;
 using GitHub.Services.Common;
 using GitHub.Services.WebApi;
+using GitHub.Services.WebApi.Utilities.Internal;
 using Newtonsoft.Json;
 
 namespace GitHub.Runner.Common
@@ -143,8 +146,10 @@ namespace GitHub.Runner.Common
 
         public ValueTask DisposeAsync()
         {
-            _websocketClient?.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "Shutdown", CancellationToken.None);
+            CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);
+
             GC.SuppressFinalize(this);
+
             return ValueTask.CompletedTask;
         }
 
@@ -169,6 +174,11 @@ namespace GitHub.Runner.Common
                     Trace.Info($"Creating websocket client ..." + feedStreamUrl);
                     this._websocketClient = new ClientWebSocket();
                     this._websocketClient.Options.SetRequestHeader("Authorization", $"Bearer {accessToken}");
+                    var userAgentValues = new List<ProductInfoHeaderValue>();
+                    userAgentValues.AddRange(UserAgentUtility.GetDefaultRestUserAgent());
+                    userAgentValues.AddRange(HostContext.UserAgents);
+                    this._websocketClient.Options.SetRequestHeader("User-Agent", string.Join(" ", userAgentValues.Select(x=>x.ToString())));
+
                     this._websocketConnectTask = ConnectWebSocketClient(feedStreamUrl, delay);
                 }
                 else
@@ -248,7 +258,8 @@ namespace GitHub.Runner.Common
                         if (failedAttemptsToPostBatchedLinesByWebsocket * 100 / totalBatchedLinesAttemptedByWebsocket > _minWebsocketFailurePercentageAllowed)
                         {
                             Trace.Info($"Exhausted websocket allowed retries, we will not attempt websocket connection for this job to post lines again.");
-                            _websocketClient?.CloseOutputAsync(WebSocketCloseStatus.InternalServerError, "Shutdown due to failures", cancellationToken);
+                            CloseWebSocket(WebSocketCloseStatus.InternalServerError, cancellationToken);
+
                             // By setting it to null, we will ensure that we never try websocket path again for this job
                             _websocketClient = null;
                         }
@@ -276,6 +287,19 @@ namespace GitHub.Runner.Common
             }
         }
 
+        private void CloseWebSocket(WebSocketCloseStatus closeStatus, CancellationToken cancellationToken)
+        {
+            try
+            {
+                _websocketClient?.CloseOutputAsync(closeStatus, "Closing websocket", cancellationToken);
+            }
+            catch (Exception websocketEx)
+            {
+                // In some cases this might be okay since the websocket might be open yet, so just close and don't trace exceptions
+                Trace.Info($"Failed to close websocket gracefully {websocketEx.GetType().Name}");
+            }
+        }
+
         public Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, string type, string name, Stream uploadStream, CancellationToken cancellationToken)
         {
             CheckConnection();

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -624,9 +624,12 @@ namespace GitHub.Runner.Listener.Configuration
                     httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", base64EncodingToken);
                     httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
                     httpClient.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github.v3+json");
+                    
+                    var responseStatus = System.Net.HttpStatusCode.OK;
                     try
                     {
                         var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty));
+                        responseStatus = response.StatusCode;
 
                         if (response.IsSuccessStatusCode)
                         {
@@ -634,11 +637,6 @@ namespace GitHub.Runner.Listener.Configuration
                             var jsonResponse = await response.Content.ReadAsStringAsync();
                             return StringUtil.ConvertFromJson<GitHubRunnerRegisterToken>(jsonResponse);
                         }
-                        else if(response.StatusCode == System.Net.HttpStatusCode.NotFound)
-                        {
-                            // It doesn't make sense to retry in this case, so just stop
-                            break;
-                        }
                         else
                         {
                             _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
@@ -647,15 +645,15 @@ namespace GitHub.Runner.Listener.Configuration
                             response.EnsureSuccessStatusCode();
                         }
                     }
-                    catch(Exception ex) when (retryCount < 2)
+                    catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
                     {
                         retryCount++;
                         Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}");
                         Trace.Error(ex);
-                        Trace.Info("Retrying in 5 seconds");
                     }
                 }
                 var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5));
+                Trace.Info($"Retrying in {backOff.Seconds} seconds");
                 await Task.Delay(backOff);
             }
             return null;
@@ -689,9 +687,11 @@ namespace GitHub.Runner.Listener.Configuration
                         {"runner_event", runnerEvent}
                     };
 
+                    var responseStatus = System.Net.HttpStatusCode.OK;
                     try
                     {
                         var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"));
+                        responseStatus = response.StatusCode;
 
                         if(response.IsSuccessStatusCode)
                         {
@@ -699,29 +699,23 @@ namespace GitHub.Runner.Listener.Configuration
                             var jsonResponse = await response.Content.ReadAsStringAsync();
                             return StringUtil.ConvertFromJson<GitHubAuthResult>(jsonResponse);
                         }
-                        else if(response.StatusCode == System.Net.HttpStatusCode.NotFound)
-                        {
-                            // It doesn't make sense to retry in this case, so just stop
-                            break;
-                        }
                         else
                         {
                             _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
                             var errorResponse = await response.Content.ReadAsStringAsync();
                             _term.WriteError(errorResponse);
-                            // Something else bad happened, let's go to our retry logic
                             response.EnsureSuccessStatusCode();
                         }
                     }
-                    catch(Exception ex) when (retryCount < 2)
+                    catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
                     {
                         retryCount++;
                         Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}");
                         Trace.Error(ex);
-                        Trace.Info("Retrying in 5 seconds");
                     }
                 }
                 var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5));
+                Trace.Info($"Retrying in {backOff.Seconds} seconds");
                 await Task.Delay(backOff);
             }
             return null;

src/Runner.Listener/Configuration/CredentialManager.cs

@@ -40,6 +40,12 @@ namespace GitHub.Runner.Listener.Configuration
             return creds;
         }
 
+#if USE_BROKER
+        public VssCredentials LoadCredentials()
+        {
+            return new VssCredentials();
+        }
+#else
         public VssCredentials LoadCredentials()
         {
             IConfigurationStore store = HostContext.GetService<IConfigurationStore>();
@@ -69,6 +75,7 @@ namespace GitHub.Runner.Listener.Configuration
 
             return creds;
         }
+#endif
     }
 
     [DataContract]

src/Runner.Listener/MessageListener.cs

@@ -31,6 +31,7 @@ namespace GitHub.Runner.Listener
         private RunnerSettings _settings;
         private ITerminal _term;
         private IRunnerServer _runnerServer;
+        private IBrokerServer _brokerServer;
         private TaskAgentSession _session;
         private TimeSpan _getNextMessageRetryInterval;
         private bool _accessTokenRevoked = false;
@@ -45,8 +46,44 @@ namespace GitHub.Runner.Listener
 
             _term = HostContext.GetService<ITerminal>();
             _runnerServer = HostContext.GetService<IRunnerServer>();
+            _brokerServer = HostContext.GetService<IBrokerServer>();
         }
 
+#if USE_BROKER
+        public async Task<Boolean> CreateSessionAsync(CancellationToken token)
+        {
+            Trace.Entering();
+
+            // Settings
+            var configManager = HostContext.GetService<IConfigurationManager>();
+            _settings = configManager.LoadSettings();
+            var serverUrl = _settings.ServerUrl;
+            Trace.Info(_settings);
+
+            // Connect
+            token.ThrowIfCancellationRequested();
+            Trace.Info($"Attempt to create session.");
+            Trace.Info("Connecting to the Runner Server...");
+            _term.WriteLine($"Connecting to {new Uri(serverUrl)}");
+            await _brokerServer.ConnectAsync(new Uri(serverUrl));
+            _term.WriteLine();
+            _term.WriteSuccessMessage("Connected to GitHub");
+            _term.WriteLine();
+
+            // Session info
+            var agent = new TaskAgentReference
+            {
+                Id = _settings.AgentId,
+                Name = _settings.AgentName,
+                Version = BuildConstants.RunnerPackage.Version,
+                OSDescription = RuntimeInformation.OSDescription,
+            };
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
+            _session = new TaskAgentSession(sessionName, agent);
+
+            return true;
+        }
+#else
         public async Task<Boolean> CreateSessionAsync(CancellationToken token)
         {
             Trace.Entering();
@@ -151,6 +188,7 @@ namespace GitHub.Runner.Listener
                 }
             }
         }
+#endif
 
         public async Task DeleteSessionAsync()
         {
@@ -170,6 +208,116 @@ namespace GitHub.Runner.Listener
             }
         }
 
+#if USE_BROKER
+        public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
+        {
+            Trace.Entering();
+            ArgUtil.NotNull(_session, nameof(_session));
+            ArgUtil.NotNull(_settings, nameof(_settings));
+            bool encounteringError = false;
+            int continuousError = 0;
+            string errorMessage = string.Empty;
+            Stopwatch heartbeat = new Stopwatch();
+            heartbeat.Restart();
+            while (true)
+            {
+                token.ThrowIfCancellationRequested();
+                TaskAgentMessage message = null;
+                try
+                {
+                    message = await _brokerServer.GetMessageAsync(_settings.PoolId, _session.SessionId, _lastMessageId, token);
+
+                    // Decrypt the message body if the session is using encryption
+                    message = DecryptMessage(message);
+
+                    if (message != null)
+                    {
+                        _lastMessageId = message.MessageId;
+                    }
+
+                    if (encounteringError) //print the message once only if there was an error
+                    {
+                        _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
+                        encounteringError = false;
+                        continuousError = 0;
+                    }
+                }
+                catch (OperationCanceledException) when (token.IsCancellationRequested)
+                {
+                    Trace.Info("Get next message has been cancelled.");
+                    throw;
+                }
+                catch (TaskAgentAccessTokenExpiredException)
+                {
+                    Trace.Info("Runner OAuth token has been revoked. Unable to pull message.");
+                    _accessTokenRevoked = true;
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    Trace.Error("Catch exception during get next message.");
+                    Trace.Error(ex);
+
+                    // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
+                    if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token))
+                    {
+                        Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session.");
+                    }
+                    else if (!IsGetNextMessageExceptionRetriable(ex))
+                    {
+                        throw;
+                    }
+                    else
+                    {
+                        continuousError++;
+                        //retry after a random backoff to avoid service throttling
+                        //in case of there is a service error happened and all agents get kicked off of the long poll and all agent try to reconnect back at the same time.
+                        if (continuousError <= 5)
+                        {
+                            // random backoff [15, 30]
+                            _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
+                        }
+                        else
+                        {
+                            // more aggressive backoff [30, 60]
+                            _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60), _getNextMessageRetryInterval);
+                        }
+
+                        if (!encounteringError)
+                        {
+                            //print error only on the first consecutive error
+                            _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
+                            encounteringError = true;
+                        }
+
+                        // re-create VssConnection before next retry
+                        await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+
+                        Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                        await HostContext.Delay(_getNextMessageRetryInterval, token);
+                    }
+                }
+
+                if (message == null)
+                {
+                    if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
+                    {
+                        Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
+                        heartbeat.Restart();
+                    }
+                    else
+                    {
+                        Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
+                    }
+
+                    continue;
+                }
+
+                Trace.Info($"Message '{message.MessageId}' received from session '{_session.SessionId}'.");
+                return message;
+            }
+        }
+#else
         public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
         {
             Trace.Entering();
@@ -281,6 +429,7 @@ namespace GitHub.Runner.Listener
                 return message;
             }
         }
+#endif
 
         public async Task DeleteMessageAsync(TaskAgentMessage message)
         {

src/Runner.Worker/Container/DockerCommandManager.cs

@@ -11,7 +11,7 @@ using GitHub.Runner.Sdk;
 
 namespace GitHub.Runner.Worker.Container
 {
-    [ServiceLocator(Default = typeof(DockerHookCommandManager))]
+    [ServiceLocator(Default = typeof(DockerCommandManager))]
     public interface IDockerCommandManager : IRunnerService
     {
         string DockerPath { get; }
@@ -188,7 +188,7 @@ namespace GitHub.Runner.Worker.Container
             return outputStrings.FirstOrDefault();
         }
 
-        public virtual async Task<int> DockerRun(IExecutionContext context, ContainerInfo container, EventHandler<ProcessDataReceivedEventArgs> stdoutDataReceived, EventHandler<ProcessDataReceivedEventArgs> stderrDataReceived)
+        public async Task<int> DockerRun(IExecutionContext context, ContainerInfo container, EventHandler<ProcessDataReceivedEventArgs> stdoutDataReceived, EventHandler<ProcessDataReceivedEventArgs> stderrDataReceived)
         {
             IList<string> dockerOptions = new List<string>();
             // OPTIONS
@@ -258,7 +258,7 @@ namespace GitHub.Runner.Worker.Container
             return await ExecuteDockerCommandAsync(context, "run", optionsString, container.ContainerEnvironmentVariables, stdoutDataReceived, stderrDataReceived, context.CancellationToken);
         }
 
-        public virtual async Task<int> DockerStart(IExecutionContext context, string containerId)
+        public async Task<int> DockerStart(IExecutionContext context, string containerId)
         {
             return await ExecuteDockerCommandAsync(context, "start", containerId, context.CancellationToken);
         }

src/Runner.Worker/Container/DockerHookCommandManager.cs

@@ -1,50 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
-using GitHub.DistributedTask.Pipelines;
-using GitHub.Runner.Common.Util;
-using GitHub.Runner.Worker.Handlers;
-
-namespace GitHub.Runner.Worker.Container
-{
-    public class DockerHookCommandManager : DockerCommandManager
-    {
-        public override async Task<int> DockerStart(IExecutionContext context, string containerId)
-        {
-            // check for env var
-            // execute script
-            
-            // Create the handler data.
-            var path = "/home/ferenc/Documents/runner/_layout/docker_run.sh";
-            var scriptDirectory = Path.GetDirectoryName(path);
-            var stepHost = HostContext.CreateService<IDefaultStepHost>();
-            var prependPath = string.Join(Path.PathSeparator.ToString(), context.Global.PrependPath.Reverse<string>());
-            Dictionary<string, string> inputs = new()
-            {
-                ["script"] = $"CONT_ID={containerId} " + "/usr/bin/bash" + " " + path,
-                // /bin/bash
-                ["shell"] = ScriptHandlerHelpers.GetDefaultShellForScript(path, Trace, prependPath)
-            };
-
-            // Create the handler
-            var handlerFactory = HostContext.GetService<IHandlerFactory>();
-            var handler = handlerFactory.Create(
-                            context,
-                            action: new ScriptReference(),
-                            stepHost,
-                            new ScriptActionExecutionData(),
-                            inputs,
-                            environment: new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer),
-                            context.Global.Variables,
-                            actionDirectory: scriptDirectory,
-                            localActionContainerSetupSteps: null);
-            handler.PrepareExecution(ActionRunStage.Main); // TODO: find out stage
-
-            await handler.RunAsync(ActionRunStage.Main);
-
-            return ((int?) handler.ExecutionContext.CommandResult) ?? 0;
-        }
-    }
-}

src/Runner.Worker/ExecutionContext.cs

@@ -11,6 +11,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
 using GitHub.DistributedTask.Expressions2;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
 using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.Pipelines.ContextData;
 using GitHub.DistributedTask.Pipelines.ObjectTemplating;
@@ -109,7 +110,12 @@ namespace GitHub.Runner.Worker
         void ForceTaskComplete();
         void RegisterPostJobStep(IStep step);
         void PublishStepTelemetry();
+
+        void ApplyContinueOnError(TemplateToken continueOnError);
+        void UpdateGlobalStepsContext();
+
         void WriteWebhookPayload();
+
     }
 
     public sealed class ExecutionContext : RunnerService, IExecutionContext
@@ -439,14 +445,19 @@ namespace GitHub.Runner.Worker
 
             _logger.End();
 
+            UpdateGlobalStepsContext();
+
+            return Result.Value;
+        }
+
+        public void UpdateGlobalStepsContext()
+        {
             // Skip if generated context name. Generated context names start with "__". After 3.2 the server will never send an empty context name.
             if (!string.IsNullOrEmpty(ContextName) && !ContextName.StartsWith("__", StringComparison.Ordinal))
             {
                 Global.StepsContext.SetOutcome(ScopeName, ContextName, (Outcome ?? Result ?? TaskResult.Succeeded).ToActionResult());
                 Global.StepsContext.SetConclusion(ScopeName, ContextName, (Result ?? TaskResult.Succeeded).ToActionResult());
             }
-
-            return Result.Value;
         }
 
         public void SetRunnerContext(string name, string value)
@@ -1064,6 +1075,36 @@ namespace GitHub.Runner.Worker
             var newGuid = Guid.NewGuid();
             return CreateChild(newGuid, displayName, newGuid.ToString("N"), null, null, ActionRunStage.Post, intraActionState, _childTimelineRecordOrder - Root.PostJobSteps.Count, siblingScopeName: siblingScopeName);
         }
+
+        public void ApplyContinueOnError(TemplateToken continueOnErrorToken)
+        {
+            if (Result != TaskResult.Failed)
+            {
+                return;
+            }
+            var continueOnError = false;
+            try
+            {
+                var templateEvaluator = this.ToPipelineTemplateEvaluator();
+                continueOnError = templateEvaluator.EvaluateStepContinueOnError(continueOnErrorToken, ExpressionValues, ExpressionFunctions);
+            }
+            catch (Exception ex)
+            {
+                Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                Trace.Error(ex);
+                this.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                this.Error(ex);
+            }
+
+            if (continueOnError)
+            {
+                Outcome = Result;
+                Result = TaskResult.Succeeded;
+                Trace.Info($"Updated step result (continue on error)");
+            }
+
+            UpdateGlobalStepsContext();
+        }
     }
 
     // The Error/Warning/etc methods are created as extension methods to simplify unit testing.
@@ -1085,7 +1126,6 @@ namespace GitHub.Runner.Worker
             context.Error(ex.Message);
             context.Debug(ex.ToString());
         }
-
         // Do not add a format string overload. See comment on ExecutionContext.Write().
         public static void Error(this IExecutionContext context, string message)
         {

src/Runner.Worker/Handlers/CompositeActionHandler.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.Expressions2;
@@ -13,7 +11,6 @@ using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
-using GitHub.Runner.Worker;
 using GitHub.Runner.Worker.Expressions;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
@@ -86,7 +83,7 @@ namespace GitHub.Runner.Worker.Handlers
 
                 ExecutionContext.StepTelemetry.HasPreStep = Data.HasPre;
                 ExecutionContext.StepTelemetry.HasPostStep = Data.HasPost;
-                
+
                 ExecutionContext.StepTelemetry.HasRunsStep = hasRunsStep;
                 ExecutionContext.StepTelemetry.HasUsesStep = hasUsesStep;
                 ExecutionContext.StepTelemetry.StepCount = steps.Count;
@@ -407,7 +404,7 @@ namespace GitHub.Runner.Worker.Handlers
                 }
 
                 // Update context
-                SetStepsContext(step);
+                step.ExecutionContext.UpdateGlobalStepsContext();
             }
         }
 
@@ -452,6 +449,8 @@ namespace GitHub.Runner.Worker.Handlers
                 SetStepConclusion(step, Common.Util.TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value));
             }
 
+            step.ExecutionContext.ApplyContinueOnError(step.ContinueOnError);
+
             Trace.Info($"Step result: {step.ExecutionContext.Result}");
             step.ExecutionContext.Debug($"Finished: {step.DisplayName}");
             step.ExecutionContext.PublishStepTelemetry();
@@ -460,16 +459,7 @@ namespace GitHub.Runner.Worker.Handlers
         private void SetStepConclusion(IStep step, TaskResult result)
         {
             step.ExecutionContext.Result = result;
-            SetStepsContext(step);
-        }
-        private void SetStepsContext(IStep step)
-        {
-            if (!string.IsNullOrEmpty(step.ExecutionContext.ContextName) && !step.ExecutionContext.ContextName.StartsWith("__", StringComparison.Ordinal))
-            {
-                // TODO: when we support continue on error, we may need to do logic here to change conclusion based on the continue on error result
-                step.ExecutionContext.Global.StepsContext.SetOutcome(step.ExecutionContext.ScopeName, step.ExecutionContext.ContextName, (step.ExecutionContext.Result ?? TaskResult.Succeeded).ToActionResult());
-                step.ExecutionContext.Global.StepsContext.SetConclusion(step.ExecutionContext.ScopeName, step.ExecutionContext.ContextName, (step.ExecutionContext.Result ?? TaskResult.Succeeded).ToActionResult());
-            }
+            step.ExecutionContext.UpdateGlobalStepsContext();
         }
     }
 }

src/Runner.Worker/Handlers/NodeScriptActionHandler.cs

@@ -3,6 +3,8 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.DistributedTask.Pipelines.ContextData;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Sdk;
@@ -112,6 +114,17 @@ namespace GitHub.Runner.Worker.Handlers
             // Remove environment variable that may cause conflicts with the node within the runner.
             Environment.Remove("NODE_ICU_DATA"); // https://github.com/actions/runner/issues/795
 
+            if (Data.NodeVersion == "node12" && (ExecutionContext.Global.Variables.GetBoolean(Constants.Runner.Features.Node12Warning) ?? false))
+            {
+                if (!ExecutionContext.JobContext.ContainsKey("Node12ActionsWarnings"))
+                {                     
+                    ExecutionContext.JobContext["Node12ActionsWarnings"] = new ArrayContextData();
+                }
+                var repoAction = Action as RepositoryPathReference;
+                var actionDisplayName = new StringContextData(repoAction.Name ?? repoAction.Path); // local actions don't have a 'Name'                
+                ExecutionContext.JobContext["Node12ActionsWarnings"].AssertArray("Node12ActionsWarnings").Add(actionDisplayName);
+            }
+
             using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager))
             using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager))
             {

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -153,7 +153,8 @@ namespace GitHub.Runner.Worker.Handlers
             string workingDirectory = null;
             if (!Inputs.TryGetValue("workingDirectory", out workingDirectory))
             {
-                if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
+                // Don't use job level working directories for hooks
+                if (IsActionStep && string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
                 {
                     if (runDefaults.TryGetValue("working-directory", out workingDirectory))
                     {

src/Runner.Worker/JobRunner.cs

@@ -6,6 +6,7 @@ using System.Net.Http;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines.ContextData;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
@@ -257,6 +258,12 @@ namespace GitHub.Runner.Worker
                 }
             }
 
+            if (jobContext.JobContext.ContainsKey("Node12ActionsWarnings"))
+            {
+                var actions = string.Join(", ", jobContext.JobContext["Node12ActionsWarnings"].AssertArray("Node12ActionsWarnings").Select(action => action.ToString()));
+                jobContext.Warning(string.Format(Constants.Runner.Node12DetectedAfterEndOfLife, actions));
+            }
+
             try
             {
                 await ShutdownQueue(throwOnFailure: true);

src/Runner.Worker/StepsRunner.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -133,8 +134,10 @@ namespace GitHub.Runner.Worker
                             // Test the condition again. The job was cancelled after the condition was originally evaluated.
                             jobCancelRegister = jobContext.CancellationToken.Register(() =>
                             {
-                                // Mark job as cancelled
-                                jobContext.Result = TaskResult.Canceled;
+                                // Mark job as Cancelled or Failed depending on HostContext shutdown token's cancellation
+                                jobContext.Result = HostContext.RunnerShutdownToken.IsCancellationRequested
+                                                    ? TaskResult.Failed
+                                                    : TaskResult.Canceled;
                                 jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
 
                                 step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
@@ -172,8 +175,10 @@ namespace GitHub.Runner.Worker
                         {
                             if (jobContext.Result != TaskResult.Canceled)
                             {
-                                // Mark job as cancelled
-                                jobContext.Result = TaskResult.Canceled;
+                                // Mark job as Cancelled or Failed depending on HostContext shutdown token's cancellation
+                                jobContext.Result = HostContext.RunnerShutdownToken.IsCancellationRequested
+                                    ? TaskResult.Failed
+                                    : TaskResult.Canceled;
                                 jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
                             }
                         }
@@ -319,29 +324,8 @@ namespace GitHub.Runner.Worker
                 step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
             }
 
-            // Fixup the step result if ContinueOnError
-            if (step.ExecutionContext.Result == TaskResult.Failed)
-            {
-                var continueOnError = false;
-                try
-                {
-                    continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
-                }
-                catch (Exception ex)
-                {
-                    Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
-                    Trace.Error(ex);
-                    step.ExecutionContext.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
-                    step.ExecutionContext.Error(ex);
-                }
+            step.ExecutionContext.ApplyContinueOnError(step.ContinueOnError);
 
-                if (continueOnError)
-                {
-                    step.ExecutionContext.Outcome = step.ExecutionContext.Result;
-                    step.ExecutionContext.Result = TaskResult.Succeeded;
-                    Trace.Info($"Updated step result (continue on error)");
-                }
-            }
             Trace.Info($"Step result: {step.ExecutionContext.Result}");
 
             // Complete the step context

src/Runner.Worker/action_yaml.json

@@ -129,6 +129,7 @@
                         "required": true
                     },
                     "env": "step-env",
+                    "continue-on-error": "boolean-steps-context",
                     "working-directory": "string-steps-context",
                     "shell": {
                         "type": "non-empty-string",
@@ -147,6 +148,7 @@
                         "type": "non-empty-string",
                         "required": true
                     },
+                    "continue-on-error": "boolean-steps-context",
                     "with": "step-with",
                     "env": "step-env"
                 }
@@ -201,6 +203,20 @@
             ],
             "string": {}
         },
+        "boolean-steps-context": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "boolean": {}
+        },
         "step-env": {
             "context": [
                 "github",

src/Test/L0/Worker/ExecutionContextL0.cs

@@ -8,6 +8,7 @@ using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Worker;
 using Moq;
 using Xunit;
+using GitHub.DistributedTask.ObjectTemplating.Tokens;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Common.Tests.Worker
@@ -90,6 +91,63 @@ namespace GitHub.Runner.Common.Tests.Worker
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public void ApplyContinueOnError_CheckResultAndOutcome()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+
+                // Arrange: Create a job request message.
+                TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
+                TimelineReference timeline = new TimelineReference();
+                Guid jobId = Guid.NewGuid();
+                string jobName = "some job name";
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary<string, VariableValue>(), new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null);
+                jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
+                {
+                    Alias = Pipelines.PipelineConstants.SelfAlias,
+                    Id = "github",
+                    Version = "sha1"
+                });
+                jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
+                jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true";
+
+                // Arrange: Setup the paging logger.
+                var pagingLogger = new Mock<IPagingLogger>();
+                var jobServerQueue = new Mock<IJobServerQueue>();
+                jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny<Guid>(), It.IsAny<TimelineRecord>()));
+                jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<long>())).Callback((Guid id, string msg, long? lineNumber) => { hc.GetTrace().Info(msg); });
+
+                hc.EnqueueInstance(pagingLogger.Object);
+                hc.SetSingleton(jobServerQueue.Object);
+
+                var ec = new Runner.Worker.ExecutionContext();
+                ec.Initialize(hc);
+
+                // Act.
+                ec.InitializeJob(jobRequest, CancellationToken.None);
+
+                foreach (var tc in new List<(TemplateToken token, TaskResult result, TaskResult? expectedResult, TaskResult? expectedOutcome)> {
+                (token: new BooleanToken(null, null, null, true), result: TaskResult.Failed, expectedResult: TaskResult.Succeeded, expectedOutcome: TaskResult.Failed),
+                (token: new BooleanToken(null, null, null, true), result: TaskResult.Succeeded, expectedResult: TaskResult.Succeeded, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, true), result: TaskResult.Canceled, expectedResult: TaskResult.Canceled, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, false), result: TaskResult.Failed, expectedResult: TaskResult.Failed, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, false), result: TaskResult.Succeeded, expectedResult: TaskResult.Succeeded, expectedOutcome: null),
+                (token: new BooleanToken(null, null, null, false), result: TaskResult.Canceled, expectedResult: TaskResult.Canceled, expectedOutcome: null),
+            })
+                {
+                    ec.Result = tc.result;
+                    ec.Outcome = null;
+                    ec.ApplyContinueOnError(tc.token);
+                    Assert.Equal(ec.Result, tc.expectedResult);
+                    Assert.Equal(ec.Outcome, tc.expectedOutcome);
+                }
+            }
+        }
+
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]

src/Test/L0/Worker/StepsRunnerL0.cs

@@ -622,6 +622,40 @@ namespace GitHub.Runner.Common.Tests.Worker
                     _stepContext.SetOutcome("", stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
                     _stepContext.SetConclusion("", stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
                 });
+
+            stepContext.Setup(x => x.UpdateGlobalStepsContext()).Callback(() =>
+            {
+                if (!string.IsNullOrEmpty(stepContext.Object.ContextName) && !stepContext.Object.ContextName.StartsWith("__", StringComparison.Ordinal))
+                {
+                    stepContext.Object.Global.StepsContext.SetOutcome(stepContext.Object.ScopeName, stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
+                    stepContext.Object.Global.StepsContext.SetConclusion(stepContext.Object.ScopeName, stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
+                }
+            });
+            stepContext.Setup(x => x.ApplyContinueOnError(It.IsAny<TemplateToken>())).Callback((TemplateToken token) =>
+            {
+                if (stepContext.Object.Result != TaskResult.Failed)
+                {
+                    return;
+                }
+                var continueOnError = false;
+                try
+                {
+                    var templateEvaluator = stepContext.Object.ToPipelineTemplateEvaluator();
+                    continueOnError = templateEvaluator.EvaluateStepContinueOnError(token, stepContext.Object.ExpressionValues, stepContext.Object.ExpressionFunctions);
+                }
+                catch (Exception ex)
+                {
+                    stepContext.Object.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
+                    stepContext.Object.Error(ex);
+                }
+
+                if (continueOnError)
+                {
+                    stepContext.Object.Outcome = stepContext.Object.Result;
+                    stepContext.Object.Result = TaskResult.Succeeded;
+                }
+                stepContext.Object.UpdateGlobalStepsContext();
+            });
             var trace = hc.GetTrace();
             stepContext.Setup(x => x.Write(It.IsAny<string>(), It.IsAny<string>())).Callback((string tag, string message) => { trace.Info($"[{tag}]{message}"); });
             stepContext.Object.Result = result;

src/dev.sh

@@ -2,7 +2,7 @@
 
 ###############################################################################
 #  
-#  ./dev.sh build/layout/test/package [Debug/Release]
+#  ./dev.sh build/layout/test/package [Debug/Release] [linux-x64|linux-x86|linux-arm64|linux-arm|osx-x64|win-x64|win-x86] [use-broker]
 #
 ###############################################################################
 
@@ -11,6 +11,7 @@ set -e
 DEV_CMD=$1
 DEV_CONFIG=$2
 DEV_TARGET_RUNTIME=$3
+DEV_USE_BROKER=$4
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 LAYOUT_DIR="$SCRIPT_DIR/../_layout"
@@ -81,6 +82,13 @@ elif [[ "$CURRENT_PLATFORM" == 'darwin' ]]; then
     fi
 fi
 
+if [ -n "$DEV_USE_BROKER" ]; then
+    USE_BROKER='-p:USE_BROKER="true"'
+else
+    USE_BROKER=''
+fi
+
+
 function failed()
 {
     local error=${1:-Undefined error}
@@ -114,13 +122,13 @@ function heading()
 function build ()
 {
     heading "Building ..."
-    dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build
+    dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" $USE_BROKER ./dir.proj || failed build
 }
 
 function layout ()
 {
     heading "Create layout ..."
-    dotnet msbuild -t:layout -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build
+    dotnet msbuild -t:layout -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" $USE_BROKER ./dir.proj || failed build
 
     #change execution flag to allow running with sudo
     if [[ ("$CURRENT_PLATFORM" == "linux") || ("$CURRENT_PLATFORM" == "darwin") ]]; then

src/runnerversion

@@ -1 +1 @@
-2.289.1
+2.289.2