Small refactoring

* Allow admins to fail jobs without container

* Make method static

* Update src/Runner.Common/Constants.cs

Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com>

* Update src/Runner.Worker/JobExtension.cs

Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com>

* Update src/Runner.Worker/JobExtension.cs

Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com>

* Rename env

* Add test for throwing when no container but required

* Update src/Runner.Worker/JobExtension.cs

* Update src/Test/L0/Worker/JobExtensionL0.cs

* Update src/Runner.Common/Constants.cs

Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com>

Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com>

.vscode/launch.json

@@ -12,8 +12,7 @@
             ],
             "cwd": "${workspaceFolder}/src",
             "console": "integratedTerminal",
-            "requireExactSource": false,
-            "targetArchitecture": "x86_64"
+            "requireExactSource": false
         },
         {
             "name": "Run",
@@ -25,8 +24,7 @@
             ],
             "cwd": "${workspaceFolder}/src",
             "console": "integratedTerminal",
-            "requireExactSource": false,
-            "targetArchitecture": "x86_64"
+            "requireExactSource": false
         },
         {
             "name": "Configure",
@@ -39,24 +37,21 @@
             ],
             "cwd": "${workspaceFolder}/src",
             "console": "integratedTerminal",
-            "requireExactSource": false,
-            "targetArchitecture": "x86_64"
+            "requireExactSource": false
         },
         {
             "name": "Debug Worker",
             "type": "coreclr",
             "request": "attach",
             "processName": "Runner.Worker",
-            "requireExactSource": false,
-            "targetArchitecture": "x86_64"
+            "requireExactSource": false
         },
         {
             "name": "Attach Debugger",
             "type": "coreclr",
             "request": "attach",
             "processId": "${command:pickProcess}",
-            "requireExactSource": false,
-            "targetArchitecture": "x86_64"
+            "requireExactSource": false
         },
     ],
 }

src/Runner.Common/Constants.cs

@@ -227,6 +227,7 @@ namespace GitHub.Runner.Common
                 //
                 public static readonly string AllowUnsupportedCommands = "ACTIONS_ALLOW_UNSECURE_COMMANDS";
                 public static readonly string AllowUnsupportedStopCommandTokens = "ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS";
+                public static readonly string RequireJobContainer = "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER";
                 public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
                 public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
                 public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION";

src/Runner.Common/RunServer.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Common.Util;
+using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
+using GitHub.Services.WebApi;
+
+namespace GitHub.Runner.Common
+{
+    [ServiceLocator(Default = typeof(RunServer))]
+    public interface IRunServer : IRunnerService
+    {
+        Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
+
+        Task<AgentJobRequestMessage> GetJobMessageAsync(string id);
+    }
+
+    public sealed class RunServer : RunnerService, IRunServer
+    {
+        private bool _hasConnection;
+        private VssConnection _connection;
+        private TaskAgentHttpClient _taskAgentClient;
+
+        public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
+        {
+            _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            _taskAgentClient = _connection.GetClient<TaskAgentHttpClient>();
+            _hasConnection = true;
+        }
+
+        private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout)
+        {
+            Trace.Info($"EstablishVssConnection");
+            Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout.");
+            int attemptCount = 5;
+            while (attemptCount-- > 0)
+            {
+                var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout);
+                try
+                {
+                    await connection.ConnectAsync();
+                    return connection;
+                }
+                catch (Exception ex) when (attemptCount > 0)
+                {
+                    Trace.Info($"Catch exception during connect. {attemptCount} attempt left.");
+                    Trace.Error(ex);
+
+                    await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None);
+                }
+            }
+
+            // should never reach here.
+            throw new InvalidOperationException(nameof(EstablishVssConnection));
+        }
+
+        private void CheckConnection()
+        {
+            if (!_hasConnection)
+            {
+                throw new InvalidOperationException($"SetConnection");
+            }
+        }
+
+        public Task<AgentJobRequestMessage> GetJobMessageAsync(string id)
+        {
+            CheckConnection();
+            return _taskAgentClient.GetJobMessageAsync(id);
+        }
+    }
+}

src/Runner.Listener/Runner.cs

@@ -1,18 +1,24 @@
-using GitHub.DistributedTask.WebApi;
-using GitHub.Runner.Listener.Configuration;
 using System;
-using System.Threading;
-using System.Threading.Tasks;
-using GitHub.Services.WebApi;
-using Pipelines = GitHub.DistributedTask.Pipelines;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Reflection;
 using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
-using GitHub.Runner.Sdk;
-using System.Linq;
 using GitHub.Runner.Listener.Check;
-using System.Collections.Generic;
+using GitHub.Runner.Listener.Configuration;
+using GitHub.Runner.Sdk;
+using GitHub.Services.WebApi;
+using Pipelines = GitHub.DistributedTask.Pipelines;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Services.Common;
+using System.Runtime.Serialization;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
 
 namespace GitHub.Runner.Listener
 {
@@ -322,6 +328,7 @@ namespace GitHub.Runner.Listener
 
                 // Should we try to cleanup ephemeral runners
                 bool runOnceJobCompleted = false;
+                bool skipSessionDeletion = false;
                 try
                 {
                     var notification = HostContext.GetService<IJobNotification>();
@@ -457,6 +464,42 @@ namespace GitHub.Runner.Listener
                                     }
                                 }
                             }
+                            // Broker flow
+                            else if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase))
+                            {
+                                if (autoUpdateInProgress || runOnceJobReceived)
+                                {
+                                    skipMessageDeletion = true;
+                                    Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
+                                }
+                                else
+                                {
+                                    var messageRef = StringUtil.ConvertFromJson<RunnerJobRequestRef>(message.Body);
+
+                                    // Create connection
+                                    var credMgr = HostContext.GetService<ICredentialManager>();
+                                    var creds = credMgr.LoadCredentials();
+
+                                    // todo: add retries https://github.com/github/actions-broker/issues/49
+                                    var runServer = HostContext.CreateService<IRunServer>();
+                                    await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
+
+                                    var jobMessage = await RetriesHelper<AgentJobRequestMessage>.RetryWithTimeoutAsync(async () =>
+                                                                                                    {
+                                                                                                        return await runServer.GetJobMessageAsync(messageRef.RunnerRequestId);
+                                                                                                    },
+                                                                                                    TimeSpan.FromSeconds(5),
+                                                                                                    TimeSpan.FromSeconds(10),
+                                                                                                    5);
+
+                                    jobDispatcher.Run(jobMessage, runOnce);
+                                    if (runOnce)
+                                    {
+                                        Trace.Info("One time used runner received job message.");
+                                        runOnceJobReceived = true;
+                                    }
+                                }
+                            }
                             else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
                             {
                                 var cancelJobMessage = JsonUtility.FromString<JobCancelMessage>(message.Body);
@@ -468,6 +511,14 @@ namespace GitHub.Runner.Listener
                                     Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'.");
                                 }
                             }
+                            else if (string.Equals(message.MessageType, Pipelines.HostedRunnerShutdownMessage.MessageType, StringComparison.OrdinalIgnoreCase))
+                            {
+                                var HostedRunnerShutdownMessage = JsonUtility.FromString<Pipelines.HostedRunnerShutdownMessage>(message.Body);
+                                skipMessageDeletion = true;
+                                skipSessionDeletion = true;
+                                Trace.Info($"Service requests the hosted runner to shutdown. Reason: '{HostedRunnerShutdownMessage.Reason}'.");
+                                return Constants.Runner.ReturnCode.Success;
+                            }
                             else
                             {
                                 Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
@@ -501,6 +552,8 @@ namespace GitHub.Runner.Listener
                         await jobDispatcher.ShutdownAsync();
                     }
 
+                    if (!skipSessionDeletion)
+                    {
                         try
                         {
                             await _listener.DeleteSessionAsync();
@@ -511,6 +564,7 @@ namespace GitHub.Runner.Listener
                             // and the delete session call will ends up with 401.
                             Trace.Info($"Ignore any exception during DeleteSession for an ephemeral runner. {ex}");
                         }
+                    }
 
                     messageQueueLoopTokenSource.Dispose();
 

src/Runner.Listener/RunnerJobRequestRef.cs

@@ -0,0 +1,13 @@
+using System.Runtime.Serialization;
+
+namespace GitHub.Runner.Listener
+{
+    [DataContract]
+    public sealed class RunnerJobRequestRef
+    {
+        [DataMember(Name = "id")]
+        public string Id { get; set; }
+        [DataMember(Name = "runner_request_id")]
+        public string RunnerRequestId { get; set; }
+    }
+}

src/Runner.Sdk/Util/VssUtil.cs

@@ -57,6 +57,10 @@ namespace GitHub.Runner.Sdk
                 settings.SendTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200));
             }
 
+            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_ALLOW_REDIRECT")))
+            {
+                settings.AllowAutoRedirect = true;
+            }
 
             // Remove Invariant from the list of accepted languages.
             //

src/Runner.Sdk/Util/WhichUtil.cs

@@ -1,7 +1,6 @@
 using System;
 using System.IO;
 using System.Linq;
-using GitHub.Runner.Sdk;
 
 namespace GitHub.Runner.Sdk
 {

src/Runner.Worker/FileCommandManager.cs

@@ -275,12 +275,6 @@ namespace GitHub.Runner.Worker
 
         public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container)
         {
-            if (!context.Global.Variables.GetBoolean("DistributedTask.UploadStepSummary") ?? true)
-            {
-                Trace.Info("Step Summary is disabled; skipping attachment upload");
-                return;
-            }
-
             if (String.IsNullOrEmpty(filePath) || !File.Exists(filePath))
             {
                 Trace.Info($"Step Summary file ({filePath}) does not exist; skipping attachment upload");

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -201,6 +201,23 @@ namespace GitHub.Runner.Worker.Handlers
                 argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
             }
             else
+            {
+                // For these shells, we want to use system binaries
+                var systemShells = new string[] { "bash", "sh", "powershell", "pwsh" };
+                if (!IsActionStep && systemShells.Contains(shell))
+                {
+                    shellCommand = shell;
+                    commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
+                    if (shell == "bash")
+                    {
+                        argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat("sh");
+                    }
+                    else
+                    {
+                        argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shell);
+                    }
+                }
+                else
                 {
                     var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell);
                     shellCommand = parsed.shellCommand;
@@ -212,6 +229,7 @@ namespace GitHub.Runner.Worker.Handlers
                         argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
                     }
                 }
+            }
 
             // Don't override runner telemetry here
             if (!string.IsNullOrEmpty(shellCommand) && IsActionStep)
@@ -229,7 +247,7 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 // We do not not the full path until we know what shell is being used, so that we can determine the file extension
                 scriptFilePath = Path.Combine(tempDirectory, $"{Guid.NewGuid()}{ScriptHandlerHelpers.GetScriptFileExtension(shellCommand)}");
-                resolvedScriptPath = $"{StepHost.ResolvePathForStepHost(scriptFilePath).Replace("\"", "\\\"")}";
+                resolvedScriptPath = StepHost.ResolvePathForStepHost(scriptFilePath).Replace("\"", "\\\"");
             }
             else
             {

src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs

@@ -82,18 +82,23 @@ namespace GitHub.Runner.Worker.Handlers
             }
         }
 
-        internal static string GetDefaultShellForScript(string path, Common.Tracing trace, string prependPath)
+        internal static string GetDefaultShellNameForScript(string path, Common.Tracing trace, string prependPath)
         {
-            var format = "{0} {1}";
             switch (Path.GetExtension(path))
             {
                 case ".sh":
                     // use 'sh' args but prefer bash
-                    var pathToShell = WhichUtil.Which("bash", false, trace, prependPath) ?? WhichUtil.Which("sh", true, trace, prependPath);
-                    return string.Format(format, pathToShell, _defaultArguments["sh"]);
+                    if (WhichUtil.Which("bash", false, trace, prependPath) != null)
+                    {
+                        return "bash";
+                    }
+                    return "sh";
                 case ".ps1":
-                    var pathToPowershell = WhichUtil.Which("pwsh", false, trace, prependPath) ?? WhichUtil.Which("powershell", true, trace, prependPath);
-                    return string.Format(format, pathToPowershell, _defaultArguments["powershell"]);
+                    if (WhichUtil.Which("pwsh", false, trace, prependPath) != null)
+                    {
+                        return "pwsh";
+                    }
+                    return "powershell";
                 default:
                     throw new ArgumentException($"{path} is not a valid path to a script. Make sure it ends in '.sh' or '.ps1'.");
             }

src/Runner.Worker/JobExtension.cs

@@ -9,6 +9,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.Expressions2;
 using GitHub.DistributedTask.ObjectTemplating.Tokens;
+using GitHub.DistributedTask.Pipelines;
 using GitHub.DistributedTask.Pipelines.ContextData;
 using GitHub.DistributedTask.Pipelines.ObjectTemplating;
 using GitHub.DistributedTask.WebApi;
@@ -206,6 +207,7 @@ namespace GitHub.Runner.Worker
                     // Evaluate the job container
                     context.Debug("Evaluating job container");
                     var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
+                    ValidateJobContainer(container);
                     if (container != null)
                     {
                         jobContext.Global.Container = new Container.ContainerInfo(HostContext, container);
@@ -672,5 +674,13 @@ namespace GitHub.Runner.Worker
             Trace.Info($"Total accessible running process: {snapshot.Count}.");
             return snapshot;
         }
+
+        private static void ValidateJobContainer(JobContainer container)
+        {
+            if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.RequireJobContainer)) && container == null)
+            {
+                throw new ArgumentException("Jobs without a job container are forbidden on this runner, please add a 'container:' to your job or contact your self-hosted runner administrator.");
+            }
+        }
     }
 }

src/Runner.Worker/JobHookProvider.cs

@@ -60,7 +60,7 @@ namespace GitHub.Runner.Worker
             Dictionary<string, string> inputs = new()
             {
                 ["path"] = hookData.Path,
-                ["shell"] = ScriptHandlerHelpers.GetDefaultShellForScript(hookData.Path, Trace, prependPath)
+                ["shell"] = ScriptHandlerHelpers.GetDefaultShellNameForScript(hookData.Path, Trace, prependPath)
             };
 
             // Create the handler

src/Sdk/Common/Common/Utility/RetriesHelper.cs

@@ -0,0 +1,41 @@
+using System;
+using System.ComponentModel;
+using System.Threading.Tasks;
+
+namespace GitHub.Services.Common
+{
+    [EditorBrowsable(EditorBrowsableState.Never)]
+
+    public static class RetriesHelper<T>
+    {
+        public static async Task<T> RetryWithTimeoutAsync(
+            Func<Task<T>> retriableAction,
+            TimeSpan minBackoff,
+            TimeSpan maxBackoff,
+            int maxTimeoutMinutes = 5
+        )
+        {
+            var remainingTime = TimeSpan.FromMinutes(maxTimeoutMinutes);
+            while (true)
+            {
+                try
+                {
+                    return await retriableAction();
+                }
+                catch
+                {   
+                    if (remainingTime > TimeSpan.Zero)
+                    {
+                        var backOff = BackoffTimerHelper.GetRandomBackoff(minBackoff, maxBackoff);
+                        remainingTime -= backOff;
+                        await Task.Delay(backOff);
+                    }
+                    else
+                    {
+                        throw;
+                    }
+                }
+            }
+        }
+    }
+}

src/Sdk/DTPipelines/Pipelines/RunnerShutdownMessage.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Runtime.Serialization;
+using GitHub.Services.WebApi;
+using Newtonsoft.Json;
+
+namespace GitHub.DistributedTask.Pipelines
+{
+    [DataContract]
+    public sealed class HostedRunnerShutdownMessage
+    {
+        public static readonly String MessageType = "RunnerShutdown";
+
+        [JsonConstructor]
+        internal HostedRunnerShutdownMessage()
+        {
+        }
+
+        public HostedRunnerShutdownMessage(String reason)
+        {
+            this.Reason = reason;
+        }
+
+        [DataMember]
+        public String Reason
+        {
+            get;
+            private set;
+        }
+
+        public WebApi.TaskAgentMessage GetAgentMessage()
+        {
+            return new WebApi.TaskAgentMessage
+            {
+                Body = JsonUtility.ToString(this),
+                MessageType = HostedRunnerShutdownMessage.MessageType,
+            };
+        }
+    }
+}

src/Sdk/DTWebApi/WebApi/JobRequestMessageTypes.cs

@@ -5,5 +5,6 @@ namespace GitHub.DistributedTask.WebApi
     public static class JobRequestMessageTypes
     {
         public const String PipelineAgentJobRequest = "PipelineAgentJobRequest";
+        public const String RunnerJobRequest = "RunnerJobRequest";
     }
 }

src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs

@@ -141,6 +141,24 @@ namespace GitHub.DistributedTask.WebApi
             return ReplaceAgentAsync(poolId, agent.Id, agent, userState, cancellationToken);
         }
 
+        public Task<Pipelines.AgentJobRequestMessage> GetJobMessageAsync(
+            string messageId,
+            object userState = null,
+            CancellationToken cancellationToken = default)
+        {
+            HttpMethod httpMethod = new HttpMethod("GET");
+            Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a");
+            object routeValues = new { messageId = messageId };
+
+            return SendAsync<Pipelines.AgentJobRequestMessage>(
+                httpMethod,
+                locationId,
+                routeValues: routeValues,
+                version: new ApiResourceVersion(6.0, 1),
+                userState: userState,
+                cancellationToken: cancellationToken);
+        }
+
         protected Task<T> SendAsync<T>(
             HttpMethod method,
             Guid locationId,

src/Test/L0/Worker/CreateStepSummaryCommandL0.cs

@@ -25,23 +25,6 @@ namespace GitHub.Runner.Common.Tests.Worker
         private CreateStepSummaryCommand _createStepCommand;
         private ITraceWriter _trace;
 
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Worker")]
-        public void CreateStepSummaryCommand_FeatureDisabled()
-        {
-            using (var hostContext = Setup(featureFlagState: "false"))
-            {
-                var stepSummaryFile = Path.Combine(_rootDirectory, "feature-off");
-
-                _createStepCommand.ProcessCommand(_executionContext.Object, stepSummaryFile, null);
-                _jobExecutionContext.Complete();
-
-                _jobServerQueue.Verify(x => x.QueueFileUpload(It.IsAny<Guid>(), It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
-                Assert.Equal(0, _issues.Count);
-            }
-        }
-
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
@@ -199,7 +182,7 @@ namespace GitHub.Runner.Common.Tests.Worker
             File.WriteAllText(path, contentStr, encoding);
         }
 
-        private TestHostContext Setup([CallerMemberName] string name = "", string featureFlagState = "true")
+        private TestHostContext Setup([CallerMemberName] string name = "")
         {
             var hostContext = new TestHostContext(this, name);
 
@@ -241,7 +224,6 @@ namespace GitHub.Runner.Common.Tests.Worker
             _variables = new Variables(hostContext, new Dictionary<string, VariableValue>
                 {
                     { "MySecretName", new VariableValue("My secret value", true) },
-                    { "DistributedTask.UploadStepSummary", featureFlagState },
                 });
 
             // Directory for test data

src/Test/L0/Worker/JobExtensionL0.cs

@@ -211,6 +211,24 @@ namespace GitHub.Runner.Common.Tests.Worker
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public async Task JobExtensionBuildFailsWithoutContainerIfRequired()
+        {
+            Environment.SetEnvironmentVariable(Constants.Variables.Actions.RequireJobContainer, "true");
+            using (TestHostContext hc = CreateTestContext())
+            {
+                var jobExtension = new JobExtension();
+                jobExtension.Initialize(hc);
+
+                _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny<IExecutionContext>(), It.IsAny<IEnumerable<Pipelines.JobStep>>(), It.IsAny<Guid>()))
+                              .Returns(Task.FromResult(new PrepareResult(new List<JobExtensionRunner>() { new JobExtensionRunner(null, "", "prepare1", null), new JobExtensionRunner(null, "", "prepare2", null) }, new Dictionary<Guid, IActionRunner>())));
+
+                await Assert.ThrowsAsync<ArgumentException>(() => jobExtension.InitializeJob(_jobEc, _message));
+            }
+        }
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]