Compare commits

..

1 Commits

Author SHA1 Message Date
Thomas Boop
d4b794dd64 2.292.0 release 2022-05-23 11:15:56 -04:00
19 changed files with 72 additions and 340 deletions

15
.vscode/launch.json vendored
View File

@@ -12,7 +12,8 @@
],
"cwd": "${workspaceFolder}/src",
"console": "integratedTerminal",
"requireExactSource": false
"requireExactSource": false,
"targetArchitecture": "x86_64"
},
{
"name": "Run",
@@ -24,7 +25,8 @@
],
"cwd": "${workspaceFolder}/src",
"console": "integratedTerminal",
"requireExactSource": false
"requireExactSource": false,
"targetArchitecture": "x86_64"
},
{
"name": "Configure",
@@ -37,21 +39,24 @@
],
"cwd": "${workspaceFolder}/src",
"console": "integratedTerminal",
"requireExactSource": false
"requireExactSource": false,
"targetArchitecture": "x86_64"
},
{
"name": "Debug Worker",
"type": "coreclr",
"request": "attach",
"processName": "Runner.Worker",
"requireExactSource": false
"requireExactSource": false,
"targetArchitecture": "x86_64"
},
{
"name": "Attach Debugger",
"type": "coreclr",
"request": "attach",
"processId": "${command:pickProcess}",
"requireExactSource": false
"requireExactSource": false,
"targetArchitecture": "x86_64"
},
],
}

View File

@@ -1 +1 @@
<Update to ./src/runnerversion when creating release>
2.292.0

View File

@@ -227,7 +227,6 @@ namespace GitHub.Runner.Common
//
public static readonly string AllowUnsupportedCommands = "ACTIONS_ALLOW_UNSECURE_COMMANDS";
public static readonly string AllowUnsupportedStopCommandTokens = "ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS";
public static readonly string RequireJobContainer = "ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER";
public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION";

View File

@@ -1,76 +0,0 @@
using System;
using System.Collections.Generic;
using System.Runtime.Serialization;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.WebApi;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(RunServer))]
public interface IRunServer : IRunnerService
{
Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
Task<AgentJobRequestMessage> GetJobMessageAsync(string id);
}
public sealed class RunServer : RunnerService, IRunServer
{
private bool _hasConnection;
private VssConnection _connection;
private TaskAgentHttpClient _taskAgentClient;
public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
{
_connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
_taskAgentClient = _connection.GetClient<TaskAgentHttpClient>();
_hasConnection = true;
}
private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout)
{
Trace.Info($"EstablishVssConnection");
Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout.");
int attemptCount = 5;
while (attemptCount-- > 0)
{
var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout);
try
{
await connection.ConnectAsync();
return connection;
}
catch (Exception ex) when (attemptCount > 0)
{
Trace.Info($"Catch exception during connect. {attemptCount} attempt left.");
Trace.Error(ex);
await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None);
}
}
// should never reach here.
throw new InvalidOperationException(nameof(EstablishVssConnection));
}
private void CheckConnection()
{
if (!_hasConnection)
{
throw new InvalidOperationException($"SetConnection");
}
}
public Task<AgentJobRequestMessage> GetJobMessageAsync(string id)
{
CheckConnection();
return _taskAgentClient.GetJobMessageAsync(id);
}
}
}

View File

@@ -1,24 +1,18 @@
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Listener.Configuration;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Listener.Check;
using GitHub.Runner.Listener.Configuration;
using GitHub.Runner.Sdk;
using GitHub.Services.WebApi;
using Pipelines = GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.Pipelines;
using GitHub.Services.Common;
using System.Runtime.Serialization;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using System.Linq;
using GitHub.Runner.Listener.Check;
using System.Collections.Generic;
namespace GitHub.Runner.Listener
{
@@ -328,7 +322,6 @@ namespace GitHub.Runner.Listener
// Should we try to cleanup ephemeral runners
bool runOnceJobCompleted = false;
bool skipSessionDeletion = false;
try
{
var notification = HostContext.GetService<IJobNotification>();
@@ -464,42 +457,6 @@ namespace GitHub.Runner.Listener
}
}
}
// Broker flow
else if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase))
{
if (autoUpdateInProgress || runOnceJobReceived)
{
skipMessageDeletion = true;
Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
}
else
{
var messageRef = StringUtil.ConvertFromJson<RunnerJobRequestRef>(message.Body);
// Create connection
var credMgr = HostContext.GetService<ICredentialManager>();
var creds = credMgr.LoadCredentials();
// todo: add retries https://github.com/github/actions-broker/issues/49
var runServer = HostContext.CreateService<IRunServer>();
await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
var jobMessage = await RetriesHelper<AgentJobRequestMessage>.RetryWithTimeoutAsync(async () =>
{
return await runServer.GetJobMessageAsync(messageRef.RunnerRequestId);
},
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(10),
5);
jobDispatcher.Run(jobMessage, runOnce);
if (runOnce)
{
Trace.Info("One time used runner received job message.");
runOnceJobReceived = true;
}
}
}
else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var cancelJobMessage = JsonUtility.FromString<JobCancelMessage>(message.Body);
@@ -511,14 +468,6 @@ namespace GitHub.Runner.Listener
Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'.");
}
}
else if (string.Equals(message.MessageType, Pipelines.HostedRunnerShutdownMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var HostedRunnerShutdownMessage = JsonUtility.FromString<Pipelines.HostedRunnerShutdownMessage>(message.Body);
skipMessageDeletion = true;
skipSessionDeletion = true;
Trace.Info($"Service requests the hosted runner to shutdown. Reason: '{HostedRunnerShutdownMessage.Reason}'.");
return Constants.Runner.ReturnCode.Success;
}
else
{
Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
@@ -552,18 +501,15 @@ namespace GitHub.Runner.Listener
await jobDispatcher.ShutdownAsync();
}
if (!skipSessionDeletion)
try
{
try
{
await _listener.DeleteSessionAsync();
}
catch (Exception ex) when (runOnce)
{
// ignore exception during delete session for ephemeral runner since the runner might already be deleted from the server side
// and the delete session call will ends up with 401.
Trace.Info($"Ignore any exception during DeleteSession for an ephemeral runner. {ex}");
}
await _listener.DeleteSessionAsync();
}
catch (Exception ex) when (runOnce)
{
// ignore exception during delete session for ephemeral runner since the runner might already be deleted from the server side
// and the delete session call will ends up with 401.
Trace.Info($"Ignore any exception during DeleteSession for an ephemeral runner. {ex}");
}
messageQueueLoopTokenSource.Dispose();

View File

@@ -1,13 +0,0 @@
using System.Runtime.Serialization;
namespace GitHub.Runner.Listener
{
[DataContract]
public sealed class RunnerJobRequestRef
{
[DataMember(Name = "id")]
public string Id { get; set; }
[DataMember(Name = "runner_request_id")]
public string RunnerRequestId { get; set; }
}
}

View File

@@ -57,10 +57,6 @@ namespace GitHub.Runner.Sdk
settings.SendTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200));
}
if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_ALLOW_REDIRECT")))
{
settings.AllowAutoRedirect = true;
}
// Remove Invariant from the list of accepted languages.
//

View File

@@ -1,6 +1,7 @@
using System;
using System.IO;
using System.Linq;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Sdk
{

View File

@@ -275,6 +275,12 @@ namespace GitHub.Runner.Worker
public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container)
{
if (!context.Global.Variables.GetBoolean("DistributedTask.UploadStepSummary") ?? true)
{
Trace.Info("Step Summary is disabled; skipping attachment upload");
return;
}
if (String.IsNullOrEmpty(filePath) || !File.Exists(filePath))
{
Trace.Info($"Step Summary file ({filePath}) does not exist; skipping attachment upload");

View File

@@ -202,32 +202,14 @@ namespace GitHub.Runner.Worker.Handlers
}
else
{
// For these shells, we want to use system binaries
var systemShells = new string[] { "bash", "sh", "powershell", "pwsh" };
if (!IsActionStep && systemShells.Contains(shell))
var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell);
shellCommand = parsed.shellCommand;
// For non-ContainerStepHost, the command must be located on the host by Which
commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
argFormat = $"{parsed.shellArgs}".TrimStart();
if (string.IsNullOrEmpty(argFormat))
{
shellCommand = shell;
commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath);
if (shell == "bash")
{
argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat("sh");
}
else
{
argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shell);
}
}
else
{
var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell);
shellCommand = parsed.shellCommand;
// For non-ContainerStepHost, the command must be located on the host by Which
commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath);
argFormat = $"{parsed.shellArgs}".TrimStart();
if (string.IsNullOrEmpty(argFormat))
{
argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
}
argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand);
}
}
@@ -247,7 +229,7 @@ namespace GitHub.Runner.Worker.Handlers
{
// We do not not the full path until we know what shell is being used, so that we can determine the file extension
scriptFilePath = Path.Combine(tempDirectory, $"{Guid.NewGuid()}{ScriptHandlerHelpers.GetScriptFileExtension(shellCommand)}");
resolvedScriptPath = StepHost.ResolvePathForStepHost(scriptFilePath).Replace("\"", "\\\"");
resolvedScriptPath = $"{StepHost.ResolvePathForStepHost(scriptFilePath).Replace("\"", "\\\"")}";
}
else
{

View File

@@ -82,23 +82,18 @@ namespace GitHub.Runner.Worker.Handlers
}
}
internal static string GetDefaultShellNameForScript(string path, Common.Tracing trace, string prependPath)
internal static string GetDefaultShellForScript(string path, Common.Tracing trace, string prependPath)
{
var format = "{0} {1}";
switch (Path.GetExtension(path))
{
case ".sh":
// use 'sh' args but prefer bash
if (WhichUtil.Which("bash", false, trace, prependPath) != null)
{
return "bash";
}
return "sh";
var pathToShell = WhichUtil.Which("bash", false, trace, prependPath) ?? WhichUtil.Which("sh", true, trace, prependPath);
return string.Format(format, pathToShell, _defaultArguments["sh"]);
case ".ps1":
if (WhichUtil.Which("pwsh", false, trace, prependPath) != null)
{
return "pwsh";
}
return "powershell";
var pathToPowershell = WhichUtil.Which("pwsh", false, trace, prependPath) ?? WhichUtil.Which("powershell", true, trace, prependPath);
return string.Format(format, pathToPowershell, _defaultArguments["powershell"]);
default:
throw new ArgumentException($"{path} is not a valid path to a script. Make sure it ends in '.sh' or '.ps1'.");
}

View File

@@ -9,7 +9,6 @@ using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.Pipelines.ObjectTemplating;
using GitHub.DistributedTask.WebApi;
@@ -207,7 +206,6 @@ namespace GitHub.Runner.Worker
// Evaluate the job container
context.Debug("Evaluating job container");
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
ValidateJobContainer(container);
if (container != null)
{
jobContext.Global.Container = new Container.ContainerInfo(HostContext, container);
@@ -674,13 +672,5 @@ namespace GitHub.Runner.Worker
Trace.Info($"Total accessible running process: {snapshot.Count}.");
return snapshot;
}
private static void ValidateJobContainer(JobContainer container)
{
if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.RequireJobContainer)) && container == null)
{
throw new ArgumentException("Jobs without a job container are forbidden on this runner, please add a 'container:' to your job or contact your self-hosted runner administrator.");
}
}
}
}

View File

@@ -18,8 +18,8 @@ namespace GitHub.Runner.Worker
public class JobHookData
{
public string Path { get; private set; }
public ActionRunStage Stage { get; private set; }
public string Path {get; private set;}
public ActionRunStage Stage {get; private set;}
public JobHookData(ActionRunStage stage, string path)
{
@@ -60,7 +60,7 @@ namespace GitHub.Runner.Worker
Dictionary<string, string> inputs = new()
{
["path"] = hookData.Path,
["shell"] = ScriptHandlerHelpers.GetDefaultShellNameForScript(hookData.Path, Trace, prependPath)
["shell"] = ScriptHandlerHelpers.GetDefaultShellForScript(hookData.Path, Trace, prependPath)
};
// Create the handler

View File

@@ -1,41 +0,0 @@
using System;
using System.ComponentModel;
using System.Threading.Tasks;
namespace GitHub.Services.Common
{
[EditorBrowsable(EditorBrowsableState.Never)]
public static class RetriesHelper<T>
{
public static async Task<T> RetryWithTimeoutAsync(
Func<Task<T>> retriableAction,
TimeSpan minBackoff,
TimeSpan maxBackoff,
int maxTimeoutMinutes = 5
)
{
var remainingTime = TimeSpan.FromMinutes(maxTimeoutMinutes);
while (true)
{
try
{
return await retriableAction();
}
catch
{
if (remainingTime > TimeSpan.Zero)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(minBackoff, maxBackoff);
remainingTime -= backOff;
await Task.Delay(backOff);
}
else
{
throw;
}
}
}
}
}
}

View File

@@ -1,39 +0,0 @@
using System;
using System.Runtime.Serialization;
using GitHub.Services.WebApi;
using Newtonsoft.Json;
namespace GitHub.DistributedTask.Pipelines
{
[DataContract]
public sealed class HostedRunnerShutdownMessage
{
public static readonly String MessageType = "RunnerShutdown";
[JsonConstructor]
internal HostedRunnerShutdownMessage()
{
}
public HostedRunnerShutdownMessage(String reason)
{
this.Reason = reason;
}
[DataMember]
public String Reason
{
get;
private set;
}
public WebApi.TaskAgentMessage GetAgentMessage()
{
return new WebApi.TaskAgentMessage
{
Body = JsonUtility.ToString(this),
MessageType = HostedRunnerShutdownMessage.MessageType,
};
}
}
}

View File

@@ -5,6 +5,5 @@ namespace GitHub.DistributedTask.WebApi
public static class JobRequestMessageTypes
{
public const String PipelineAgentJobRequest = "PipelineAgentJobRequest";
public const String RunnerJobRequest = "RunnerJobRequest";
}
}

View File

@@ -141,24 +141,6 @@ namespace GitHub.DistributedTask.WebApi
return ReplaceAgentAsync(poolId, agent.Id, agent, userState, cancellationToken);
}
public Task<Pipelines.AgentJobRequestMessage> GetJobMessageAsync(
string messageId,
object userState = null,
CancellationToken cancellationToken = default)
{
HttpMethod httpMethod = new HttpMethod("GET");
Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a");
object routeValues = new { messageId = messageId };
return SendAsync<Pipelines.AgentJobRequestMessage>(
httpMethod,
locationId,
routeValues: routeValues,
version: new ApiResourceVersion(6.0, 1),
userState: userState,
cancellationToken: cancellationToken);
}
protected Task<T> SendAsync<T>(
HttpMethod method,
Guid locationId,

View File

@@ -25,6 +25,23 @@ namespace GitHub.Runner.Common.Tests.Worker
private CreateStepSummaryCommand _createStepCommand;
private ITraceWriter _trace;
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public void CreateStepSummaryCommand_FeatureDisabled()
{
using (var hostContext = Setup(featureFlagState: "false"))
{
var stepSummaryFile = Path.Combine(_rootDirectory, "feature-off");
_createStepCommand.ProcessCommand(_executionContext.Object, stepSummaryFile, null);
_jobExecutionContext.Complete();
_jobServerQueue.Verify(x => x.QueueFileUpload(It.IsAny<Guid>(), It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<bool>()), Times.Never());
Assert.Equal(0, _issues.Count);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
@@ -182,7 +199,7 @@ namespace GitHub.Runner.Common.Tests.Worker
File.WriteAllText(path, contentStr, encoding);
}
private TestHostContext Setup([CallerMemberName] string name = "")
private TestHostContext Setup([CallerMemberName] string name = "", string featureFlagState = "true")
{
var hostContext = new TestHostContext(this, name);
@@ -224,6 +241,7 @@ namespace GitHub.Runner.Common.Tests.Worker
_variables = new Variables(hostContext, new Dictionary<string, VariableValue>
{
{ "MySecretName", new VariableValue("My secret value", true) },
{ "DistributedTask.UploadStepSummary", featureFlagState },
});
// Directory for test data

View File

@@ -211,24 +211,6 @@ namespace GitHub.Runner.Common.Tests.Worker
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async Task JobExtensionBuildFailsWithoutContainerIfRequired()
{
Environment.SetEnvironmentVariable(Constants.Variables.Actions.RequireJobContainer, "true");
using (TestHostContext hc = CreateTestContext())
{
var jobExtension = new JobExtension();
jobExtension.Initialize(hc);
_actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny<IExecutionContext>(), It.IsAny<IEnumerable<Pipelines.JobStep>>(), It.IsAny<Guid>()))
.Returns(Task.FromResult(new PrepareResult(new List<JobExtensionRunner>() { new JobExtensionRunner(null, "", "prepare1", null), new JobExtensionRunner(null, "", "prepare2", null) }, new Dictionary<Guid, IActionRunner>())));
await Assert.ThrowsAsync<ArgumentException>(() => jobExtension.InitializeJob(_jobEc, _message));
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]