GitHub Actions Runner

This commit is contained in:
Tingluo Huang
2019-10-10 00:52:42 -04:00
commit c8afc84840
1255 changed files with 198670 additions and 0 deletions

View File

@@ -0,0 +1,314 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi;
using GitHub.Services.Common;
using GitHub.Services.WebApi;
using Newtonsoft.Json;
using Pipelines = GitHub.DistributedTask.Pipelines;
namespace GitHub.Runner.Sdk
{
public interface IRunnerActionPlugin
{
Task RunAsync(RunnerActionPluginExecutionContext executionContext, CancellationToken token);
}
public class RunnerActionPluginExecutionContext : ITraceWriter
{
private readonly string DebugEnvironmentalVariable = "ACTIONS_STEP_DEBUG";
private VssConnection _connection;
private readonly object _stdoutLock = new object();
private readonly ITraceWriter _trace; // for unit tests
public RunnerActionPluginExecutionContext()
: this(null)
{ }
public RunnerActionPluginExecutionContext(ITraceWriter trace)
{
_trace = trace;
this.Endpoints = new List<ServiceEndpoint>();
this.Inputs = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
this.Variables = new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase);
}
public List<ServiceEndpoint> Endpoints { get; set; }
public Dictionary<string, VariableValue> Variables { get; set; }
public Dictionary<string, string> Inputs { get; set; }
public DictionaryContextData Context { get; set; } = new DictionaryContextData();
[JsonIgnore]
public VssConnection VssConnection
{
get
{
if (_connection == null)
{
_connection = InitializeVssConnection();
}
return _connection;
}
}
public VssConnection InitializeVssConnection()
{
var headerValues = new List<ProductInfoHeaderValue>();
headerValues.Add(new ProductInfoHeaderValue($"GitHubActionsRunner-Plugin", BuildConstants.RunnerPackage.Version));
headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})"));
if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0)
{
headerValues.AddRange(VssClientHttpRequestSettings.Default.UserAgent);
}
VssClientHttpRequestSettings.Default.UserAgent = headerValues;
#if OS_LINUX || OS_OSX
// The .NET Core 2.1 runtime switched its HTTP default from HTTP 1.1 to HTTP 2.
// This causes problems with some versions of the Curl handler.
// See GitHub issue https://github.com/dotnet/corefx/issues/32376
VssClientHttpRequestSettings.Default.UseHttp11 = true;
#endif
var certSetting = GetCertConfiguration();
if (certSetting != null)
{
if (!string.IsNullOrEmpty(certSetting.ClientCertificateArchiveFile))
{
VssClientHttpRequestSettings.Default.ClientCertificateManager = new RunnerClientCertificateManager(certSetting.ClientCertificateArchiveFile, certSetting.ClientCertificatePassword);
}
if (certSetting.SkipServerCertificateValidation)
{
VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
}
}
var proxySetting = GetProxyConfiguration();
if (proxySetting != null)
{
if (!string.IsNullOrEmpty(proxySetting.ProxyAddress))
{
VssHttpMessageHandler.DefaultWebProxy = new RunnerWebProxyCore(proxySetting.ProxyAddress, proxySetting.ProxyUsername, proxySetting.ProxyPassword, proxySetting.ProxyBypassList);
}
}
ServiceEndpoint systemConnection = this.Endpoints.FirstOrDefault(e => string.Equals(e.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
ArgUtil.NotNull(systemConnection, nameof(systemConnection));
ArgUtil.NotNull(systemConnection.Url, nameof(systemConnection.Url));
VssCredentials credentials = VssUtil.GetVssCredential(systemConnection);
ArgUtil.NotNull(credentials, nameof(credentials));
return VssUtil.CreateConnection(systemConnection.Url, credentials);
}
public string GetInput(string name, bool required = false)
{
string value = null;
if (this.Inputs.ContainsKey(name))
{
value = this.Inputs[name];
}
Debug($"Input '{name}': '{value ?? string.Empty}'");
if (string.IsNullOrEmpty(value) && required)
{
throw new ArgumentNullException(name);
}
return value;
}
public void Info(string message)
{
Debug(message);
}
public void Verbose(string message)
{
Debug(message);
}
public void Error(string message)
{
Output($"##[error]{Escape(message)}");
}
public void Debug(string message)
{
var debugString = Variables.GetValueOrDefault(DebugEnvironmentalVariable)?.Value;
if (StringUtil.ConvertToBoolean(debugString))
{
var multilines = message?.Replace("\r\n", "\n")?.Split("\n");
if (multilines != null)
{
foreach (var line in multilines)
{
Output($"##[debug]{Escape(line)}");
}
}
}
}
public void Warning(string message)
{
Output($"##[warning]{Escape(message)}");
}
public void Output(string message)
{
lock (_stdoutLock)
{
if (_trace == null)
{
Console.WriteLine(message);
}
else
{
_trace.Info(message);
}
}
}
public void AddMask(string secret)
{
Output($"##[add-mask]{Escape(secret)}");
}
public void Command(string command)
{
Output($"##[command]{Escape(command)}");
}
public void SetRepositoryPath(string repoName, string path, bool workspaceRepo)
{
Output($"##[internal-set-repo-path repoFullName={repoName};workspaceRepo={workspaceRepo.ToString()}]{path}");
}
public void SetIntraActionState(string name, string value)
{
Output($"##[save-state name={Escape(name)}]{Escape(value)}");
}
public String GetRunnerContext(string contextName)
{
this.Context.TryGetValue("runner", out var context);
var runnerContext = context as DictionaryContextData;
ArgUtil.NotNull(runnerContext, nameof(runnerContext));
if (runnerContext.TryGetValue(contextName, out var data))
{
return data as StringContextData;
}
else
{
return null;
}
}
public String GetGitHubContext(string contextName)
{
this.Context.TryGetValue("github", out var context);
var githubContext = context as DictionaryContextData;
ArgUtil.NotNull(githubContext, nameof(githubContext));
if (githubContext.TryGetValue(contextName, out var data))
{
return data as StringContextData;
}
else
{
return null;
}
}
public RunnerCertificateSettings GetCertConfiguration()
{
bool skipCertValidation = StringUtil.ConvertToBoolean(GetRunnerContext("SkipCertValidation"));
string caFile = GetRunnerContext("CAInfo");
string clientCertFile = GetRunnerContext("ClientCert");
if (!string.IsNullOrEmpty(caFile) || !string.IsNullOrEmpty(clientCertFile) || skipCertValidation)
{
var certConfig = new RunnerCertificateSettings();
certConfig.SkipServerCertificateValidation = skipCertValidation;
certConfig.CACertificateFile = caFile;
if (!string.IsNullOrEmpty(clientCertFile))
{
certConfig.ClientCertificateFile = clientCertFile;
string clientCertKey = GetRunnerContext("ClientCertKey");
string clientCertArchive = GetRunnerContext("ClientCertArchive");
string clientCertPassword = GetRunnerContext("ClientCertPassword");
certConfig.ClientCertificatePrivateKeyFile = clientCertKey;
certConfig.ClientCertificateArchiveFile = clientCertArchive;
certConfig.ClientCertificatePassword = clientCertPassword;
certConfig.VssClientCertificateManager = new RunnerClientCertificateManager(clientCertArchive, clientCertPassword);
}
return certConfig;
}
else
{
return null;
}
}
public RunnerWebProxySettings GetProxyConfiguration()
{
string proxyUrl = GetRunnerContext("ProxyUrl");
if (!string.IsNullOrEmpty(proxyUrl))
{
string proxyUsername = GetRunnerContext("ProxyUsername");
string proxyPassword = GetRunnerContext("ProxyPassword");
List<string> proxyBypassHosts = StringUtil.ConvertFromJson<List<string>>(GetRunnerContext("ProxyBypassList") ?? "[]");
return new RunnerWebProxySettings()
{
ProxyAddress = proxyUrl,
ProxyUsername = proxyUsername,
ProxyPassword = proxyPassword,
ProxyBypassList = proxyBypassHosts,
WebProxy = new RunnerWebProxyCore(proxyUrl, proxyUsername, proxyPassword, proxyBypassHosts)
};
}
else
{
return null;
}
}
private string Escape(string input)
{
foreach (var mapping in _commandEscapeMappings)
{
input = input.Replace(mapping.Key, mapping.Value);
}
return input;
}
private Dictionary<string, string> _commandEscapeMappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
{
";", "%3B"
},
{
"\r", "%0D"
},
{
"\n", "%0A"
},
{
"]", "%5D"
},
};
}
}

View File

@@ -0,0 +1,8 @@
namespace GitHub.Runner.Sdk
{
public interface ITraceWriter
{
void Info(string message);
void Verbose(string message);
}
}

View File

@@ -0,0 +1,892 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Sdk
{
// The implementation of the process invoker does not hook up DataReceivedEvent and ErrorReceivedEvent of Process,
// instead, we read both STDOUT and STDERR stream manually on separate thread.
// The reason is we find a huge perf issue about process STDOUT/STDERR with those events.
public sealed class ProcessInvoker : IDisposable
{
private Process _proc;
private Stopwatch _stopWatch;
private int _asyncStreamReaderCount = 0;
private bool _waitingOnStreams = false;
private readonly AsyncManualResetEvent _outputProcessEvent = new AsyncManualResetEvent();
private readonly TaskCompletionSource<bool> _processExitedCompletionSource = new TaskCompletionSource<bool>();
private readonly CancellationTokenSource _processStandardInWriteCancellationTokenSource = new CancellationTokenSource();
private readonly ConcurrentQueue<string> _errorData = new ConcurrentQueue<string>();
private readonly ConcurrentQueue<string> _outputData = new ConcurrentQueue<string>();
private readonly TimeSpan _sigintTimeout = TimeSpan.FromMilliseconds(7500);
private readonly TimeSpan _sigtermTimeout = TimeSpan.FromMilliseconds(2500);
private ITraceWriter Trace { get; set; }
private class AsyncManualResetEvent
{
private volatile TaskCompletionSource<bool> m_tcs = new TaskCompletionSource<bool>();
public Task WaitAsync() { return m_tcs.Task; }
public void Set()
{
var tcs = m_tcs;
Task.Factory.StartNew(s => ((TaskCompletionSource<bool>)s).TrySetResult(true),
tcs, CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default);
tcs.Task.Wait();
}
public void Reset()
{
while (true)
{
var tcs = m_tcs;
if (!tcs.Task.IsCompleted ||
Interlocked.CompareExchange(ref m_tcs, new TaskCompletionSource<bool>(), tcs) == tcs)
return;
}
}
}
public event EventHandler<ProcessDataReceivedEventArgs> OutputDataReceived;
public event EventHandler<ProcessDataReceivedEventArgs> ErrorDataReceived;
public ProcessInvoker(ITraceWriter trace)
{
this.Trace = trace;
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: false,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: null,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: false,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: killProcessOnCancel,
redirectStandardIn: null,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: killProcessOnCancel,
redirectStandardIn: redirectStandardIn,
inheritConsoleHandler: false,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: killProcessOnCancel,
redirectStandardIn: redirectStandardIn,
inheritConsoleHandler: inheritConsoleHandler,
keepStandardInOpen: false,
highPriorityProcess: false,
cancellationToken: cancellationToken);
}
public async Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
bool keepStandardInOpen,
bool highPriorityProcess,
CancellationToken cancellationToken)
{
ArgUtil.Null(_proc, nameof(_proc));
ArgUtil.NotNullOrEmpty(fileName, nameof(fileName));
Trace.Info("Starting process:");
Trace.Info($" File name: '{fileName}'");
Trace.Info($" Arguments: '{arguments}'");
Trace.Info($" Working directory: '{workingDirectory}'");
Trace.Info($" Require exit code zero: '{requireExitCodeZero}'");
Trace.Info($" Encoding web name: {outputEncoding?.WebName} ; code page: '{outputEncoding?.CodePage}'");
Trace.Info($" Force kill process on cancellation: '{killProcessOnCancel}'");
Trace.Info($" Redirected STDIN: '{redirectStandardIn != null}'");
Trace.Info($" Persist current code page: '{inheritConsoleHandler}'");
Trace.Info($" Keep redirected STDIN open: '{keepStandardInOpen}'");
Trace.Info($" High priority process: '{highPriorityProcess}'");
_proc = new Process();
_proc.StartInfo.FileName = fileName;
_proc.StartInfo.Arguments = arguments;
_proc.StartInfo.WorkingDirectory = workingDirectory;
_proc.StartInfo.UseShellExecute = false;
_proc.StartInfo.CreateNoWindow = !inheritConsoleHandler;
_proc.StartInfo.RedirectStandardInput = true;
_proc.StartInfo.RedirectStandardError = true;
_proc.StartInfo.RedirectStandardOutput = true;
// Ensure we process STDERR even the process exit event happen before we start read STDERR stream.
if (_proc.StartInfo.RedirectStandardError)
{
Interlocked.Increment(ref _asyncStreamReaderCount);
}
// Ensure we process STDOUT even the process exit event happen before we start read STDOUT stream.
if (_proc.StartInfo.RedirectStandardOutput)
{
Interlocked.Increment(ref _asyncStreamReaderCount);
}
#if OS_WINDOWS
// If StandardErrorEncoding or StandardOutputEncoding is not specified the on the
// ProcessStartInfo object, then .NET PInvokes to resolve the default console output
// code page:
// [DllImport("api-ms-win-core-console-l1-1-0.dll", SetLastError = true)]
// public extern static uint GetConsoleOutputCP();
StringUtil.EnsureRegisterEncodings();
#endif
if (outputEncoding != null)
{
_proc.StartInfo.StandardErrorEncoding = outputEncoding;
_proc.StartInfo.StandardOutputEncoding = outputEncoding;
}
// Copy the environment variables.
if (environment != null && environment.Count > 0)
{
foreach (KeyValuePair<string, string> kvp in environment)
{
_proc.StartInfo.Environment[kvp.Key] = kvp.Value;
}
}
// Indicate GitHub Actions process.
_proc.StartInfo.Environment["GITHUB_ACTIONS"] = "true";
// Hook up the events.
_proc.EnableRaisingEvents = true;
_proc.Exited += ProcessExitedHandler;
// Start the process.
_stopWatch = Stopwatch.StartNew();
_proc.Start();
// Decrease invoked process priority, in platform specifc way, relative to parent
if (!highPriorityProcess)
{
DecreaseProcessPriority(_proc);
}
// Start the standard error notifications, if appropriate.
if (_proc.StartInfo.RedirectStandardError)
{
StartReadStream(_proc.StandardError, _errorData);
}
// Start the standard output notifications, if appropriate.
if (_proc.StartInfo.RedirectStandardOutput)
{
StartReadStream(_proc.StandardOutput, _outputData);
}
if (_proc.StartInfo.RedirectStandardInput)
{
if (redirectStandardIn != null)
{
StartWriteStream(redirectStandardIn, _proc.StandardInput, keepStandardInOpen);
}
else
{
// Close the input stream. This is done to prevent commands from blocking the build waiting for input from the user.
_proc.StandardInput.Close();
}
}
using (var registration = cancellationToken.Register(async () => await CancelAndKillProcessTree(killProcessOnCancel)))
{
Trace.Info($"Process started with process id {_proc.Id}, waiting for process exit.");
while (true)
{
Task outputSignal = _outputProcessEvent.WaitAsync();
var signaled = await Task.WhenAny(outputSignal, _processExitedCompletionSource.Task);
if (signaled == outputSignal)
{
ProcessOutput();
}
else
{
_stopWatch.Stop();
break;
}
}
// Just in case there was some pending output when the process shut down go ahead and check the
// data buffers one last time before returning
ProcessOutput();
Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}.");
}
cancellationToken.ThrowIfCancellationRequested();
// Wait for process to finish.
if (_proc.ExitCode != 0 && requireExitCodeZero)
{
throw new ProcessExitCodeException(exitCode: _proc.ExitCode, fileName: fileName, arguments: arguments);
}
return _proc.ExitCode;
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
private void Dispose(bool disposing)
{
if (disposing)
{
if (_proc != null)
{
_proc.Dispose();
_proc = null;
}
}
}
private void ProcessOutput()
{
List<string> errorData = new List<string>();
List<string> outputData = new List<string>();
string errorLine;
while (_errorData.TryDequeue(out errorLine))
{
errorData.Add(errorLine);
}
string outputLine;
while (_outputData.TryDequeue(out outputLine))
{
outputData.Add(outputLine);
}
_outputProcessEvent.Reset();
// Write the error lines.
if (errorData != null && this.ErrorDataReceived != null)
{
foreach (string line in errorData)
{
if (line != null)
{
this.ErrorDataReceived(this, new ProcessDataReceivedEventArgs(line));
}
}
}
// Process the output lines.
if (outputData != null && this.OutputDataReceived != null)
{
foreach (string line in outputData)
{
if (line != null)
{
// The line is output from the process that was invoked.
this.OutputDataReceived(this, new ProcessDataReceivedEventArgs(line));
}
}
}
}
private async Task CancelAndKillProcessTree(bool killProcessOnCancel)
{
ArgUtil.NotNull(_proc, nameof(_proc));
if (!killProcessOnCancel)
{
bool sigint_succeed = await SendSIGINT(_sigintTimeout);
if (sigint_succeed)
{
Trace.Info("Process cancelled successfully through Ctrl+C/SIGINT.");
return;
}
bool sigterm_succeed = await SendSIGTERM(_sigtermTimeout);
if (sigterm_succeed)
{
Trace.Info("Process terminate successfully through Ctrl+Break/SIGTERM.");
return;
}
}
Trace.Info("Kill entire process tree since both cancel and terminate signal has been ignored by the target process.");
KillProcessTree();
}
private async Task<bool> SendSIGINT(TimeSpan timeout)
{
#if OS_WINDOWS
return await SendCtrlSignal(ConsoleCtrlEvent.CTRL_C, timeout);
#else
return await SendSignal(Signals.SIGINT, timeout);
#endif
}
private async Task<bool> SendSIGTERM(TimeSpan timeout)
{
#if OS_WINDOWS
return await SendCtrlSignal(ConsoleCtrlEvent.CTRL_BREAK, timeout);
#else
return await SendSignal(Signals.SIGTERM, timeout);
#endif
}
private void ProcessExitedHandler(object sender, EventArgs e)
{
if ((_proc.StartInfo.RedirectStandardError || _proc.StartInfo.RedirectStandardOutput) && _asyncStreamReaderCount != 0)
{
_waitingOnStreams = true;
Task.Run(async () =>
{
// Wait 5 seconds and then Cancel/Kill process tree
await Task.Delay(TimeSpan.FromSeconds(5));
KillProcessTree();
_processExitedCompletionSource.TrySetResult(true);
_processStandardInWriteCancellationTokenSource.Cancel();
});
}
else
{
_processExitedCompletionSource.TrySetResult(true);
_processStandardInWriteCancellationTokenSource.Cancel();
}
}
private void StartReadStream(StreamReader reader, ConcurrentQueue<string> dataBuffer)
{
Task.Run(() =>
{
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
if (line != null)
{
dataBuffer.Enqueue(line);
_outputProcessEvent.Set();
}
}
Trace.Info("STDOUT/STDERR stream read finished.");
if (Interlocked.Decrement(ref _asyncStreamReaderCount) == 0 && _waitingOnStreams)
{
_processExitedCompletionSource.TrySetResult(true);
_processStandardInWriteCancellationTokenSource.Cancel();
}
});
}
private void StartWriteStream(Channel<string> redirectStandardIn, StreamWriter standardIn, bool keepStandardInOpen)
{
Task.Run(async () =>
{
// Write the contents as UTF8 to handle all characters.
var utf8Writer = new StreamWriter(standardIn.BaseStream, new UTF8Encoding(false));
while (!_processExitedCompletionSource.Task.IsCompleted)
{
ValueTask<string> dequeueTask = redirectStandardIn.Reader.ReadAsync(_processStandardInWriteCancellationTokenSource.Token);
string input = await dequeueTask;
if (input != null)
{
utf8Writer.WriteLine(input);
utf8Writer.Flush();
if (!keepStandardInOpen)
{
Trace.Info("Close STDIN after the first redirect finished.");
standardIn.Close();
break;
}
}
}
Trace.Info("STDIN stream write finished.");
});
}
private void KillProcessTree()
{
#if OS_WINDOWS
WindowsKillProcessTree();
#else
NixKillProcessTree();
#endif
}
private void DecreaseProcessPriority(Process process)
{
#if OS_LINUX
int oomScoreAdj = 500;
string userOomScoreAdj;
if (process.StartInfo.Environment.TryGetValue("PIPELINE_JOB_OOMSCOREADJ", out userOomScoreAdj))
{
int userOomScoreAdjParsed;
if (int.TryParse(userOomScoreAdj, out userOomScoreAdjParsed) && userOomScoreAdjParsed >= -1000 && userOomScoreAdjParsed <= 1000)
{
oomScoreAdj = userOomScoreAdjParsed;
}
else
{
Trace.Info($"Invalid PIPELINE_JOB_OOMSCOREADJ ({userOomScoreAdj}). Valid range is -1000:1000. Using default 500.");
}
}
// Values (up to 1000) make the process more likely to be killed under OOM scenario,
// protecting the agent by extension. Default of 500 is likely to get killed, but can
// be adjusted up or down as appropriate.
WriteProcessOomScoreAdj(process.Id, oomScoreAdj);
#endif
}
#if OS_WINDOWS
private async Task<bool> SendCtrlSignal(ConsoleCtrlEvent signal, TimeSpan timeout)
{
Trace.Info($"Sending {signal} to process {_proc.Id}.");
ConsoleCtrlDelegate ctrlEventHandler = new ConsoleCtrlDelegate(ConsoleCtrlHandler);
try
{
if (!FreeConsole())
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
if (!AttachConsole(_proc.Id))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
if (!SetConsoleCtrlHandler(ctrlEventHandler, true))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
if (!GenerateConsoleCtrlEvent(signal, 0))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
Trace.Info($"Successfully send {signal} to process {_proc.Id}.");
Trace.Info($"Waiting for process exit or {timeout.TotalSeconds} seconds after {signal} signal fired.");
var completedTask = await Task.WhenAny(Task.Delay(timeout), _processExitedCompletionSource.Task);
if (completedTask == _processExitedCompletionSource.Task)
{
Trace.Info("Process exit successfully.");
return true;
}
else
{
Trace.Info($"Process did not honor {signal} signal within {timeout.TotalSeconds} seconds.");
return false;
}
}
catch (Exception ex)
{
Trace.Info($"{signal} signal doesn't fire successfully.");
Trace.Verbose($"Catch exception during send {signal} event to process {_proc.Id}");
Trace.Verbose(ex.ToString());
return false;
}
finally
{
FreeConsole();
SetConsoleCtrlHandler(ctrlEventHandler, false);
}
}
private bool ConsoleCtrlHandler(ConsoleCtrlEvent ctrlType)
{
switch (ctrlType)
{
case ConsoleCtrlEvent.CTRL_C:
Trace.Info($"Ignore Ctrl+C to current process.");
// We return True, so the default Ctrl handler will not take action.
return true;
case ConsoleCtrlEvent.CTRL_BREAK:
Trace.Info($"Ignore Ctrl+Break to current process.");
// We return True, so the default Ctrl handler will not take action.
return true;
}
// If the function handles the control signal, it should return TRUE.
// If it returns FALSE, the next handler function in the list of handlers for this process is used.
return false;
}
private void WindowsKillProcessTree()
{
var pid = _proc?.Id;
if (pid == null)
{
// process already exit, stop here.
return;
}
Dictionary<int, int> processRelationship = new Dictionary<int, int>();
Trace.Info($"Scan all processes to find relationship between all processes.");
foreach (Process proc in Process.GetProcesses())
{
try
{
if (!proc.SafeHandle.IsInvalid)
{
PROCESS_BASIC_INFORMATION pbi = new PROCESS_BASIC_INFORMATION();
int returnLength = 0;
int queryResult = NtQueryInformationProcess(proc.SafeHandle.DangerousGetHandle(), PROCESSINFOCLASS.ProcessBasicInformation, ref pbi, Marshal.SizeOf(pbi), ref returnLength);
if (queryResult == 0) // == 0 is OK
{
Trace.Verbose($"Process: {proc.Id} is child process of {pbi.InheritedFromUniqueProcessId}.");
processRelationship[proc.Id] = (int)pbi.InheritedFromUniqueProcessId;
}
else
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
}
}
catch (Exception ex)
{
// Ignore all exceptions, since KillProcessTree is best effort.
Trace.Verbose("Ignore any catched exception during detecting process relationship.");
Trace.Verbose(ex.ToString());
}
}
Trace.Verbose($"Start killing process tree of process '{pid.Value}'.");
Stack<ProcessTerminationInfo> processesNeedtoKill = new Stack<ProcessTerminationInfo>();
processesNeedtoKill.Push(new ProcessTerminationInfo(pid.Value, false));
while (processesNeedtoKill.Count() > 0)
{
ProcessTerminationInfo procInfo = processesNeedtoKill.Pop();
List<int> childProcessesIds = new List<int>();
if (!procInfo.ChildPidExpanded)
{
Trace.Info($"Find all child processes of process '{procInfo.Pid}'.");
childProcessesIds = processRelationship.Where(p => p.Value == procInfo.Pid).Select(k => k.Key).ToList();
}
if (childProcessesIds.Count > 0)
{
Trace.Info($"Need kill all child processes trees before kill process '{procInfo.Pid}'.");
processesNeedtoKill.Push(new ProcessTerminationInfo(procInfo.Pid, true));
foreach (var childPid in childProcessesIds)
{
Trace.Info($"Child process '{childPid}' needs be killed first.");
processesNeedtoKill.Push(new ProcessTerminationInfo(childPid, false));
}
}
else
{
Trace.Info($"Kill process '{procInfo.Pid}'.");
try
{
Process leafProcess = Process.GetProcessById(procInfo.Pid);
try
{
leafProcess.Kill();
}
catch (InvalidOperationException ex)
{
// The process has already exited
Trace.Verbose("Ignore InvalidOperationException during Process.Kill().");
Trace.Verbose(ex.ToString());
}
catch (Win32Exception ex) when (ex.NativeErrorCode == 5)
{
// The associated process could not be terminated
// The process is terminating
// NativeErrorCode 5 means Access Denied
Trace.Verbose("Ignore Win32Exception with NativeErrorCode 5 during Process.Kill().");
Trace.Verbose(ex.ToString());
}
catch (Exception ex)
{
// Ignore any additional exception
Trace.Verbose("Ignore additional exceptions during Process.Kill().");
Trace.Verbose(ex.ToString());
}
}
catch (ArgumentException ex)
{
// process already gone, nothing needs killed.
Trace.Verbose("Ignore ArgumentException during Process.GetProcessById().");
Trace.Verbose(ex.ToString());
}
catch (Exception ex)
{
// Ignore any additional exception
Trace.Verbose("Ignore additional exceptions during Process.GetProcessById().");
Trace.Verbose(ex.ToString());
}
}
}
}
private class ProcessTerminationInfo
{
public ProcessTerminationInfo(int pid, bool expanded)
{
Pid = pid;
ChildPidExpanded = expanded;
}
public int Pid { get; }
public bool ChildPidExpanded { get; }
}
private enum ConsoleCtrlEvent
{
CTRL_C = 0,
CTRL_BREAK = 1
}
private enum PROCESSINFOCLASS : int
{
ProcessBasicInformation = 0
};
[StructLayout(LayoutKind.Sequential)]
private struct PROCESS_BASIC_INFORMATION
{
public long ExitStatus;
public long PebBaseAddress;
public long AffinityMask;
public long BasePriority;
public long UniqueProcessId;
public long InheritedFromUniqueProcessId;
};
[DllImport("ntdll.dll", SetLastError = true)]
private static extern int NtQueryInformationProcess(IntPtr processHandle, PROCESSINFOCLASS processInformationClass, ref PROCESS_BASIC_INFORMATION processInformation, int processInformationLength, ref int returnLength);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool GenerateConsoleCtrlEvent(ConsoleCtrlEvent sigevent, int dwProcessGroupId);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool FreeConsole();
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool AttachConsole(int dwProcessId);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool SetConsoleCtrlHandler(ConsoleCtrlDelegate HandlerRoutine, bool Add);
// Delegate type to be used as the Handler Routine for SetConsoleCtrlHandler
private delegate Boolean ConsoleCtrlDelegate(ConsoleCtrlEvent CtrlType);
#else
private async Task<bool> SendSignal(Signals signal, TimeSpan timeout)
{
Trace.Info($"Sending {signal} to process {_proc.Id}.");
int errorCode = kill(_proc.Id, (int)signal);
if (errorCode != 0)
{
Trace.Info($"{signal} signal doesn't fire successfully.");
Trace.Info($"Error code: {errorCode}.");
return false;
}
Trace.Info($"Successfully send {signal} to process {_proc.Id}.");
Trace.Info($"Waiting for process exit or {timeout.TotalSeconds} seconds after {signal} signal fired.");
var completedTask = await Task.WhenAny(Task.Delay(timeout), _processExitedCompletionSource.Task);
if (completedTask == _processExitedCompletionSource.Task)
{
Trace.Info("Process exit successfully.");
return true;
}
else
{
Trace.Info($"Process did not honor {signal} signal within {timeout.TotalSeconds} seconds.");
return false;
}
}
private void NixKillProcessTree()
{
try
{
if (_proc?.HasExited == false)
{
_proc?.Kill();
}
}
catch (InvalidOperationException ex)
{
Trace.Info("Ignore InvalidOperationException during Process.Kill().");
Trace.Info(ex.ToString());
}
}
#if OS_LINUX
private void WriteProcessOomScoreAdj(int processId, int oomScoreAdj)
{
try
{
string procFilePath = $"/proc/{processId}/oom_score_adj";
if (File.Exists(procFilePath))
{
File.WriteAllText(procFilePath, oomScoreAdj.ToString());
Trace.Info($"Updated oom_score_adj to {oomScoreAdj} for PID: {processId}.");
}
}
catch (Exception ex)
{
Trace.Info($"Failed to update oom_score_adj for PID: {processId}.");
Trace.Info(ex.ToString());
}
}
#endif
private enum Signals : int
{
SIGINT = 2,
SIGTERM = 15
}
[DllImport("libc", SetLastError = true)]
private static extern int kill(int pid, int sig);
#endif
}
public sealed class ProcessExitCodeException : Exception
{
public int ExitCode { get; private set; }
public ProcessExitCodeException(int exitCode, string fileName, string arguments)
: base($"Exit code {exitCode} returned from process: file name '{fileName}', arguments '{arguments}'.")
{
ExitCode = exitCode;
}
}
public sealed class ProcessDataReceivedEventArgs : EventArgs
{
public ProcessDataReceivedEventArgs(string data)
{
Data = data;
}
public string Data { get; set; }
}
}

View File

@@ -0,0 +1,65 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp2.2</TargetFramework>
<OutputType>Library</OutputType>
<RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm;rhel.6-x64;osx-x64</RuntimeIdentifiers>
<TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
<AssetTargetFallback>portable-net45+win8</AssetTargetFallback>
<NoWarn>NU1701;NU1603</NoWarn>
<Version>$(Version)</Version>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Sdk\Sdk.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
<PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
<PackageReference Include="System.Threading.Channels" Version="4.4.0" />
</ItemGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
<DebugType>portable</DebugType>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(PackageRuntime)' == 'win-x64'">
<DefineConstants>OS_WINDOWS;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(PackageRuntime)' == 'win-x86'">
<DefineConstants>OS_WINDOWS;X86;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'win-x64'">
<DefineConstants>OS_WINDOWS;X64;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'win-x86'">
<DefineConstants>OS_WINDOWS;X86;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">
<DefineConstants>OS_OSX;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true' AND '$(Configuration)' == 'Debug'">
<DefineConstants>OS_OSX;DEBUG;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(PackageRuntime)' == 'linux-x64'">
<DefineConstants>OS_LINUX;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(PackageRuntime)' == 'rhel.6-x64'">
<DefineConstants>OS_LINUX;OS_RHEL6;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(PackageRuntime)' == 'linux-arm'">
<DefineConstants>OS_LINUX;ARM;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'linux-x64'">
<DefineConstants>OS_LINUX;X64;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'rhel.6-x64'">
<DefineConstants>OS_LINUX;OS_RHEL6;X64;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'linux-arm'">
<DefineConstants>OS_LINUX;ARM;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
</Project>

View File

@@ -0,0 +1,40 @@

using System.Security.Cryptography.X509Certificates;
using GitHub.Services.Common;
namespace GitHub.Runner.Sdk
{
public class RunnerCertificateSettings
{
public bool SkipServerCertificateValidation { get; set; }
public string CACertificateFile { get; set; }
public string ClientCertificateFile { get; set; }
public string ClientCertificatePrivateKeyFile { get; set; }
public string ClientCertificateArchiveFile { get; set; }
public string ClientCertificatePassword { get; set; }
public IVssClientCertificateManager VssClientCertificateManager { get; set; }
}
public class RunnerClientCertificateManager : IVssClientCertificateManager
{
private readonly X509Certificate2Collection _clientCertificates = new X509Certificate2Collection();
public X509Certificate2Collection ClientCertificates => _clientCertificates;
public RunnerClientCertificateManager()
{
}
public RunnerClientCertificateManager(string clientCertificateArchiveFile, string clientCertificatePassword)
{
AddClientCertificate(clientCertificateArchiveFile, clientCertificatePassword);
}
public void AddClientCertificate(string clientCertificateArchiveFile, string clientCertificatePassword)
{
if (!string.IsNullOrEmpty(clientCertificateArchiveFile))
{
_clientCertificates.Add(new X509Certificate2(clientCertificateArchiveFile, clientCertificatePassword));
}
}
}
}

View File

@@ -0,0 +1,104 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Text.RegularExpressions;
namespace GitHub.Runner.Sdk
{
public class RunnerWebProxySettings
{
public string ProxyAddress { get; set; }
public string ProxyUsername { get; set; }
public string ProxyPassword { get; set; }
public List<string> ProxyBypassList { get; set; }
public IWebProxy WebProxy { get; set; }
}
public class RunnerWebProxyCore : IWebProxy
{
private string _proxyAddress;
private readonly List<Regex> _regExBypassList = new List<Regex>();
public ICredentials Credentials { get; set; }
public RunnerWebProxyCore()
{
}
public RunnerWebProxyCore(string proxyAddress, string proxyUsername, string proxyPassword, List<string> proxyBypassList)
{
Update(proxyAddress, proxyUsername, proxyPassword, proxyBypassList);
}
public void Update(string proxyAddress, string proxyUsername, string proxyPassword, List<string> proxyBypassList)
{
_proxyAddress = proxyAddress?.Trim();
if (string.IsNullOrEmpty(proxyUsername) || string.IsNullOrEmpty(proxyPassword))
{
Credentials = CredentialCache.DefaultNetworkCredentials;
}
else
{
Credentials = new NetworkCredential(proxyUsername, proxyPassword);
}
if (proxyBypassList != null)
{
foreach (string bypass in proxyBypassList)
{
if (string.IsNullOrWhiteSpace(bypass))
{
continue;
}
else
{
try
{
Regex bypassRegex = new Regex(bypass.Trim(), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.ECMAScript);
_regExBypassList.Add(bypassRegex);
}
catch (Exception)
{
// eat all exceptions
}
}
}
}
}
public Uri GetProxy(Uri destination)
{
if (IsBypassed(destination))
{
return destination;
}
else
{
return new Uri(_proxyAddress);
}
}
public bool IsBypassed(Uri uri)
{
return string.IsNullOrEmpty(_proxyAddress) || uri.IsLoopback || IsMatchInBypassList(uri);
}
private bool IsMatchInBypassList(Uri input)
{
string matchUriString = input.IsDefaultPort ?
input.Scheme + "://" + input.Host :
input.Scheme + "://" + input.Host + ":" + input.Port.ToString();
foreach (Regex r in _regExBypassList)
{
if (r.IsMatch(matchUriString))
{
return true;
}
}
return false;
}
}
}

View File

@@ -0,0 +1,78 @@
using System;
using System.IO;
namespace GitHub.Runner.Sdk
{
public static class ArgUtil
{
public static void Directory(string directory, string name)
{
ArgUtil.NotNullOrEmpty(directory, name);
if (!System.IO.Directory.Exists(directory))
{
throw new DirectoryNotFoundException(
message: $"Directory not found: '{directory}'");
}
}
public static void Equal<T>(T expected, T actual, string name)
{
if (object.ReferenceEquals(expected, actual))
{
return;
}
if (object.ReferenceEquals(expected, null) ||
!expected.Equals(actual))
{
throw new ArgumentOutOfRangeException(
paramName: name,
actualValue: actual,
message: $"{name} does not equal expected value. Expected '{expected}'. Actual '{actual}'.");
}
}
public static void File(string fileName, string name)
{
ArgUtil.NotNullOrEmpty(fileName, name);
if (!System.IO.File.Exists(fileName))
{
throw new FileNotFoundException(
message: $"File not found: '{fileName}'",
fileName: fileName);
}
}
public static void NotNull(object value, string name)
{
if (object.ReferenceEquals(value, null))
{
throw new ArgumentNullException(name);
}
}
public static void NotNullOrEmpty(string value, string name)
{
if (string.IsNullOrEmpty(value))
{
throw new ArgumentNullException(name);
}
}
public static void NotEmpty(Guid value, string name)
{
if (value == Guid.Empty)
{
throw new ArgumentNullException(name);
}
}
public static void Null(object value, string name)
{
if (!object.ReferenceEquals(value, null))
{
throw new ArgumentException(message: $"{name} should be null.", paramName: name);
}
}
}
}

View File

@@ -0,0 +1,467 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
namespace GitHub.Runner.Sdk
{
public static class IOUtil
{
public static string ExeExtension
{
get
{
#if OS_WINDOWS
return ".exe";
#else
return string.Empty;
#endif
}
}
public static StringComparison FilePathStringComparison
{
get
{
#if OS_LINUX
return StringComparison.Ordinal;
#else
return StringComparison.OrdinalIgnoreCase;
#endif
}
}
public static void SaveObject(object obj, string path)
{
File.WriteAllText(path, StringUtil.ConvertToJson(obj), Encoding.UTF8);
}
public static T LoadObject<T>(string path)
{
string json = File.ReadAllText(path, Encoding.UTF8);
return StringUtil.ConvertFromJson<T>(json);
}
public static string GetPathHash(string path)
{
string hashString = path.ToLowerInvariant();
using (SHA256 sha256hash = SHA256.Create())
{
byte[] data = sha256hash.ComputeHash(Encoding.UTF8.GetBytes(hashString));
StringBuilder sBuilder = new StringBuilder();
for (int i = 0; i < data.Length; i++)
{
sBuilder.Append(data[i].ToString("x2"));
}
string hash = sBuilder.ToString();
return hash;
}
}
public static void Delete(string path, CancellationToken cancellationToken)
{
DeleteDirectory(path, cancellationToken);
DeleteFile(path);
}
public static void DeleteDirectory(string path, CancellationToken cancellationToken)
{
DeleteDirectory(path, contentsOnly: false, continueOnContentDeleteError: false, cancellationToken: cancellationToken);
}
public static void DeleteDirectory(string path, bool contentsOnly, bool continueOnContentDeleteError, CancellationToken cancellationToken)
{
ArgUtil.NotNullOrEmpty(path, nameof(path));
DirectoryInfo directory = new DirectoryInfo(path);
if (!directory.Exists)
{
return;
}
if (!contentsOnly)
{
// Remove the readonly flag.
RemoveReadOnly(directory);
// Check if the directory is a reparse point.
if (directory.Attributes.HasFlag(FileAttributes.ReparsePoint))
{
// Delete the reparse point directory and short-circuit.
directory.Delete();
return;
}
}
// Initialize a concurrent stack to store the directories. The directories
// cannot be deleted until the files are deleted.
var directories = new ConcurrentStack<DirectoryInfo>();
if (!contentsOnly)
{
directories.Push(directory);
}
// Create a new token source for the parallel query. The parallel query should be
// canceled after the first error is encountered. Otherwise the number of exceptions
// could get out of control for a large directory with access denied on every file.
using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
try
{
// Recursively delete all files and store all subdirectories.
Enumerate(directory, tokenSource)
.AsParallel()
.WithCancellation(tokenSource.Token)
.ForAll((FileSystemInfo item) =>
{
bool success = false;
try
{
// Remove the readonly attribute.
RemoveReadOnly(item);
// Check if the item is a file.
if (item is FileInfo)
{
// Delete the file.
item.Delete();
}
else
{
// Check if the item is a directory reparse point.
var subdirectory = item as DirectoryInfo;
ArgUtil.NotNull(subdirectory, nameof(subdirectory));
if (subdirectory.Attributes.HasFlag(FileAttributes.ReparsePoint))
{
try
{
// Delete the reparse point.
subdirectory.Delete();
}
catch (DirectoryNotFoundException)
{
// The target of the reparse point directory has been deleted.
// Therefore the item is no longer a directory and is now a file.
//
// Deletion of reparse point directories happens in parallel. This case can occur
// when reparse point directory FOO points to some other reparse point directory BAR,
// and BAR is deleted after the DirectoryInfo for FOO has already been initialized.
File.Delete(subdirectory.FullName);
}
}
else
{
// Store the directory.
directories.Push(subdirectory);
}
}
success = true;
}
catch (Exception) when (continueOnContentDeleteError)
{
// ignore any exception when continueOnContentDeleteError is true.
success = true;
}
finally
{
if (!success)
{
tokenSource.Cancel(); // Cancel is thread-safe.
}
}
});
}
catch (Exception)
{
tokenSource.Cancel();
throw;
}
}
// Delete the directories.
foreach (DirectoryInfo dir in directories.OrderByDescending(x => x.FullName.Length))
{
cancellationToken.ThrowIfCancellationRequested();
dir.Delete();
}
}
public static void DeleteFile(string path)
{
ArgUtil.NotNullOrEmpty(path, nameof(path));
var file = new FileInfo(path);
if (file.Exists)
{
RemoveReadOnly(file);
file.Delete();
}
}
public static void MoveDirectory(string sourceDir, string targetDir, string stagingDir, CancellationToken token)
{
ArgUtil.Directory(sourceDir, nameof(sourceDir));
ArgUtil.NotNullOrEmpty(targetDir, nameof(targetDir));
ArgUtil.NotNullOrEmpty(stagingDir, nameof(stagingDir));
// delete existing stagingDir
DeleteDirectory(stagingDir, token);
// make sure parent dir of stagingDir exist
Directory.CreateDirectory(Path.GetDirectoryName(stagingDir));
// move source to staging
Directory.Move(sourceDir, stagingDir);
// delete existing targetDir
DeleteDirectory(targetDir, token);
// make sure parent dir of targetDir exist
Directory.CreateDirectory(Path.GetDirectoryName(targetDir));
// move staging to target
Directory.Move(stagingDir, targetDir);
}
/// <summary>
/// Given a path and directory, return the path relative to the directory. If the path is not
/// under the directory the path is returned un modified. Examples:
/// MakeRelative(@"d:\src\project\foo.cpp", @"d:\src") -> @"project\foo.cpp"
/// MakeRelative(@"d:\src\project\foo.cpp", @"d:\specs") -> @"d:\src\project\foo.cpp"
/// MakeRelative(@"d:\src\project\foo.cpp", @"d:\src\proj") -> @"d:\src\project\foo.cpp"
/// </summary>
/// <remarks>Safe for remote paths. Does not access the local disk.</remarks>
/// <param name="path">Path to make relative.</param>
/// <param name="folder">Folder to make it relative to.</param>
/// <returns>Relative path.</returns>
public static string MakeRelative(string path, string folder)
{
ArgUtil.NotNullOrEmpty(path, nameof(path));
ArgUtil.NotNull(folder, nameof(folder));
// Replace all Path.AltDirectorySeparatorChar with Path.DirectorySeparatorChar from both inputs
path = path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
folder = folder.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
// Check if the dir is a prefix of the path (if not, it isn't relative at all).
if (!path.StartsWith(folder, IOUtil.FilePathStringComparison))
{
return path;
}
// Dir is a prefix of the path, if they are the same length then the relative path is empty.
if (path.Length == folder.Length)
{
return string.Empty;
}
// If the dir ended in a '\\' (like d:\) or '/' (like user/bin/) then we have a relative path.
if (folder.Length > 0 && folder[folder.Length - 1] == Path.DirectorySeparatorChar)
{
return path.Substring(folder.Length);
}
// The next character needs to be a '\\' or they aren't really relative.
else if (path[folder.Length] == Path.DirectorySeparatorChar)
{
return path.Substring(folder.Length + 1);
}
else
{
return path;
}
}
public static string ResolvePath(String rootPath, String relativePath)
{
ArgUtil.NotNullOrEmpty(rootPath, nameof(rootPath));
ArgUtil.NotNullOrEmpty(relativePath, nameof(relativePath));
if (!Path.IsPathRooted(rootPath))
{
throw new ArgumentException($"{rootPath} should be a rooted path.");
}
if (relativePath.IndexOfAny(Path.GetInvalidPathChars()) > -1)
{
throw new InvalidOperationException($"{relativePath} contains invalid path characters.");
}
else if (Path.GetFileName(relativePath).IndexOfAny(Path.GetInvalidFileNameChars()) > -1)
{
throw new InvalidOperationException($"{relativePath} contains invalid folder name characters.");
}
else if (Path.IsPathRooted(relativePath))
{
throw new InvalidOperationException($"{relativePath} can not be a rooted path.");
}
else
{
rootPath = rootPath.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
relativePath = relativePath.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
// Root the path
relativePath = String.Concat(rootPath, Path.AltDirectorySeparatorChar, relativePath);
// Collapse ".." directories with their parent, and skip "." directories.
String[] split = relativePath.Split(new[] { Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries);
var segments = new Stack<String>(split.Length);
Int32 skip = 0;
for (Int32 i = split.Length - 1; i >= 0; i--)
{
String segment = split[i];
if (String.Equals(segment, ".", StringComparison.Ordinal))
{
continue;
}
else if (String.Equals(segment, "..", StringComparison.Ordinal))
{
skip++;
}
else if (skip > 0)
{
skip--;
}
else
{
segments.Push(segment);
}
}
if (skip > 0)
{
throw new InvalidOperationException($"The file path {relativePath} is invalid");
}
#if OS_WINDOWS
if (segments.Count > 1)
{
return String.Join(Path.DirectorySeparatorChar, segments);
}
else
{
return segments.Pop() + Path.DirectorySeparatorChar;
}
#else
return Path.DirectorySeparatorChar + String.Join(Path.DirectorySeparatorChar, segments);
#endif
}
}
public static void CopyDirectory(string source, string target, CancellationToken cancellationToken)
{
// Validate args.
ArgUtil.Directory(source, nameof(source));
ArgUtil.NotNullOrEmpty(target, nameof(target));
ArgUtil.NotNull(cancellationToken, nameof(cancellationToken));
cancellationToken.ThrowIfCancellationRequested();
// Create the target directory.
Directory.CreateDirectory(target);
// Get the file contents of the directory to copy.
DirectoryInfo sourceDir = new DirectoryInfo(source);
foreach (FileInfo sourceFile in sourceDir.GetFiles() ?? new FileInfo[0])
{
// Check if the file already exists.
cancellationToken.ThrowIfCancellationRequested();
FileInfo targetFile = new FileInfo(Path.Combine(target, sourceFile.Name));
if (!targetFile.Exists ||
sourceFile.Length != targetFile.Length ||
sourceFile.LastWriteTime != targetFile.LastWriteTime)
{
// Copy the file.
sourceFile.CopyTo(targetFile.FullName, true);
}
}
// Copy the subdirectories.
foreach (DirectoryInfo subDir in sourceDir.GetDirectories() ?? new DirectoryInfo[0])
{
CopyDirectory(
source: subDir.FullName,
target: Path.Combine(target, subDir.Name),
cancellationToken: cancellationToken);
}
}
public static void ValidateExecutePermission(string directory)
{
ArgUtil.Directory(directory, nameof(directory));
string dir = directory;
string failsafeString = Environment.GetEnvironmentVariable("AGENT_TEST_VALIDATE_EXECUTE_PERMISSIONS_FAILSAFE");
int failsafe;
if (string.IsNullOrEmpty(failsafeString) || !int.TryParse(failsafeString, out failsafe))
{
failsafe = 100;
}
for (int i = 0; i < failsafe; i++)
{
try
{
Directory.EnumerateFileSystemEntries(dir).FirstOrDefault();
}
catch (UnauthorizedAccessException ex)
{
// Permission to read the directory contents is required for '{0}' and each directory up the hierarchy. {1}
string message = $"Permission to read the directory contents is required for '{directory}' and each directory up the hierarchy. {ex.Message}";
throw new UnauthorizedAccessException(message, ex);
}
dir = Path.GetDirectoryName(dir);
if (string.IsNullOrEmpty(dir))
{
return;
}
}
// This should never happen.
throw new NotSupportedException($"Unable to validate execute permissions for directory '{directory}'. Exceeded maximum iterations.");
}
/// <summary>
/// Recursively enumerates a directory without following directory reparse points.
/// </summary>
private static IEnumerable<FileSystemInfo> Enumerate(DirectoryInfo directory, CancellationTokenSource tokenSource)
{
ArgUtil.NotNull(directory, nameof(directory));
ArgUtil.Equal(false, directory.Attributes.HasFlag(FileAttributes.ReparsePoint), nameof(directory.Attributes.HasFlag));
// Push the directory onto the processing stack.
var directories = new Stack<DirectoryInfo>(new[] { directory });
while (directories.Count > 0)
{
// Pop the next directory.
directory = directories.Pop();
foreach (FileSystemInfo item in directory.GetFileSystemInfos())
{
// Push non-reparse-point directories onto the processing stack.
directory = item as DirectoryInfo;
if (directory != null &&
!item.Attributes.HasFlag(FileAttributes.ReparsePoint))
{
directories.Push(directory);
}
// Then yield the directory. Otherwise there is a race condition when this method attempts to initialize
// the Attributes and the caller is deleting the reparse point in parallel (FileNotFoundException).
yield return item;
}
}
}
private static void RemoveReadOnly(FileSystemInfo item)
{
ArgUtil.NotNull(item, nameof(item));
if (item.Attributes.HasFlag(FileAttributes.ReadOnly))
{
item.Attributes = item.Attributes & ~FileAttributes.ReadOnly;
}
}
}
}

View File

@@ -0,0 +1,36 @@
using System;
using System.IO;
namespace GitHub.Runner.Sdk
{
public static class PathUtil
{
#if OS_WINDOWS
public static readonly string PathVariable = "Path";
#else
public static readonly string PathVariable = "PATH";
#endif
public static string PrependPath(string path, string currentPath)
{
ArgUtil.NotNullOrEmpty(path, nameof(path));
if (string.IsNullOrEmpty(currentPath))
{
// Careful not to add a trailing separator if the PATH is empty.
// On OSX/Linux, a trailing separator indicates that "current directory"
// is added to the PATH, which is considered a security risk.
return path;
}
// Not prepend path if it is already the first path in %PATH%
if (currentPath.StartsWith(path + Path.PathSeparator, IOUtil.FilePathStringComparison))
{
return currentPath;
}
else
{
return path + Path.PathSeparator + currentPath;
}
}
}
}

View File

@@ -0,0 +1,126 @@
using GitHub.Services.WebApi;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Text;
namespace GitHub.Runner.Sdk
{
public static class StringUtil
{
private static readonly object[] s_defaultFormatArgs = new object[] { null };
private static Lazy<JsonSerializerSettings> s_serializerSettings = new Lazy<JsonSerializerSettings>(() =>
{
var settings = new VssJsonMediaTypeFormatter().SerializerSettings;
settings.DateParseHandling = DateParseHandling.None;
settings.FloatParseHandling = FloatParseHandling.Double;
return settings;
});
static StringUtil()
{
#if OS_WINDOWS
// By default, only Unicode encodings, ASCII, and code page 28591 are supported.
// This line is required to support the full set of encodings that were included
// in Full .NET prior to 4.6.
//
// For example, on an en-US box, this is required for loading the encoding for the
// default console output code page '437'. Without loading the correct encoding for
// code page IBM437, some characters cannot be translated correctly, e.g. write 'ç'
// from powershell.exe.
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
#endif
}
public static T ConvertFromJson<T>(string value)
{
return JsonConvert.DeserializeObject<T>(value, s_serializerSettings.Value);
}
/// <summary>
/// Convert String to boolean, valid true string: "1", "true", "$true", valid false string: "0", "false", "$false".
/// </summary>
/// <param name="value">value to convert.</param>
/// <param name="defaultValue">default result when value is null or empty or not a valid true/false string.</param>
/// <returns></returns>
public static bool ConvertToBoolean(string value, bool defaultValue = false)
{
if (string.IsNullOrEmpty(value))
{
return defaultValue;
}
switch (value.ToLowerInvariant())
{
case "1":
case "true":
case "$true":
return true;
case "0":
case "false":
case "$false":
return false;
default:
return defaultValue;
}
}
public static string ConvertToJson(object obj, Formatting formatting = Formatting.Indented)
{
return JsonConvert.SerializeObject(obj, formatting, s_serializerSettings.Value);
}
public static void EnsureRegisterEncodings()
{
// The static constructor should have registered the required encodings.
}
public static string Format(string format, params object[] args)
{
return Format(CultureInfo.InvariantCulture, format, args);
}
public static Encoding GetSystemEncoding()
{
#if OS_WINDOWS
// The static constructor should have registered the required encodings.
// Code page 0 is equivalent to the current system default (i.e. CP_ACP).
// E.g. code page 1252 on an en-US box.
return Encoding.GetEncoding(0);
#else
throw new NotSupportedException(nameof(GetSystemEncoding)); // Should never reach here.
#endif
}
private static string Format(CultureInfo culture, string format, params object[] args)
{
try
{
// 1) Protect against argument null exception for the format parameter.
// 2) Protect against argument null exception for the args parameter.
// 3) Coalesce null or empty args with an array containing one null element.
// This protects against format exceptions where string.Format thinks
// that not enough arguments were supplied, even though the intended arg
// literally is null or an empty array.
return string.Format(
culture,
format ?? string.Empty,
args == null || args.Length == 0 ? s_defaultFormatArgs : args);
}
catch (FormatException)
{
// TODO: Log that string format failed. Consider moving this into a context base class if that's the only place it's used. Then the current trace scope would be available as well.
if (args != null)
{
return string.Format(culture, "{0} {1}", format, string.Join(", ", args));
}
return format;
}
}
}
}

View File

@@ -0,0 +1,37 @@
using System;
namespace GitHub.Runner.Sdk
{
public static class UrlUtil
{
public static Uri GetCredentialEmbeddedUrl(Uri baseUrl, string username, string password)
{
ArgUtil.NotNull(baseUrl, nameof(baseUrl));
// return baseurl when there is no username and password
if (string.IsNullOrEmpty(username) && string.IsNullOrEmpty(password))
{
return baseUrl;
}
UriBuilder credUri = new UriBuilder(baseUrl);
// ensure we have a username, uribuild will throw if username is empty but password is not.
if (string.IsNullOrEmpty(username))
{
username = "emptyusername";
}
// escape chars in username for uri
credUri.UserName = Uri.EscapeDataString(username);
// escape chars in password for uri
if (!string.IsNullOrEmpty(password))
{
credUri.Password = Uri.EscapeDataString(password);
}
return credUri.Uri;
}
}
}

View File

@@ -0,0 +1,99 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Net.Http;
using GitHub.DistributedTask.WebApi;
using GitHub.Services.Common;
using GitHub.Services.WebApi;
using GitHub.Services.OAuth;
using System.Net.Http.Headers;
using System.Runtime.InteropServices;
using System.Net;
namespace GitHub.Runner.Sdk
{
public static class VssUtil
{
public static void InitializeVssClientSettings(ProductInfoHeaderValue additionalUserAgent, IWebProxy proxy, IVssClientCertificateManager clientCert)
{
var headerValues = new List<ProductInfoHeaderValue>();
headerValues.Add(additionalUserAgent);
headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})"));
if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0)
{
headerValues.AddRange(VssClientHttpRequestSettings.Default.UserAgent);
}
VssClientHttpRequestSettings.Default.UserAgent = headerValues;
VssClientHttpRequestSettings.Default.ClientCertificateManager = clientCert;
#if OS_LINUX || OS_OSX
// The .NET Core 2.1 runtime switched its HTTP default from HTTP 1.1 to HTTP 2.
// This causes problems with some versions of the Curl handler.
// See GitHub issue https://github.com/dotnet/corefx/issues/32376
VssClientHttpRequestSettings.Default.UseHttp11 = true;
#endif
VssHttpMessageHandler.DefaultWebProxy = proxy;
}
public static VssConnection CreateConnection(Uri serverUri, VssCredentials credentials, IEnumerable<DelegatingHandler> additionalDelegatingHandler = null, TimeSpan? timeout = null)
{
VssClientHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone();
int maxRetryRequest;
if (!int.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_HTTP_RETRY") ?? string.Empty, out maxRetryRequest))
{
maxRetryRequest = 3;
}
// make sure MaxRetryRequest in range [3, 10]
settings.MaxRetryRequest = Math.Min(Math.Max(maxRetryRequest, 3), 10);
if (!int.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_HTTP_TIMEOUT") ?? string.Empty, out int httpRequestTimeoutSeconds))
{
settings.SendTimeout = timeout ?? TimeSpan.FromSeconds(100);
}
else
{
// prefer environment variable
settings.SendTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200));
}
// Remove Invariant from the list of accepted languages.
//
// The constructor of VssHttpRequestSettings (base class of VssClientHttpRequestSettings) adds the current
// UI culture to the list of accepted languages. The UI culture will be Invariant on OSX/Linux when the
// LANG environment variable is not set when the program starts. If Invariant is in the list of accepted
// languages, then "System.ArgumentException: The value cannot be null or empty." will be thrown when the
// settings are applied to an HttpRequestMessage.
settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture);
VssConnection connection = new VssConnection(serverUri, new VssHttpMessageHandler(credentials, settings), additionalDelegatingHandler);
return connection;
}
public static VssCredentials GetVssCredential(ServiceEndpoint serviceEndpoint)
{
ArgUtil.NotNull(serviceEndpoint, nameof(serviceEndpoint));
ArgUtil.NotNull(serviceEndpoint.Authorization, nameof(serviceEndpoint.Authorization));
ArgUtil.NotNullOrEmpty(serviceEndpoint.Authorization.Scheme, nameof(serviceEndpoint.Authorization.Scheme));
if (serviceEndpoint.Authorization.Parameters.Count == 0)
{
throw new ArgumentOutOfRangeException(nameof(serviceEndpoint));
}
VssCredentials credentials = null;
string accessToken;
if (serviceEndpoint.Authorization.Scheme == EndpointAuthorizationSchemes.OAuth &&
serviceEndpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out accessToken))
{
credentials = new VssCredentials(null, new VssOAuthAccessTokenCredential(accessToken), CredentialPromptType.DoNotPrompt);
}
return credentials;
}
}
}

View File

@@ -0,0 +1,120 @@
using System;
using System.IO;
using System.Linq;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Sdk
{
public static class WhichUtil
{
public static string Which(string command, bool require = false, ITraceWriter trace = null)
{
ArgUtil.NotNullOrEmpty(command, nameof(command));
trace?.Info($"Which: '{command}'");
string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
if (string.IsNullOrEmpty(path))
{
trace?.Info("PATH environment variable not defined.");
path = path ?? string.Empty;
}
string[] pathSegments = path.Split(new Char[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < pathSegments.Length; i++)
{
pathSegments[i] = Environment.ExpandEnvironmentVariables(pathSegments[i]);
}
foreach (string pathSegment in pathSegments)
{
if (!string.IsNullOrEmpty(pathSegment) && Directory.Exists(pathSegment))
{
string[] matches = null;
#if OS_WINDOWS
string pathExt = Environment.GetEnvironmentVariable("PATHEXT");
if (string.IsNullOrEmpty(pathExt))
{
// XP's system default value for PATHEXT system variable
pathExt = ".com;.exe;.bat;.cmd;.vbs;.vbe;.js;.jse;.wsf;.wsh";
}
string[] pathExtSegments = pathExt.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
// if command already has an extension.
if (pathExtSegments.Any(ext => command.EndsWith(ext, StringComparison.OrdinalIgnoreCase)))
{
try
{
matches = Directory.GetFiles(pathSegment, command);
}
catch (UnauthorizedAccessException ex)
{
trace?.Info("Ignore UnauthorizedAccess exception during Which.");
trace?.Verbose(ex.ToString());
}
if (matches != null && matches.Length > 0)
{
trace?.Info($"Location: '{matches.First()}'");
return matches.First();
}
}
else
{
string searchPattern;
searchPattern = StringUtil.Format($"{command}.*");
try
{
matches = Directory.GetFiles(pathSegment, searchPattern);
}
catch (UnauthorizedAccessException ex)
{
trace?.Info("Ignore UnauthorizedAccess exception during Which.");
trace?.Verbose(ex.ToString());
}
if (matches != null && matches.Length > 0)
{
// add extension.
for (int i = 0; i < pathExtSegments.Length; i++)
{
string fullPath = Path.Combine(pathSegment, $"{command}{pathExtSegments[i]}");
if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)))
{
trace?.Info($"Location: '{fullPath}'");
return fullPath;
}
}
}
}
#else
try
{
matches = Directory.GetFiles(pathSegment, command);
}
catch (UnauthorizedAccessException ex)
{
trace?.Info("Ignore UnauthorizedAccess exception during Which.");
trace?.Verbose(ex.ToString());
}
if (matches != null && matches.Length > 0)
{
trace?.Info($"Location: '{matches.First()}'");
return matches.First();
}
#endif
}
}
trace?.Info("Not found.");
if (require)
{
throw new FileNotFoundException(
message: $"File not found: '{command}'",
fileName: command);
}
return null;
}
}
}