Compare commits

..

8 Commits

Author SHA1 Message Date
Lokesh Gopu
17a36d58be Max attempts while adding runner 2020-05-19 17:46:20 -04:00
PJ Quirk
f798f5606b Use the API_URL and munge action URLs for GHES (#437)
* First pass at logic for GHES, not all correct

* Need to mock out file downloading

* Allowed for mocking of HTTP responses

* Added test for builtin GHES action download

* More tests

* Don't retry on action 404

* Remove commented out code

* Add a using statement back, because Windows

* Make windows happy again

* Another windows fix

* Always delete the cache since it isn't fully implemented

* Use RunnerService base class

* Add examples, update URL path

* Remove forceDotCom

* Fix a bug

* Remove a test that's no longer relevant

* PR feedback

* Add missing return

* More trace info

* Use the new agreed-upon format

* Use the auth token since we're hitting GHES directly

* Fixing tests on windows

* Fixed one more test
2020-04-23 17:02:13 -04:00
Tingluo Huang
3f7a01af93 add secret masker for trimming double qoutes. (#440) 2020-04-21 22:07:55 -04:00
Tingluo Huang
d5c54f9819 Raise warning when action input does not match action.yml. (#429) 2020-04-21 19:42:53 -04:00
Mathias LANG
9f78ad3b34 Make release notes code blocks copy-paste-able (#430)
The release notes used C-style comments (`//`) instead of shell-style (`#`),
causing the code snippet to not be easily copy-paste-able.

Additionally, the code fence for Windows had the extra `powershell` added,
as well as a comment to direct users towards `powershell` over `cmd`.
2020-04-19 22:11:44 -04:00
Jan Pazdziora
97883c8cd5 Fix spelling of RHEL and CentOS. (#436) 2020-04-19 16:53:06 -04:00
Tingluo Huang
c5fa9fb062 Print node version in debug instead of output. (#433) 2020-04-17 11:53:51 -04:00
Bryan MacFarlane
b2dcdc21dc Sample scripts to automate scaleable runners (#427) 2020-04-17 11:08:45 -04:00
15 changed files with 547 additions and 134 deletions

View File

@@ -40,7 +40,7 @@ Debian based OS (Debian, Ubuntu, Linux Mint)
- libssl1.1, libssl1.0.2 or libssl1.0.0
- libicu63, libicu60, libicu57 or libicu55
Fedora based OS (Fedora, Redhat, Centos, Oracle Linux 7)
Fedora based OS (Fedora, Red Hat Enterprise Linux, CentOS, Oracle Linux 7)
- lttng-ust
- openssl-libs

View File

@@ -8,13 +8,15 @@
- N/A
## Windows x64
We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows
```
// Create a folder under the drive root
We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.
The following snipped needs to be run on `powershell`:
``` powershell
# Create a folder under the drive root
mkdir \actions-runner ; cd \actions-runner
// Download the latest runner package
# Download the latest runner package
Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-win-x64-<RUNNER_VERSION>.zip -OutFile actions-runner-win-x64-<RUNNER_VERSION>.zip
// Extract the installer
# Extract the installer
Add-Type -AssemblyName System.IO.Compression.FileSystem ;
[System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-<RUNNER_VERSION>.zip", "$PWD")
```
@@ -22,44 +24,44 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ;
## OSX
``` bash
// Create a folder
# Create a folder
mkdir actions-runner && cd actions-runner
// Download the latest runner package
# Download the latest runner package
curl -O -L https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz
// Extract the installer
# Extract the installer
tar xzf ./actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz
```
## Linux x64
``` bash
// Create a folder
# Create a folder
mkdir actions-runner && cd actions-runner
// Download the latest runner package
# Download the latest runner package
curl -O -L https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz
// Extract the installer
# Extract the installer
tar xzf ./actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz
```
## Linux arm64 (Pre-release)
``` bash
// Create a folder
# Create a folder
mkdir actions-runner && cd actions-runner
// Download the latest runner package
# Download the latest runner package
curl -O -L https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-linux-arm64-<RUNNER_VERSION>.tar.gz
// Extract the installer
# Extract the installer
tar xzf ./actions-runner-linux-arm64-<RUNNER_VERSION>.tar.gz
```
## Linux arm (Pre-release)
``` bash
// Create a folder
# Create a folder
mkdir actions-runner && cd actions-runner
// Download the latest runner package
# Download the latest runner package
curl -O -L https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-linux-arm-<RUNNER_VERSION>.tar.gz
// Extract the installer
# Extract the installer
tar xzf ./actions-runner-linux-arm-<RUNNER_VERSION>.tar.gz
```

View File

@@ -9,7 +9,7 @@ fi
# Determine OS type
# Debian based OS (Debian, Ubuntu, Linux Mint) has /etc/debian_version
# Fedora based OS (Fedora, Redhat, Centos, Oracle Linux 7) has /etc/redhat-release
# Fedora based OS (Fedora, Red Hat Enterprise Linux, CentOS, Oracle Linux 7) has /etc/redhat-release
# SUSE based OS (OpenSUSE, SUSE Enterprise) has ID_LIKE=suse in /etc/os-release
function print_errormessage()
@@ -116,12 +116,12 @@ then
elif [ -e /etc/redhat-release ]
then
echo "The current OS is Fedora based"
echo "--------Redhat Version--------"
echo "--Fedora/RHEL/CentOS Version--"
cat /etc/redhat-release
echo "------------------------------"
# use dnf on fedora
# use yum on centos and redhat
# use yum on centos and rhel
if [ -e /etc/fedora-release ]
then
command -v dnf
@@ -191,7 +191,7 @@ then
redhatRelease=$(</etc/redhat-release)
if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]
then
echo "The current OS is Red Hat Enterprise Linux 6 or Centos 6"
echo "The current OS is Red Hat Enterprise Linux 6 or CentOS 6"
# Install known dependencies, as a best effort.
# The remaining dependencies are covered by the GitHub doc that will be shown by `print_rhel6message`

View File

@@ -1,19 +1,18 @@
using GitHub.Runner.Common.Util;
using System;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.Tracing;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection;
using System.Runtime.Loader;
using System.Threading;
using System.Threading.Tasks;
using System.Diagnostics;
using System.Net.Http;
using System.Diagnostics.Tracing;
using GitHub.DistributedTask.Logging;
using System.Net.Http.Headers;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
@@ -89,6 +88,7 @@ namespace GitHub.Runner.Common
this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape);
this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape);
this.SecretMasker.AddValueEncoder(ValueEncoders.XmlDataEscape);
this.SecretMasker.AddValueEncoder(ValueEncoders.TrimDoubleQuotes);
// Create the trace manager.
if (string.IsNullOrEmpty(logFile))
@@ -614,9 +614,8 @@ namespace GitHub.Runner.Common
{
public static HttpClientHandler CreateHttpClientHandler(this IHostContext context)
{
HttpClientHandler clientHandler = new HttpClientHandler();
clientHandler.Proxy = context.WebProxy;
return clientHandler;
var handlerFactory = context.GetService<IHttpClientHandlerFactory>();
return handlerFactory.CreateClientHandler(context.WebProxy);
}
}

View File

@@ -0,0 +1,19 @@
using System.Net.Http;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(HttpClientHandlerFactory))]
public interface IHttpClientHandlerFactory : IRunnerService
{
HttpClientHandler CreateClientHandler(RunnerWebProxy webProxy);
}
public class HttpClientHandlerFactory : RunnerService, IHttpClientHandlerFactory
{
public HttpClientHandler CreateClientHandler(RunnerWebProxy webProxy)
{
return new HttpClientHandler() { Proxy = webProxy };
}
}
}

View File

@@ -160,8 +160,11 @@ namespace GitHub.Runner.Listener.Configuration
}
TaskAgent agent;
int attemptCount = 5;
while (true)
{
attemptCount--;
runnerSettings.AgentName = command.GetRunnerName();
_term.WriteLine();
@@ -192,10 +195,10 @@ namespace GitHub.Runner.Listener.Configuration
_term.WriteError("Failed to replace the runner. Try again or ctrl-c to quit");
}
}
else if (command.Unattended)
else if (command.Unattended || attemptCount <= 0)
{
// if not replace and it is unattended config.
throw new TaskAgentExistsException($"Pool {runnerSettings.PoolId} already contains a runner with name {runnerSettings.AgentName}.");
throw new TaskAgentExistsException($"A runner already exists with the name {runnerSettings.AgentName}.");
}
}
else
@@ -216,6 +219,7 @@ namespace GitHub.Runner.Listener.Configuration
}
}
}
// Add Agent Id to settings
runnerSettings.AgentId = agent.Id;

View File

@@ -1,21 +1,19 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Runner.Worker.Container;
using GitHub.Services.Common;
using Newtonsoft.Json;
using Pipelines = GitHub.DistributedTask.Pipelines;
using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants;
@@ -73,13 +71,7 @@ namespace GitHub.Runner.Worker
}
// Clear the cache (for self-hosted runners)
// Note, temporarily avoid this step for the on-premises product, to avoid rate limiting.
var configurationStore = HostContext.GetService<IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
}
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
foreach (var action in actions)
{
@@ -490,7 +482,7 @@ namespace GitHub.Runner.Worker
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string watermarkFile = destDirectory + ".completed";
string watermarkFile = GetWatermarkFilePath(destDirectory);
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
@@ -504,27 +496,84 @@ namespace GitHub.Runner.Worker
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
}
#if OS_WINDOWS
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}";
#else
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}";
#endif
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
var configurationStore = HostContext.GetService<IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
string apiUrl = GetApiUrl(executionContext);
string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref);
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory);
return;
}
else
{
string apiUrl = GetApiUrl(executionContext);
// URLs to try:
var archiveLinks = new List<string> {
// A built-in action or an action the user has created, on their GHES instance
// Example: https://my-ghes/api/v3/repos/my-org/my-action/tarball/v1
BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref),
// A community action, synced to their GHES instance
// Example: https://my-ghes/api/v3/repos/actions-community/some-org-some-action/tarball/v1
BuildLinkToActionArchive(apiUrl, $"actions-community/{repositoryReference.Name.Replace("/", "-")}", repositoryReference.Ref)
};
foreach (var archiveLink in archiveLinks)
{
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
try
{
await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory);
return;
}
catch (ActionNotFoundException)
{
Trace.Info($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}' at {archiveLink}");
continue;
}
}
throw new ActionNotFoundException($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}'. Paths attempted: {string.Join(", ", archiveLinks)}");
}
}
private string GetApiUrl(IExecutionContext executionContext)
{
string apiUrl = executionContext.GetGitHubContext("api_url");
if (!string.IsNullOrEmpty(apiUrl))
{
return apiUrl;
}
// Once the api_url is set for hosted, we can remove this fallback (it doesn't make sense for GHES)
return "https://api.github.com";
}
private static string BuildLinkToActionArchive(string apiUrl, string repository, string @ref)
{
#if OS_WINDOWS
return $"{apiUrl}/repos/{repository}/zipball/{@ref}";
#else
return $"{apiUrl}/repos/{repository}/tarball/{@ref}";
#endif
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, string link, string destDirectory)
{
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
Trace.Info($"Save archive '{archiveLink}' into {archiveFile}.");
Trace.Info($"Save archive '{link}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
@@ -541,64 +590,76 @@ namespace GitHub.Runner.Worker
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var configurationStore = HostContext.GetService<IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}
if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
// Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated.
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
using (var result = await httpClient.GetStreamAsync(archiveLink))
using (var response = await httpClient.GetAsync(link))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
if (response.IsSuccessStatusCode)
{
using (var result = await response.Content.ReadAsStreamAsync())
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
// download succeed, break out the retry loop.
break;
}
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
// It doesn't make sense to retry in this case, so just stop
throw new ActionNotFoundException(new Uri(link));
}
else
{
// Something else bad happened, let's go to our retry logic
response.EnsureSuccessStatusCode();
}
}
}
}
catch (OperationCanceledException) when (executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Action download has been cancelled.");
Trace.Info("Action download has been cancelled.");
throw;
}
catch (ActionNotFoundException)
{
Trace.Info($"The action at '{link}' does not exist");
throw;
}
catch (Exception ex) when (retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}");
Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds.");
executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}");
executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}");
}
}
}
@@ -612,7 +673,7 @@ namespace GitHub.Runner.Worker
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'");
executionContext.Debug($"Download '{link}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
@@ -662,6 +723,7 @@ namespace GitHub.Runner.Worker
}
Trace.Verbose("Create watermark file indicate action download succeed.");
string watermarkFile = GetWatermarkFilePath(destDirectory);
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
@@ -686,6 +748,8 @@ namespace GitHub.Runner.Worker
}
}
private string GetWatermarkFilePath(string directory) => directory + ".completed";
private ActionContainer PrepareRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
@@ -931,4 +995,3 @@ namespace GitHub.Runner.Worker
public string ActionRepository { get; set; }
}
}

View File

@@ -0,0 +1,33 @@
using System;
using System.Runtime.Serialization;
namespace GitHub.Runner.Worker
{
public class ActionNotFoundException : Exception
{
public ActionNotFoundException(Uri actionUri)
: base(FormatMessage(actionUri))
{
}
public ActionNotFoundException(string message)
: base(message)
{
}
public ActionNotFoundException(string message, System.Exception inner)
: base(message, inner)
{
}
protected ActionNotFoundException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
}
private static string FormatMessage(Uri actionUri)
{
return $"An action could not be found at the URI '{actionUri}'";
}
}
}

View File

@@ -143,8 +143,10 @@ namespace GitHub.Runner.Worker
var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator();
var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions);
var userInputs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (KeyValuePair<string, string> input in inputs)
{
userInputs.Add(input.Key);
string message = "";
if (definition.Data?.Deprecated?.TryGetValue(input.Key, out message) == true)
{
@@ -152,13 +154,15 @@ namespace GitHub.Runner.Worker
}
}
var validInputs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
// Merge the default inputs from the definition
if (definition.Data?.Inputs != null)
{
var manifestManager = HostContext.GetService<IActionManifestManager>();
foreach (var input in (definition.Data?.Inputs))
foreach (var input in definition.Data.Inputs)
{
string key = input.Key.AssertString("action input name").Value;
validInputs.Add(key);
if (!inputs.ContainsKey(key))
{
inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value);
@@ -166,6 +170,14 @@ namespace GitHub.Runner.Worker
}
}
foreach (var input in userInputs)
{
if (!validInputs.Contains(input))
{
ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']");
}
}
// Load the action environment.
ExecutionContext.Debug("Loading env");
var environment = new Dictionary<String, String>(VarUtil.EnvironmentVariableKeyComparer);

View File

@@ -149,14 +149,14 @@ namespace GitHub.Runner.Worker.Handlers
throw new NotSupportedException(msg);
}
nodeExternal = "node12_alpine";
executionContext.Output($"Container distribution is alpine. Running JavaScript Action with external tool: {nodeExternal}");
executionContext.Debug($"Container distribution is alpine. Running JavaScript Action with external tool: {nodeExternal}");
return nodeExternal;
}
}
}
// Optimistically use the default
nodeExternal = "node12";
executionContext.Output($"Running JavaScript Action with default external tool: {nodeExternal}");
executionContext.Debug($"Running JavaScript Action with default external tool: {nodeExternal}");
return nodeExternal;
}

View File

@@ -60,6 +60,20 @@ namespace GitHub.DistributedTask.Logging
return SecurityElement.Escape(value);
}
public static String TrimDoubleQuotes(String value)
{
var trimmed = string.Empty;
if (!string.IsNullOrEmpty(value) &&
value.Length > 8 &&
value.StartsWith('"') &&
value.EndsWith('"'))
{
trimmed = value.Substring(1, value.Length - 2);
}
return trimmed;
}
private static string Base64StringEscapeShift(String value, int shift)
{
var bytes = Encoding.UTF8.GetBytes(value);

View File

@@ -85,6 +85,8 @@ namespace GitHub.Runner.Common.Tests
_hc.SecretMasker.AddValue("Pass word 123!");
_hc.SecretMasker.AddValue("Pass<word>123!");
_hc.SecretMasker.AddValue("Pass'word'123!");
_hc.SecretMasker.AddValue("\"Password123!!\"");
_hc.SecretMasker.AddValue("\"short\"");
// Assert.
Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Password123!123"));
@@ -99,6 +101,9 @@ namespace GitHub.Runner.Common.Tests
Assert.Equal("YWJjOlBh***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abc:Password123!"))));
Assert.Equal("YWJjZDpQ***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcd:Password123!"))));
Assert.Equal("YWJjZGU6***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcde:Password123!"))));
Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Password123!!123"));
Assert.Equal("123short123", _hc.SecretMasker.MaskSecrets("123short123"));
Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123\"short\"123"));
}
finally
{

View File

@@ -16,7 +16,9 @@ namespace GitHub.Runner.Common.Tests
private static readonly List<string> SkippedFiles = new List<string>()
{
"Runner.Common\\HostContext.cs",
"Runner.Common/HostContext.cs"
"Runner.Common/HostContext.cs",
"Runner.Common\\HttpClientHandlerFactory.cs",
"Runner.Common/HttpClientHandlerFactory.cs"
};
[Fact]

View File

@@ -1,19 +1,21 @@
using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Worker;
using GitHub.Runner.Worker.Container;
using Moq;
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Reflection;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Sdk;
using GitHub.Runner.Worker;
using GitHub.Runner.Worker.Container;
using Moq;
using Moq.Protected;
using Xunit;
using Pipelines = GitHub.DistributedTask.Pipelines;
@@ -114,47 +116,175 @@ namespace GitHub.Runner.Common.Tests.Worker
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void PrepareActions_SkipDownloadActionFromGraphWhenCached_OnPremises()
public async void PrepareActions_DownloadBuiltInActionFromGraph_OnPremises()
{
try
{
// Arrange
Setup();
var actionId = Guid.NewGuid();
const string ActionName = "actions/sample-action";
var actions = new List<Pipelines.ActionStep>
{
new Pipelines.ActionStep()
{
Name = "action",
Id = actionId,
Id = Guid.NewGuid(),
Reference = new Pipelines.RepositoryPathReference()
{
Name = "actions/no-such-action",
Name = ActionName,
Ref = "master",
RepositoryType = "GitHub"
}
}
};
_configurationStore.Object.GetSettings().IsHostedServer = false;
var actionDirectory = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/no-such-action", "master");
Directory.CreateDirectory(actionDirectory);
var watermarkFile = $"{actionDirectory}.completed";
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
var actionFile = Path.Combine(actionDirectory, "action.yml");
File.WriteAllText(actionFile, @"
name: ""no-such-action""
runs:
using: node12
main: no-such-action.js
");
var testFile = Path.Combine(actionDirectory, "test-file");
File.WriteAllText(testFile, "asdf");
// Act
// Return a valid action from GHES via mock
const string ApiUrl = "https://ghes.example.com/api/v3";
string expectedArchiveLink = GetLinkToActionArchive(ApiUrl, ActionName, "master");
string archiveFile = await CreateRepoArchive();
using var stream = File.OpenRead(archiveFile);
var mockClientHandler = new Mock<HttpClientHandler>();
mockClientHandler.Protected().Setup<Task<HttpResponseMessage>>("SendAsync", ItExpr.Is<HttpRequestMessage>(m => m.RequestUri == new Uri(expectedArchiveLink)), ItExpr.IsAny<CancellationToken>())
.ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(stream) });
var mockHandlerFactory = new Mock<IHttpClientHandlerFactory>();
mockHandlerFactory.Setup(p => p.CreateClientHandler(It.IsAny<RunnerWebProxy>())).Returns(mockClientHandler.Object);
_hc.SetSingleton(mockHandlerFactory.Object);
_ec.Setup(x => x.GetGitHubContext("api_url")).Returns(ApiUrl);
_configurationStore.Object.GetSettings().IsHostedServer = false;
//Act
await _actionManager.PrepareActionsAsync(_ec.Object, actions);
// Assert
Assert.True(File.Exists(testFile));
//Assert
var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master.completed");
Assert.True(File.Exists(watermarkFile));
var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master", "action.yml");
Assert.True(File.Exists(actionYamlFile));
_hc.GetTrace().Info(File.ReadAllText(actionYamlFile));
}
finally
{
Teardown();
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void PrepareActions_DownloadCommunityActionFromGraph_OnPremises()
{
try
{
// Arrange
Setup();
const string ActionName = "ownerName/sample-action";
const string MungedActionName = "actions-community/ownerName-sample-action";
var actions = new List<Pipelines.ActionStep>
{
new Pipelines.ActionStep()
{
Name = "action",
Id = Guid.NewGuid(),
Reference = new Pipelines.RepositoryPathReference()
{
Name = ActionName,
Ref = "master",
RepositoryType = "GitHub"
}
}
};
// Return a valid action from GHES via mock
const string ApiUrl = "https://ghes.example.com/api/v3";
string builtInArchiveLink = GetLinkToActionArchive(ApiUrl, ActionName, "master");
string mungedArchiveLink = GetLinkToActionArchive(ApiUrl, MungedActionName, "master");
string archiveFile = await CreateRepoArchive();
using var stream = File.OpenRead(archiveFile);
var mockClientHandler = new Mock<HttpClientHandler>();
mockClientHandler.Protected().Setup<Task<HttpResponseMessage>>("SendAsync", ItExpr.Is<HttpRequestMessage>(m => m.RequestUri == new Uri(builtInArchiveLink)), ItExpr.IsAny<CancellationToken>())
.ReturnsAsync(new HttpResponseMessage(HttpStatusCode.NotFound));
mockClientHandler.Protected().Setup<Task<HttpResponseMessage>>("SendAsync", ItExpr.Is<HttpRequestMessage>(m => m.RequestUri == new Uri(mungedArchiveLink)), ItExpr.IsAny<CancellationToken>())
.ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(stream) });
var mockHandlerFactory = new Mock<IHttpClientHandlerFactory>();
mockHandlerFactory.Setup(p => p.CreateClientHandler(It.IsAny<RunnerWebProxy>())).Returns(mockClientHandler.Object);
_hc.SetSingleton(mockHandlerFactory.Object);
_ec.Setup(x => x.GetGitHubContext("api_url")).Returns(ApiUrl);
_configurationStore.Object.GetSettings().IsHostedServer = false;
//Act
await _actionManager.PrepareActionsAsync(_ec.Object, actions);
//Assert
var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master.completed");
Assert.True(File.Exists(watermarkFile));
var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master", "action.yml");
Assert.True(File.Exists(actionYamlFile));
_hc.GetTrace().Info(File.ReadAllText(actionYamlFile));
}
finally
{
Teardown();
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void PrepareActions_DownloadUnknownActionFromGraph_OnPremises()
{
try
{
// Arrange
Setup();
const string ActionName = "ownerName/sample-action";
var actions = new List<Pipelines.ActionStep>
{
new Pipelines.ActionStep()
{
Name = "action",
Id = Guid.NewGuid(),
Reference = new Pipelines.RepositoryPathReference()
{
Name = ActionName,
Ref = "master",
RepositoryType = "GitHub"
}
}
};
// Return a valid action from GHES via mock
const string ApiUrl = "https://ghes.example.com/api/v3";
string archiveLink = GetLinkToActionArchive(ApiUrl, ActionName, "master");
string archiveFile = await CreateRepoArchive();
using var stream = File.OpenRead(archiveFile);
var mockClientHandler = new Mock<HttpClientHandler>();
mockClientHandler.Protected().Setup<Task<HttpResponseMessage>>("SendAsync", ItExpr.IsAny<HttpRequestMessage>(), ItExpr.IsAny<CancellationToken>())
.ReturnsAsync(new HttpResponseMessage(HttpStatusCode.NotFound));
var mockHandlerFactory = new Mock<IHttpClientHandlerFactory>();
mockHandlerFactory.Setup(p => p.CreateClientHandler(It.IsAny<RunnerWebProxy>())).Returns(mockClientHandler.Object);
_hc.SetSingleton(mockHandlerFactory.Object);
_ec.Setup(x => x.GetGitHubContext("api_url")).Returns(ApiUrl);
_configurationStore.Object.GetSettings().IsHostedServer = false;
//Act
Func<Task> action = async () => await _actionManager.PrepareActionsAsync(_ec.Object, actions);
//Assert
await Assert.ThrowsAsync<ActionNotFoundException>(action);
var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master.completed");
Assert.False(File.Exists(watermarkFile));
var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master", "action.yml");
Assert.False(File.Exists(actionYamlFile));
}
finally
{
@@ -862,7 +992,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -962,7 +1092,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1061,7 +1191,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1129,7 +1259,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1211,7 +1341,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1310,7 +1440,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1408,7 +1538,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1476,7 +1606,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1547,7 +1677,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'GitHub'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1647,7 +1777,7 @@ runs:
name: 'Hello World'
description: 'Greet the world and record the time'
author: 'Test Corporation'
inputs:
inputs:
greeting: # id of input
description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout'
required: true
@@ -1737,6 +1867,82 @@ runs:
};
}
/// <summary>
/// Creates a sample action in an archive on disk, similar to the archive
/// retrieved from GitHub's or GHES' repository API.
/// </summary>
/// <returns>The path on disk to the archive.</returns>
#if OS_WINDOWS
private Task<string> CreateRepoArchive()
#else
private async Task<string> CreateRepoArchive()
#endif
{
const string Content = @"
# Container action
name: 'Hello World'
description: 'Greet the world'
author: 'GitHub'
icon: 'hello.svg' # vector art to display in the GitHub Marketplace
color: 'green' # optional, decorates the entry in the GitHub Marketplace
runs:
using: 'node12'
main: 'task.js'
";
CreateAction(yamlContent: Content, instance: out _, directory: out string directory);
var tempDir = _hc.GetDirectory(WellKnownDirectory.Temp);
Directory.CreateDirectory(tempDir);
var archiveFile = Path.Combine(tempDir, Path.GetRandomFileName());
var trace = _hc.GetTrace();
#if OS_WINDOWS
ZipFile.CreateFromDirectory(directory, archiveFile, CompressionLevel.Fastest, includeBaseDirectory: true);
return Task.FromResult(archiveFile);
#else
string tar = WhichUtil.Which("tar", require: true, trace: trace);
// tar -xzf
using (var processInvoker = new ProcessInvokerWrapper())
{
processInvoker.Initialize(_hc);
processInvoker.OutputDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
trace.Error(args.Data);
}
});
string cwd = Path.GetDirectoryName(directory);
string inputDirectory = Path.GetFileName(directory);
int exitCode = await processInvoker.ExecuteAsync(_hc.GetDirectory(WellKnownDirectory.Bin), tar, $"-czf \"{archiveFile}\" -C \"{cwd}\" \"{inputDirectory}\"", null, CancellationToken.None);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -czf' to create archive file: {archiveFile}. return code: {exitCode}.");
}
}
return archiveFile;
#endif
}
private static string GetLinkToActionArchive(string apiUrl, string repository, string @ref)
{
#if OS_WINDOWS
return $"{apiUrl}/repos/{repository}/zipball/{@ref}";
#else
return $"{apiUrl}/repos/{repository}/tarball/{@ref}";
#endif
}
private void Setup([CallerMemberName] string name = "")
{
_ecTokenSource?.Dispose();
@@ -1772,6 +1978,7 @@ runs:
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
_hc.SetSingleton<IRunnerPluginManager>(_pluginManager.Object);
_hc.SetSingleton<IActionManifestManager>(actionManifest);
_hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
_configurationStore = new Mock<IConfigurationStore>();
_configurationStore

View File

@@ -278,6 +278,59 @@ namespace GitHub.Runner.Common.Tests.Worker
Assert.Equal("${{ matrix.node }}", _actionRunner.DisplayName);
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public async void WarnInvalidInputs()
{
//Arrange
Setup();
var actionId = Guid.NewGuid();
var actionInputs = new MappingToken(null, null, null);
actionInputs.Add(new StringToken(null, null, null, "input1"), new StringToken(null, null, null, "test1"));
actionInputs.Add(new StringToken(null, null, null, "input2"), new StringToken(null, null, null, "test2"));
actionInputs.Add(new StringToken(null, null, null, "invalid1"), new StringToken(null, null, null, "invalid1"));
actionInputs.Add(new StringToken(null, null, null, "invalid2"), new StringToken(null, null, null, "invalid2"));
var action = new Pipelines.ActionStep()
{
Name = "action",
Id = actionId,
Reference = new Pipelines.ContainerRegistryReference()
{
Image = "ubuntu:16.04"
},
Inputs = actionInputs
};
_actionRunner.Action = action;
Dictionary<string, string> finialInputs = new Dictionary<string, string>();
_handlerFactory.Setup(x => x.Create(It.IsAny<IExecutionContext>(), It.IsAny<ActionStepDefinitionReference>(), It.IsAny<IStepHost>(), It.IsAny<ActionExecutionData>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<Variables>(), It.IsAny<string>()))
.Callback((IExecutionContext executionContext, Pipelines.ActionStepDefinitionReference actionReference, IStepHost stepHost, ActionExecutionData data, Dictionary<string, string> inputs, Dictionary<string, string> environment, Variables runtimeVariables, string taskDirectory) =>
{
finialInputs = inputs;
})
.Returns(new Mock<IHandler>().Object);
//Act
await _actionRunner.RunAsync();
foreach (var input in finialInputs)
{
_hc.GetTrace().Info($"Input: {input.Key}={input.Value}");
}
//Assert
Assert.Equal("test1", finialInputs["input1"]);
Assert.Equal("test2", finialInputs["input2"]);
Assert.Equal("github", finialInputs["input3"]);
Assert.Equal("invalid1", finialInputs["invalid1"]);
Assert.Equal("invalid2", finialInputs["invalid2"]);
_ec.Verify(x => x.AddIssue(It.Is<Issue>(s => s.Message.Contains("Unexpected input 'invalid1'")), It.IsAny<string>()), Times.Once);
_ec.Verify(x => x.AddIssue(It.Is<Issue>(s => s.Message.Contains("Unexpected input 'invalid2'")), It.IsAny<string>()), Times.Once);
}
private void Setup([CallerMemberName] string name = "")
{
_ecTokenSource?.Dispose();