Compare commits

...

5 Commits

Author SHA1 Message Date
eric sciple
8770849c3b add set-workspace command 2019-10-30 22:16:31 -04:00
eric sciple
61f9d3f656 cherry picks from master (#156)
* Update dotnet install script (#155)

* root search pattern for hashfiles and allow forward slash on windows (#151)
2019-10-30 14:46:40 -04:00
Thomas Boop
08b9f6e045 Merge release 160.0 into master (#153)
* Update to Version 2.160.0 (#144)

* Revert "remove issue generation on warning/error commands (#137)" (#147)

* Revert "remove issue generation on warning/error commands (#137)"

This reverts commit 53da198867.

* Updated Release notes

* Users/thboop/port directory changes (#152)

* Clear action cache for local runner

* update release notes for actions directory cache changes
2019-10-28 15:45:27 -04:00
eric sciple
0129e8111f Clear action cache for local runner (#148) 2019-10-28 11:56:12 -04:00
eric sciple
ccca13ac07 restrict hashFiles to basic globbing and globstar (#150) 2019-10-28 11:52:22 -04:00
10 changed files with 281 additions and 86 deletions

View File

@@ -1,12 +1,12 @@
## Features
- Set Default shell to powershell for windows runners (#135)
- Use Powershell as fallback if Powershell Core is not available for default shell on windows (#142)
- N/A
## Bugs
- Removed unintended additional fields on error and warning commands (#137)
- Reverted removal of additional fields error and warning fields (#147)
- Actions cache would incorrectly cache the action if the tag was updated (#148)
## Misc
- N/A
- Updated to .NET Core 3.0 (#127)
## Agent Downloads

View File

@@ -37,10 +37,7 @@
.PARAMETER SharedRuntime
This parameter is obsolete and may be removed in a future version of this script.
The recommended alternative is '-Runtime dotnet'.
Default: false
Installs just the shared runtime bits, not the entire SDK.
This is equivalent to specifying `-Runtime dotnet`.
.PARAMETER Runtime
Installs just a shared runtime, not the entire SDK.
Possible values:
@@ -77,11 +74,15 @@
Skips installing non-versioned files if they already exist, such as dotnet.exe.
.PARAMETER NoCdn
Disable downloading from the Azure CDN, and use the uncached feed directly.
.PARAMETER JSonFile
Determines the SDK version from a user specified global.json file
Note: global.json must have a value for 'SDK:Version'
#>
[cmdletbinding()]
param(
[string]$Channel="LTS",
[string]$Version="Latest",
[string]$JSonFile,
[string]$InstallDir="<auto>",
[string]$Architecture="<auto>",
[ValidateSet("dotnet", "aspnetcore", "windowsdesktop", IgnoreCase = $false)]
@@ -258,7 +259,6 @@ function GetHTTPResponse([Uri] $Uri)
})
}
function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) {
Say-Invocation $MyInvocation
@@ -304,20 +304,64 @@ function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Co
return $VersionInfo
}
function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version) {
function Parse-Jsonfile-For-Version([string]$JSonFile) {
Say-Invocation $MyInvocation
switch ($Version.ToLower()) {
{ $_ -eq "latest" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False
return $LatestVersionInfo.Version
If (-Not (Test-Path $JSonFile)) {
throw "Unable to find '$JSonFile'"
exit 0
}
try {
$JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue
}
catch {
throw "Json file unreadable: '$JSonFile'"
exit 0
}
if ($JSonContent) {
try {
$JSonContent.PSObject.Properties | ForEach-Object {
$PropertyName = $_.Name
if ($PropertyName -eq "version") {
$Version = $_.Value
Say-Verbose "Version = $Version"
}
}
}
{ $_ -eq "coherent" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True
return $LatestVersionInfo.Version
catch {
throw "Unable to parse the SDK node in '$JSonFile'"
exit 0
}
default { return $Version }
}
else {
throw "Unable to find the SDK node in '$JSonFile'"
exit 0
}
If ($Version -eq $null) {
throw "Unable to find the SDK:version node in '$JSonFile'"
exit 0
}
return $Version
}
function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version, [string]$JSonFile) {
Say-Invocation $MyInvocation
if (-not $JSonFile) {
switch ($Version.ToLower()) {
{ $_ -eq "latest" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False
return $LatestVersionInfo.Version
}
{ $_ -eq "coherent" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True
return $LatestVersionInfo.Version
}
default { return $Version }
}
}
else {
return Parse-Jsonfile-For-Version $JSonFile
}
}
@@ -382,23 +426,6 @@ function Resolve-Installation-Path([string]$InstallDir) {
return $InstallDir
}
function Get-Version-Info-From-Version-File([string]$InstallRoot, [string]$RelativePathToVersionFile) {
Say-Invocation $MyInvocation
$VersionFile = Join-Path -Path $InstallRoot -ChildPath $RelativePathToVersionFile
Say-Verbose "Local version file: $VersionFile"
if (Test-Path $VersionFile) {
$VersionText = cat $VersionFile
Say-Verbose "Local version file text: $VersionText"
return Get-Version-Info-From-Version-Text $VersionText
}
Say-Verbose "Local version file not found."
return $null
}
function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) {
Say-Invocation $MyInvocation
@@ -534,7 +561,7 @@ function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolde
}
$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture
$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version
$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile
$DownloadLink = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture

View File

@@ -435,11 +435,52 @@ get_latest_version_info() {
return $?
}
# args:
# json_file - $1
parse_jsonfile_for_version() {
eval $invocation
local json_file="$1"
if [ ! -f "$json_file" ]; then
say_err "Unable to find \`$json_file\`"
return 1
fi
sdk_section=$(cat $json_file | awk '/"sdk"/,/}/')
if [ -z "$sdk_section" ]; then
say_err "Unable to parse the SDK node in \`$json_file\`"
return 1
fi
sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}')
sdk_list=${sdk_list//[\" ]/}
sdk_list=${sdk_list//,/$'\n'}
sdk_list="$(echo -e "${sdk_list}" | tr -d '[[:space:]]')"
local version_info=""
while read -r line; do
IFS=:
while read -r key value; do
if [[ "$key" == "version" ]]; then
version_info=$value
fi
done <<< "$line"
done <<< "$sdk_list"
if [ -z "$version_info" ]; then
say_err "Unable to find the SDK:version node in \`$json_file\`"
return 1
fi
echo "$version_info"
return 0
}
# args:
# azure_feed - $1
# channel - $2
# normalized_architecture - $3
# version - $4
# json_file - $5
get_specific_version_from_version() {
eval $invocation
@@ -447,27 +488,35 @@ get_specific_version_from_version() {
local channel="$2"
local normalized_architecture="$3"
local version="$(to_lowercase "$4")"
local json_file="$5"
case "$version" in
latest)
local version_info
version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1
say_verbose "get_specific_version_from_version: version_info=$version_info"
echo "$version_info" | get_version_from_version_info
return 0
;;
coherent)
local version_info
version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1
say_verbose "get_specific_version_from_version: version_info=$version_info"
echo "$version_info" | get_version_from_version_info
return 0
;;
*)
echo "$version"
return 0
;;
esac
if [ -z "$json_file" ]; then
case "$version" in
latest)
local version_info
version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1
say_verbose "get_specific_version_from_version: version_info=$version_info"
echo "$version_info" | get_version_from_version_info
return 0
;;
coherent)
local version_info
version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1
say_verbose "get_specific_version_from_version: version_info=$version_info"
echo "$version_info" | get_version_from_version_info
return 0
;;
*)
echo "$version"
return 0
;;
esac
else
local version_info
version_info="$(parse_jsonfile_for_version "$json_file")" || return 1
echo "$version_info"
return 0
fi
}
# args:
@@ -558,24 +607,6 @@ resolve_installation_path() {
return 0
}
# args:
# install_root - $1
get_installed_version_info() {
eval $invocation
local install_root="$1"
local version_file="$(combine_paths "$install_root" "$local_version_file_relative_path")"
say_verbose "Local version file: $version_file"
if [ ! -z "$version_file" ] | [ -r "$version_file" ]; then
local version_info="$(cat "$version_file")"
echo "$version_info"
return 0
fi
say_verbose "Local version file not found."
return 0
}
# args:
# relative_or_absolute_path - $1
get_absolute_path() {
@@ -724,7 +755,7 @@ calculate_vars() {
normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")"
say_verbose "normalized_architecture=$normalized_architecture"
specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version")"
specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")"
say_verbose "specific_version=$specific_version"
if [ -z "$specific_version" ]; then
say_err "Could not resolve version information."
@@ -826,6 +857,7 @@ temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX"
channel="LTS"
version="Latest"
json_file=""
install_dir="<auto>"
architecture="<auto>"
dry_run=false
@@ -912,6 +944,10 @@ do
runtime_id="$1"
non_dynamic_parameters+=" $name "\""$1"\"""
;;
--jsonfile|-[Jj][Ss]on[Ff]ile)
shift
json_file="$1"
;;
--skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles)
override_non_versioned_files=false
non_dynamic_parameters+=" $name"
@@ -953,22 +989,25 @@ do
echo " Possible values:"
echo " - dotnet - the Microsoft.NETCore.App shared runtime"
echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime"
echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable."
echo " -SkipNonVersionedFiles"
echo " --dry-run,-DryRun Do not perform installation. Display download link."
echo " --no-path, -NoPath Do not set PATH for the current process."
echo " --verbose,-Verbose Display diagnostics information."
echo " --azure-feed,-AzureFeed Azure feed location. Defaults to $azure_feed, This parameter typically is not changed by the user."
echo " --uncached-feed,-UncachedFeed Uncached feed location. This parameter typically is not changed by the user."
echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly."
echo " --feed-credential,-FeedCredential Azure feed shared access token. This parameter typically is not specified."
echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable."
echo " -SkipNonVersionedFiles"
echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly."
echo " --jsonfile <JSONFILE> Determines the SDK version from a user specified global.json file."
echo " Note: global.json must have a value for 'SDK:Version'"
echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)."
echo " -RuntimeId"
echo " -?,--?,-h,--help,-Help Shows this help message"
echo ""
echo "Obsolete parameters:"
echo " --shared-runtime The recommended alternative is '--runtime dotnet'."
echo " -SharedRuntime Installs just the shared runtime bits, not the entire SDK."
echo " This parameter is obsolete and may be removed in a future version of this script."
echo " Installs just the shared runtime bits, not the entire SDK."
echo ""
echo "Install Location:"
echo " Location is chosen in following order:"

View File

@@ -42,6 +42,7 @@ namespace GitHub.Runner.Common
// Action command extensions.
case "GitHub.Runner.Worker.IActionCommandExtension":
Add<T>(extensions, "GitHub.Runner.Worker.InternalPluginSetRepoPathCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SetWorkspaceCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SetEnvCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SetOutputCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SaveStateCommandExtension, Runner.Worker");

View File

@@ -184,6 +184,40 @@ namespace GitHub.Runner.Worker
}
}
public sealed class SetWorkspaceCommandExtension : RunnerService, IActionCommandExtension
{
public string Command => "set-workspace";
public Type ExtensionType => typeof(IActionCommandExtension);
public void ProcessCommand(IExecutionContext context, string line, ActionCommand command)
{
string path = command.Data;
if (string.IsNullOrWhiteSpace(path))
{
throw new Exception("Path not specified");
}
if (!Path.IsPathRooted(path))
{
throw new Exception("Expected a rooted path");
}
string runnerWorkspace = context.GetRunnerContext("workspace");
ArgUtil.Directory(runnerWorkspace, nameof(runnerWorkspace));
// Must be under runner workspace
path = Path.GetFullPath(path); // remove relative pathing and normalize slashes
if (!path.StartsWith(runnerWorkspace + Path.DirectorySeparatorChar, IOUtil.FilePathStringComparison))
{
throw new Exception($"Expected path to be under {runnerWorkspace}");
}
Trace.Info($"Setting GitHub workspace to '{path}'");
context.SetGitHubContext("workspace", path);
}
}
public sealed class SetEnvCommandExtension : RunnerService, IActionCommandExtension
{
public string Command => "set-env";
@@ -436,6 +470,10 @@ namespace GitHub.Runner.Worker
public void ProcessCommand(IExecutionContext context, string inputLine, ActionCommand command)
{
command.Properties.TryGetValue(IssueCommandProperties.File, out string file);
command.Properties.TryGetValue(IssueCommandProperties.Line, out string line);
command.Properties.TryGetValue(IssueCommandProperties.Column, out string column);
Issue issue = new Issue()
{
Category = "General",
@@ -443,8 +481,54 @@ namespace GitHub.Runner.Worker
Message = command.Data
};
if (!string.IsNullOrEmpty(file))
{
issue.Category = "Code";
if (context.Container != null)
{
// Translate file path back from container path
file = context.Container.TranslateToHostPath(file);
command.Properties[IssueCommandProperties.File] = file;
}
// Get the values that represent the server path given a local path
string repoName = context.GetGitHubContext("repository");
var repoPath = context.GetGitHubContext("workspace");
string relativeSourcePath = IOUtil.MakeRelative(file, repoPath);
if (!string.Equals(relativeSourcePath, file, IOUtil.FilePathStringComparison))
{
// add repo info
if (!string.IsNullOrEmpty(repoName))
{
command.Properties["repo"] = repoName;
}
if (!string.IsNullOrEmpty(relativeSourcePath))
{
// replace sourcePath with the new relative path
// prefer `/` on all platforms
command.Properties[IssueCommandProperties.File] = relativeSourcePath.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
}
}
foreach (var property in command.Properties)
{
issue.Data[property.Key] = property.Value;
}
context.AddIssue(issue);
}
private static class IssueCommandProperties
{
public const String File = "file";
public const String Line = "line";
public const String Column = "col";
}
}
public sealed class GroupCommandExtension : GroupingCommandExtension

View File

@@ -58,6 +58,9 @@ namespace GitHub.Runner.Worker
executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is depreciated. Please remove it from the repository's secrets");
}
// Clear the cache (local runner)
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
foreach (var action in actions)
{
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
@@ -445,7 +448,7 @@ namespace GitHub.Runner.Worker
}
else
{
// make sure we get an clean folder ready to use.
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");

View File

@@ -78,6 +78,7 @@ namespace GitHub.Runner.Worker
void Start(string currentOperation = null);
TaskResult Complete(TaskResult? result = null, string currentOperation = null, string resultCode = null);
void SetEnvContext(string name, string value);
string GetRunnerContext(string name);
void SetRunnerContext(string name, string value);
string GetGitHubContext(string name);
void SetGitHubContext(string name, string value);
@@ -365,6 +366,13 @@ namespace GitHub.Runner.Worker
return Result.Value;
}
public string GetRunnerContext(string name)
{
ArgUtil.NotNullOrEmpty(name, nameof(name));
var runnerContext = ExpressionValues["runner"] as RunnerContext;
return runnerContext[name].AssertString($"runner[{name}]");
}
public void SetRunnerContext(string name, string value)
{
ArgUtil.NotNullOrEmpty(name, nameof(name));

View File

@@ -5,6 +5,7 @@ using System.Text;
using Minimatch;
using System.IO;
using System.Security.Cryptography;
using GitHub.DistributedTask.Expressions2.Sdk;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.Pipelines.ObjectTemplating;
namespace GitHub.DistributedTask.Expressions2.Sdk.Functions
@@ -28,29 +29,50 @@ namespace GitHub.DistributedTask.Expressions2.Sdk.Functions
string searchRoot = workspaceData.Value;
string pattern = Parameters[0].Evaluate(context).ConvertToString();
// Convert slashes on Windows
if (s_isWindows)
{
pattern = pattern.Replace('\\', '/');
}
// Root the pattern
if (!Path.IsPathRooted(pattern))
{
var patternRoot = s_isWindows ? searchRoot.Replace('\\', '/').TrimEnd('/') : searchRoot.TrimEnd('/');
pattern = string.Concat(patternRoot, "/", pattern);
}
// Get all files
context.Trace.Info($"Search root directory: '{searchRoot}'");
context.Trace.Info($"Search pattern: '{pattern}'");
var files = Directory.GetFiles(searchRoot, "*", SearchOption.AllDirectories).OrderBy(x => x).ToList();
var files = Directory.GetFiles(searchRoot, "*", SearchOption.AllDirectories)
.Select(x => s_isWindows ? x.Replace('\\', '/') : x)
.OrderBy(x => x, StringComparer.Ordinal)
.ToList();
if (files.Count == 0)
{
throw new ArgumentException($"'hashFiles({pattern})' failed. Directory '{searchRoot}' is empty");
throw new ArgumentException($"hashFiles('{ExpressionUtility.StringEscape(pattern)}') failed. Directory '{searchRoot}' is empty");
}
else
{
context.Trace.Info($"Found {files.Count} files");
}
// Match
var matcher = new Minimatcher(pattern, s_minimatchOptions);
files = matcher.Filter(files).ToList();
files = matcher.Filter(files)
.Select(x => s_isWindows ? x.Replace('/', '\\') : x)
.ToList();
if (files.Count == 0)
{
throw new ArgumentException($"'hashFiles({pattern})' failed. Search pattern '{pattern}' doesn't match any file under '{searchRoot}'");
throw new ArgumentException($"hashFiles('{ExpressionUtility.StringEscape(pattern)}') failed. Search pattern '{pattern}' doesn't match any file under '{searchRoot}'");
}
else
{
context.Trace.Info($"{files.Count} matches to hash");
}
// Hash each file
List<byte> filesSha256 = new List<byte>();
foreach (var file in files)
{
@@ -64,6 +86,7 @@ namespace GitHub.DistributedTask.Expressions2.Sdk.Functions
}
}
// Hash the hashes
using (SHA256 sha256hash = SHA256.Create())
{
var hashBytes = sha256hash.ComputeHash(filesSha256.ToArray());
@@ -83,11 +106,17 @@ namespace GitHub.DistributedTask.Expressions2.Sdk.Functions
}
}
private static readonly bool s_isWindows = Environment.OSVersion.Platform != PlatformID.Unix && Environment.OSVersion.Platform != PlatformID.MacOSX;
// Only support basic globbing (* ? and []) and globstar (**)
private static readonly Options s_minimatchOptions = new Options
{
Dot = true,
NoBrace = true,
NoCase = Environment.OSVersion.Platform != PlatformID.Unix && Environment.OSVersion.Platform != PlatformID.MacOSX
NoCase = s_isWindows,
NoComment = true,
NoExt = true,
NoNegate = true,
};
}
}

View File

@@ -233,6 +233,7 @@ namespace GitHub.Runner.Common.Tests.Worker
}
}
/*
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
@@ -272,6 +273,7 @@ namespace GitHub.Runner.Common.Tests.Worker
Teardown();
}
}
*/
[Fact]
[Trait("Level", "L0")]
@@ -306,6 +308,7 @@ namespace GitHub.Runner.Common.Tests.Worker
}
}
/*
#if OS_LINUX
[Fact]
[Trait("Level", "L0")]
@@ -772,6 +775,7 @@ namespace GitHub.Runner.Common.Tests.Worker
Teardown();
}
}
*/
[Fact]
[Trait("Level", "L0")]

View File

@@ -1 +1 @@
2.159.2
2.160.0