From eaf39bb05820924b4d49c99285ee3d18dcbd25c7 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 12:11:13 -0400 Subject: [PATCH 01/11] add libicu66 for Ubuntu 20.04 (#535) --- src/Misc/layoutbin/installdependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Misc/layoutbin/installdependencies.sh b/src/Misc/layoutbin/installdependencies.sh index 50a19982e..786714925 100755 --- a/src/Misc/layoutbin/installdependencies.sh +++ b/src/Misc/layoutbin/installdependencies.sh @@ -70,8 +70,8 @@ then exit 1 fi - # libicu version prefer: libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 - apt install -y libicu63 || apt install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 + # libicu version prefer: libicu66 -> libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 + apt install -y libicu66 || apt install -y libicu63 || apt install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 if [ $? -ne 0 ] then echo "'apt' failed with exit code '$?'" @@ -99,8 +99,8 @@ then exit 1 fi - # libicu version prefer: libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 - apt-get install -y libicu63 || apt-get install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 + # libicu version prefer: libicu66 -> libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 + apt-get install -y libicu66 || apt-get install -y libicu63 || apt-get install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 if [ $? -ne 0 ] then echo "'apt-get' failed with exit code '$?'" From 312c7668a88daec33adad05d4682b3ffb44e2bdd Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 12:11:35 -0400 Subject: [PATCH 02/11] Fix DataContract with Token service (#532) --- src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs index fab331726..8ecf34271 100644 --- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs +++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs @@ -39,7 +39,7 @@ namespace GitHub.Services.OAuth /// /// Gets or sets the error description for the response. /// - [DataMember(Name = "errordescription", EmitDefaultValue = false)] + [DataMember(Name = "error_description", EmitDefaultValue = false)] public String ErrorDescription { get; From 2e800f857e6a49e9d3095c38f96d5c9fb475c5b3 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 13:52:42 -0400 Subject: [PATCH 03/11] Skip search $PATH on command with fully qualified path (#526) --- src/Runner.Sdk/Util/WhichUtil.cs | 5 +++++ src/Test/L0/Util/WhichUtilL0.cs | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/Runner.Sdk/Util/WhichUtil.cs b/src/Runner.Sdk/Util/WhichUtil.cs index 71acd92d9..b4d94e513 100644 --- a/src/Runner.Sdk/Util/WhichUtil.cs +++ b/src/Runner.Sdk/Util/WhichUtil.cs @@ -11,6 +11,11 @@ namespace GitHub.Runner.Sdk { ArgUtil.NotNullOrEmpty(command, nameof(command)); trace?.Info($"Which: '{command}'"); + if (Path.IsPathFullyQualified(command) && File.Exists(command)) + { + trace?.Info($"Fully qualified path: '{command}'"); + return command; + } string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable); if (string.IsNullOrEmpty(path)) { diff --git a/src/Test/L0/Util/WhichUtilL0.cs b/src/Test/L0/Util/WhichUtilL0.cs index 99e4a92a5..7271bc283 100644 --- a/src/Test/L0/Util/WhichUtilL0.cs +++ b/src/Test/L0/Util/WhichUtilL0.cs @@ -70,5 +70,24 @@ namespace GitHub.Runner.Common.Tests.Util } } } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void WhichHandleFullyQualifiedPath() + { + using (TestHostContext hc = new TestHostContext(this)) + { + //Arrange + Tracing trace = hc.GetTrace(); + + // Act. + var gitPath = WhichUtil.Which("git", require: true, trace: trace); + var gitPath2 = WhichUtil.Which(gitPath, require: true, trace: trace); + + // Assert. + Assert.Equal(gitPath, gitPath2); + } + } } } From de4490d06d5ef58a8bf0c24b41f1f84f61c8df21 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 15:40:09 -0400 Subject: [PATCH 04/11] Restore SELinux context on service file when SELinux is enabled (#525) --- src/Misc/layoutbin/systemd.svc.sh.template | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Misc/layoutbin/systemd.svc.sh.template b/src/Misc/layoutbin/systemd.svc.sh.template index cbec33197..bdbc998f7 100644 --- a/src/Misc/layoutbin/systemd.svc.sh.template +++ b/src/Misc/layoutbin/systemd.svc.sh.template @@ -63,12 +63,25 @@ function install() sed "s/{{User}}/${run_as_user}/g; s/{{Description}}/$(echo ${SVC_DESCRIPTION} | sed -e 's/[\/&]/\\&/g')/g; s/{{RunnerRoot}}/$(echo ${RUNNER_ROOT} | sed -e 's/[\/&]/\\&/g')/g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file" mv "${TEMP_PATH}" "${UNIT_PATH}" || failed "failed to copy unit file" + + # Recent Fedora based Linux (CentOS/Redhat) has SELinux enabled by default + # We need to restore security context on the unit file we added otherwise SystemD have no access to it. + command -v getenforce > /dev/null + if [ $? -eq 0 ] + then + selinuxEnabled=$(getenforce) + if [[ $selinuxEnabled == "Enforcing" ]] + then + # SELinux is enabled, we will need to Restore SELinux Context for the service file + restorecon -r -v "${UNIT_PATH}" || failed "failed to restore SELinux context on ${UNIT_PATH}" + fi + fi # unit file should not be executable and world writable - chmod 664 ${UNIT_PATH} || failed "failed to set permissions on ${UNIT_PATH}" + chmod 664 "${UNIT_PATH}" || failed "failed to set permissions on ${UNIT_PATH}" systemctl daemon-reload || failed "failed to reload daemons" - # Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user. + # Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user. cp ./bin/runsvc.sh ./runsvc.sh || failed "failed to copy runsvc.sh" chown ${run_as_uid}:${run_as_gid} ./runsvc.sh || failed "failed to set owner for runsvc.sh" chmod 755 ./runsvc.sh || failed "failed to set permission for runsvc.sh" From e728b8594d6fca9c2a0207a116eb819135655317 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 16:17:24 -0400 Subject: [PATCH 05/11] fix race condition. (#538) --- src/Runner.Sdk/ProcessInvoker.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index 4ed4ce3b0..78a9f2dd2 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -346,14 +346,14 @@ namespace GitHub.Runner.Sdk // data buffers one last time before returning ProcessOutput(); - Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}."); - } + if (cancellationToken.IsCancellationRequested) + { + // Ensure cancellation also finish on the cancellationToken.Register thread. + await cancellationFinished.Task; + Trace.Info($"Process Cancellation finished."); + } - if (cancellationToken.IsCancellationRequested) - { - // Ensure cancellation also finish on the cancellationToken.Register thread. - await cancellationFinished.Task; - Trace.Info($"Process Cancellation finished."); + Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}."); } cancellationToken.ThrowIfCancellationRequested(); From 89d1418e4856c2378aab28f72ea15850c117911a Mon Sep 17 00:00:00 2001 From: Lokesh Gopu Date: Thu, 11 Jun 2020 17:25:50 -0400 Subject: [PATCH 06/11] Update exception message (#540) --- src/Runner.Listener/Configuration/ConfigurationManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 7d634067f..ce0863e1a 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -210,7 +210,7 @@ namespace GitHub.Runner.Listener.Configuration else if (command.Unattended) { // if not replace and it is unattended config. - throw new TaskAgentExistsException($"Pool {runnerSettings.PoolId} already contains a runner with name {runnerSettings.AgentName}."); + throw new TaskAgentExistsException($"A runner exists with the same name {runnerSettings.AgentName}."); } } else From 4e7d27a53c84466562d366256b960433bafc469b Mon Sep 17 00:00:00 2001 From: eric sciple Date: Mon, 15 Jun 2020 13:13:47 -0400 Subject: [PATCH 07/11] remove temporary logic when resolving action download info (#550) --- src/Runner.Worker/ActionManager.cs | 67 +-------------------------- src/Test/L0/Worker/ActionManagerL0.cs | 4 +- 2 files changed, 4 insertions(+), 67 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 347eb3d0d..b9720610d 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -546,56 +546,6 @@ namespace GitHub.Runner.Worker } } - // This implementation is temporary and will be removed when we switch to a REST API call to the service to resolve the download info - private async Task RepoExistsAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo actionDownloadInfo, string token) - { - var apiUrl = GetApiUrl(executionContext); - var repoUrl = $"{apiUrl}/repos/{actionDownloadInfo.NameWithOwner}"; - for (var attempt = 1; attempt <= 3; attempt++) - { - executionContext.Debug($"Checking whether repo exists: {repoUrl}"); - try - { - using (var httpClientHandler = HostContext.CreateHttpClientHandler()) - using (var httpClient = new HttpClient(httpClientHandler)) - { - httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(token); - httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); - using (var response = await httpClient.GetAsync(repoUrl)) - { - if (response.IsSuccessStatusCode) - { - return true; - } - else if (response.StatusCode == HttpStatusCode.NotFound) - { - return false; - } - else - { - // Throw - response.EnsureSuccessStatusCode(); - } - } - } - } - catch (Exception ex) - { - if (attempt < 3) - { - executionContext.Debug($"Failed checking whether repo '{actionDownloadInfo.NameWithOwner}' exists: {ex.Message}"); - } - else - { - executionContext.Error($"Failed checking whether repo '{actionDownloadInfo.NameWithOwner}' exists: {ex.Message}"); - throw; - } - } - } - - return false; // Never reaches here - } - // This implementation is temporary and will be replaced with a REST API call to the service to resolve private async Task> GetDownloadInfoAsync(IExecutionContext executionContext, List actions) { @@ -659,23 +609,10 @@ namespace GitHub.Runner.Worker // Add secret HostContext.SecretMasker.AddValue(actionDownloadInfo.Authentication?.Token); - // Temporary code: Fix token and download URL - if (runnerSettings.IsHostedServer) + // Default auth token + if (string.IsNullOrEmpty(actionDownloadInfo.Authentication?.Token)) { actionDownloadInfo.Authentication = new WebApi.ActionDownloadAuthentication { Token = defaultAccessToken }; - actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", apiUrl); - actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", apiUrl); - } - else if (await RepoExistsAsync(executionContext, actionDownloadInfo, defaultAccessToken)) - { - actionDownloadInfo.Authentication = new WebApi.ActionDownloadAuthentication { Token = defaultAccessToken }; - actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", apiUrl); - actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", apiUrl); - } - else - { - actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", "https://api.github.com"); - actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", "https://api.github.com"); } } diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 85851a55a..58ffea5aa 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -3607,8 +3607,8 @@ runs: { NameWithOwner = action.NameWithOwner, Ref = action.Ref, - TarballUrl = $"/repos/{action.NameWithOwner}/tarball/{action.Ref}", - ZipballUrl = $"/repos/{action.NameWithOwner}/zipball/{action.Ref}", + TarballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/tarball/{action.Ref}", + ZipballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/zipball/{action.Ref}", }; } return Task.FromResult(result); From df7e16954e9bbcab82b85be44cad9e624f02cbe0 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 23 Jun 2020 13:57:37 -0400 Subject: [PATCH 08/11] print runner and machine name to log. (#539) --- src/Runner.Worker/JobExtension.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 235cc9063..61f080c71 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -64,6 +64,20 @@ namespace GitHub.Runner.Worker context.Debug($"Starting: Set up job"); context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'"); + var setting = HostContext.GetService().GetSettings(); + var credFile = HostContext.GetConfigFile(WellKnownConfigFile.Credentials); + if (File.Exists(credFile)) + { + var credData = IOUtil.LoadObject(credFile); + if (credData != null && + credData.Data.TryGetValue("clientId", out var clientId)) + { + // print out HostName for self-hosted runner + context.Output($"Runner name: '{setting.AgentName}'"); + context.Output($"Machine name: '{Environment.MachineName}'"); + } + } + var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo); if (File.Exists(setupInfoFile)) { From 7cef9a27ca514e72c10f3c7f2bf1418b52275a68 Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Tue, 23 Jun 2020 14:05:28 -0400 Subject: [PATCH 09/11] release 2.267.0 runner. --- releaseNote.md | 20 ++++++++++++-------- src/runnerversion | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index b75e056e3..48d725bc5 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,14 +1,18 @@ ## Features - - N/A + - Resolve action download info from server (#508, #515, #550) + - Print runner and machine name to log. (#539) ## Bugs - - Handle `jq` returns "null" if the field does not exist in create-latest-svc.sh (#478) - - Switch GITHUB_URL to GITHUB_SERVER_URL (#482) - - Fix problem matcher for GHES (#488) - - Fix container action inputs validation warning (#490) - - Fix post step display name (#490) - - Fix worker crash due to exception from evaluating step.env (#490) + - Reduce input validation warnings (#506) + - Fix null ref exception in SecretMasker caused by `hashfiles` timeout. (#516) + - Add libicu66 to `./installDependencies.sh` for Ubuntu 20.04 (#535) + - Fix DataContract with Token service (#532) + - Skip search $PATH on command with fully qualified path (#526) + - Restore SELinux context on service file when SELinux is enabled (#525) ## Misc - - N/A + - Remove SPS/Token migration code. Remove GHES url manipulate code. (#513) + - Add sub-step for developer flow for clarity (#523) + - Update Links and Language to Git + VSCode (#522) + - Update runner configuration exception message (#540) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/runnerversion b/src/runnerversion index f9d36d71d..58301aa10 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.263.0 +2.267.0 From a0942ed3459f199a293cabbe6f1d584e4ad352a5 Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Tue, 23 Jun 2020 15:35:32 -0400 Subject: [PATCH 10/11] Composite Actions Support for Multiple Run Steps (#549) * Composite Action Run Steps * Clean up trace messages + add Trace debug in ActionManager * Change String to string * Add comma to Composite * Change JobSteps to a List, Change Register Step function name * Add TODO, remove unn. content * Remove unnecessary code * Fix unit tests * Add verbose trace logs which are only viewable by devs * Sort usings in Composite Action Handler * Change 0 to location * Update context variables in composite action yaml * Add helpful error message for null steps --- src/Runner.Worker/ActionManager.cs | 20 + src/Runner.Worker/ActionManifestManager.cs | 32 +- src/Runner.Worker/ExecutionContext.cs | 22 +- .../Handlers/CompositeActionHandler.cs | 98 +++++ src/Runner.Worker/Handlers/HandlerFactory.cs | 5 + src/Runner.Worker/JobRunner.cs | 2 +- src/Runner.Worker/StepsRunner.cs | 19 +- src/Runner.Worker/action_yaml.json | 33 +- .../PipelineTemplateConstants.cs | 1 + .../PipelineTemplateConverter.cs | 347 +++++++++++++++++- .../PipelineTemplateEvaluator.cs | 48 ++- .../Pipelines/PipelineConstants.cs | 7 + src/Test/L0/Worker/StepsRunnerL0.cs | 26 +- 13 files changed, 624 insertions(+), 36 deletions(-) create mode 100644 src/Runner.Worker/Handlers/CompositeActionHandler.cs diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index b9720610d..242ab79e0 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -395,6 +395,12 @@ namespace GitHub.Runner.Worker Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}."); } } + else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + var compositeAction = definition.Data.Execution as CompositeActionExecutionData; + Trace.Info($"Load {compositeAction.Steps.Count} action steps."); + Trace.Verbose($"Details: {StringUtil.ConvertToJson(compositeAction.Steps)}"); + } else { throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString()); @@ -1038,6 +1044,11 @@ namespace GitHub.Runner.Worker Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation."); return null; } + else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation."); + return null; + } else { throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString()); @@ -1148,6 +1159,7 @@ namespace GitHub.Runner.Worker NodeJS, Plugin, Script, + Composite, } public sealed class ContainerActionExecutionData : ActionExecutionData @@ -1204,6 +1216,14 @@ namespace GitHub.Runner.Worker public override bool HasPost => false; } + public sealed class CompositeActionExecutionData : ActionExecutionData + { + public override ActionExecutionType ExecutionType => ActionExecutionType.Composite; + public override bool HasPre => false; + public override bool HasPost => false; + public List Steps { get; set; } + } + public abstract class ActionExecutionData { private string _initCondition = $"{Constants.Expressions.Always}()"; diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 4e9149d26..9095f498d 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -14,6 +14,7 @@ using YamlDotNet.Core; using YamlDotNet.Core.Events; using System.Globalization; using System.Linq; +using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -92,7 +93,7 @@ namespace GitHub.Runner.Worker break; case "runs": - actionDefinition.Execution = ConvertRuns(context, actionPair.Value); + actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value); break; default: Trace.Info($"Ignore action property {propertyName}."); @@ -284,7 +285,7 @@ namespace GitHub.Runner.Worker // Add the file table if (_fileTable?.Count > 0) { - for (var i = 0 ; i < _fileTable.Count ; i++) + for (var i = 0; i < _fileTable.Count; i++) { result.GetFileId(_fileTable[i]); } @@ -294,6 +295,7 @@ namespace GitHub.Runner.Worker } private ActionExecutionData ConvertRuns( + IExecutionContext executionContext, TemplateContext context, TemplateToken inputsToken) { @@ -311,6 +313,8 @@ namespace GitHub.Runner.Worker var postToken = default(StringToken); var postEntrypointToken = default(StringToken); var postIfToken = default(StringToken); + var stepsLoaded = default(List); + foreach (var run in runsMapping) { var runsKey = run.Key.AssertString("runs key").Value; @@ -355,6 +359,15 @@ namespace GitHub.Runner.Worker case "pre-if": preIfToken = run.Value.AssertString("pre-if"); break; + case "steps": + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + var steps = run.Value.AssertSequence("steps"); + var evaluator = executionContext.ToPipelineTemplateEvaluator(); + stepsLoaded = evaluator.LoadCompositeSteps(steps); + break; + } + throw new Exception("You aren't supposed to be using Composite Actions yet!"); default: Trace.Info($"Ignore run property {runsKey}."); break; @@ -402,6 +415,21 @@ namespace GitHub.Runner.Worker }; } } + else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + if (stepsLoaded == null) + { + // TODO: Add a more helpful error message + including file name, etc. to show user that it's because of their yaml file + throw new ArgumentNullException($"No steps provided."); + } + else + { + return new CompositeActionExecutionData() + { + Steps = stepsLoaded, + }; + } + } else { throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead."); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 0318974b0..cea1e2fd4 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -63,7 +63,7 @@ namespace GitHub.Runner.Worker JobContext JobContext { get; } // Only job level ExecutionContext has JobSteps - Queue JobSteps { get; } + List JobSteps { get; } // Only job level ExecutionContext has PostJobSteps Stack PostJobSteps { get; } @@ -105,6 +105,7 @@ namespace GitHub.Runner.Worker // others void ForceTaskComplete(); void RegisterPostJobStep(IStep step); + void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location); } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -159,7 +160,7 @@ namespace GitHub.Runner.Worker public List ServiceContainers { get; private set; } // Only job level ExecutionContext has JobSteps - public Queue JobSteps { get; private set; } + public List JobSteps { get; private set; } // Only job level ExecutionContext has PostJobSteps public Stack PostJobSteps { get; private set; } @@ -169,7 +170,6 @@ namespace GitHub.Runner.Worker public bool EchoOnActionCommand { get; set; } - public TaskResult? Result { get @@ -266,6 +266,20 @@ namespace GitHub.Runner.Worker Root.PostJobSteps.Push(step); } + /// + /// Helper function used in CompositeActionHandler::RunAsync to + /// add a child node, aka a step, to the current job to the Root.JobSteps based on the location. + /// + public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location) + { + // TODO: For UI purposes, look at figuring out how to condense steps in one node => maybe use the same previous GUID + var newGuid = Guid.NewGuid(); + step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null); + step.ExecutionContext.ExpressionValues["inputs"] = inputsData; + // TODO: confirm whether not copying message contexts is safe + Root.JobSteps.Insert(location, step); + } + public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null) { Trace.Entering(); @@ -660,7 +674,7 @@ namespace GitHub.Runner.Worker PrependPath = new List(); // JobSteps for job ExecutionContext - JobSteps = new Queue(); + JobSteps = new List(); // PostJobSteps for job ExecutionContext PostJobSteps = new Stack(); diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs new file mode 100644 index 000000000..7c5b25ed5 --- /dev/null +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; +using Pipelines = GitHub.DistributedTask.Pipelines; + + +namespace GitHub.Runner.Worker.Handlers +{ + [ServiceLocator(Default = typeof(CompositeActionHandler))] + public interface ICompositeActionHandler : IHandler + { + CompositeActionExecutionData Data { get; set; } + } + public sealed class CompositeActionHandler : Handler, ICompositeActionHandler + { + public CompositeActionExecutionData Data { get; set; } + + public Task RunAsync(ActionRunStage stage) + { + // Validate args. + Trace.Entering(); + ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); + ArgUtil.NotNull(Inputs, nameof(Inputs)); + + var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(githubContext, nameof(githubContext)); + + var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp); + + // Resolve action steps + var actionSteps = Data.Steps; + + // Create Context Data to reuse for each composite action step + var inputsData = new DictionaryContextData(); + foreach (var i in Inputs) + { + inputsData[i.Key] = new StringContextData(i.Value); + } + + // Add each composite action step to the front of the queue + int location = 0; + foreach (Pipelines.ActionStep aStep in actionSteps) + { + // Ex: + // runs: + // using: "composite" + // steps: + // - uses: example/test-composite@v2 (a) + // - run echo hello world (b) + // - run echo hello world 2 (c) + // + // ethanchewy/test-composite/action.yaml + // runs: + // using: "composite" + // steps: + // - run echo hello world 3 (d) + // - run echo hello world 4 (e) + // + // Steps processed as follow: + // | a | + // | a | => | d | + // (Run step d) + // | a | + // | a | => | e | + // (Run step e) + // | a | + // (Run step a) + // | b | + // (Run step b) + // | c | + // (Run step c) + // Done. + + var actionRunner = HostContext.CreateService(); + actionRunner.Action = aStep; + actionRunner.Stage = stage; + actionRunner.Condition = aStep.Condition; + actionRunner.DisplayName = aStep.DisplayName; + // TODO: Do we need to add any context data from the job message? + // (See JobExtension.cs ~line 236) + + ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location); + location++; + } + + return Task.CompletedTask; + } + + } +} diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index 0f2413ef5..db4d6559c 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -66,6 +66,11 @@ namespace GitHub.Runner.Worker.Handlers handler = HostContext.CreateService(); (handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData; } + else if (data.ExecutionType == ActionExecutionType.Composite) + { + handler = HostContext.CreateService(); + (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData; + } else { // This should never happen. diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 31dfb1714..33b291adb 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -152,7 +152,7 @@ namespace GitHub.Runner.Worker { foreach (var step in jobSteps) { - jobContext.JobSteps.Enqueue(step); + jobContext.JobSteps.Add(step); } await stepsRunner.RunAsync(jobContext); diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 485a4cdf9..e75d2e106 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -59,14 +59,15 @@ namespace GitHub.Runner.Worker checkPostJobActions = true; while (jobContext.PostJobSteps.TryPop(out var postStep)) { - jobContext.JobSteps.Enqueue(postStep); + jobContext.JobSteps.Add(postStep); } continue; } - var step = jobContext.JobSteps.Dequeue(); - var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null; + var step = jobContext.JobSteps[0]; + jobContext.JobSteps.RemoveAt(0); + var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps[0] : null; Trace.Info($"Processing step: DisplayName='{step.DisplayName}'"); ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext)); @@ -409,7 +410,11 @@ namespace GitHub.Runner.Worker scope = scopesToInitialize.Pop(); executionContext.Debug($"Initializing scope '{scope.Name}'"); executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName); - executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + } var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); var inputs = default(DictionaryContextData); try @@ -432,7 +437,11 @@ namespace GitHub.Runner.Worker // Setup expression values var scopeName = executionContext.ScopeName; executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName); - executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + } return true; } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 7a8b847d3..cb1d90b2e 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -32,7 +32,8 @@ "one-of": [ "container-runs", "node12-runs", - "plugin-runs" + "plugin-runs", + "composite-runs" ] }, "container-runs": { @@ -83,6 +84,36 @@ } } }, + "composite-runs": { + "mapping": { + "properties": { + "using": "non-empty-string", + "steps": "composite-steps" + } + } + }, + "composite-steps": { + "context": [ + "github", + "needs", + "strategy", + "matrix", + "secrets", + "steps", + "inputs", + "job", + "runner", + "env", + "always(0,0)", + "failure(0,0)", + "cancelled(0,0)", + "success(0,0)", + "hashFiles(1,255)" + ], + "sequence": { + "item-type": "any" + } + }, "container-runs-context": { "context": [ "inputs" diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index d1c886dd8..f2609462b 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -65,6 +65,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating public const String StepEnv = "step-env"; public const String StepIfResult = "step-if-result"; public const String Steps = "steps"; + public const String StepsInTemplate = "steps-in-template"; public const String StepsScopeInputs = "steps-scope-inputs"; public const String StepsScopeOutputs = "steps-scope-outputs"; public const String StepsTemplateRoot = "steps-template-root"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 43be43d33..a952f58fb 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -29,7 +29,6 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult); return evaluationResult.IsTruthy; } - internal static Boolean? ConvertToStepContinueOnError( TemplateContext context, TemplateToken token, @@ -264,5 +263,351 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating return result; } + + //Note: originally was List but we need to change to List to use the "Inputs" attribute + internal static List ConvertToSteps( + TemplateContext context, + TemplateToken steps) + { + var stepsSequence = steps.AssertSequence($"job {PipelineTemplateConstants.Steps}"); + + var result = new List(); + foreach (var stepsItem in stepsSequence) + { + var step = ConvertToStep(context, stepsItem); + if (step != null) // step = null means we are hitting error during step conversion, there should be an error in context.errors + { + if (step.Enabled) + { + result.Add(step); + } + } + } + + return result; + } + + private static ActionStep ConvertToStep( + TemplateContext context, + TemplateToken stepsItem) + { + var step = stepsItem.AssertMapping($"{PipelineTemplateConstants.Steps} item"); + var continueOnError = default(ScalarToken); + var env = default(TemplateToken); + var id = default(StringToken); + var ifCondition = default(String); + var ifToken = default(ScalarToken); + var name = default(ScalarToken); + var run = default(ScalarToken); + var scope = default(StringToken); + var timeoutMinutes = default(ScalarToken); + var uses = default(StringToken); + var with = default(TemplateToken); + var workingDir = default(ScalarToken); + var path = default(ScalarToken); + var clean = default(ScalarToken); + var fetchDepth = default(ScalarToken); + var lfs = default(ScalarToken); + var submodules = default(ScalarToken); + var shell = default(ScalarToken); + + foreach (var stepProperty in step) + { + var propertyName = stepProperty.Key.AssertString($"{PipelineTemplateConstants.Steps} item key"); + + switch (propertyName.Value) + { + case PipelineTemplateConstants.Clean: + clean = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Clean}"); + break; + + case PipelineTemplateConstants.ContinueOnError: + ConvertToStepContinueOnError(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + continueOnError = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} {PipelineTemplateConstants.ContinueOnError}"); + break; + + case PipelineTemplateConstants.Env: + ConvertToStepEnvironment(context, stepProperty.Value, StringComparer.Ordinal, allowExpressions: true); // Validate early if possible + env = stepProperty.Value; + break; + + case PipelineTemplateConstants.FetchDepth: + fetchDepth = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.FetchDepth}"); + break; + + case PipelineTemplateConstants.Id: + id = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Id}"); + if (!NameValidation.IsValid(id.Value, true)) + { + context.Error(id, $"Step id {id.Value} is invalid. Ids must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'"); + } + break; + + case PipelineTemplateConstants.If: + ifToken = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.If}"); + break; + + case PipelineTemplateConstants.Lfs: + lfs = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Lfs}"); + break; + + case PipelineTemplateConstants.Name: + name = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Name}"); + break; + + case PipelineTemplateConstants.Path: + path = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Path}"); + break; + + case PipelineTemplateConstants.Run: + run = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Run}"); + break; + + case PipelineTemplateConstants.Shell: + shell = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Shell}"); + break; + + case PipelineTemplateConstants.Scope: + scope = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Scope}"); + break; + + case PipelineTemplateConstants.Submodules: + submodules = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Submodules}"); + break; + + case PipelineTemplateConstants.TimeoutMinutes: + ConvertToStepTimeout(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + timeoutMinutes = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.TimeoutMinutes}"); + break; + + case PipelineTemplateConstants.Uses: + uses = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + break; + + case PipelineTemplateConstants.With: + ConvertToStepInputs(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + with = stepProperty.Value; + break; + + case PipelineTemplateConstants.WorkingDirectory: + workingDir = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.WorkingDirectory}"); + break; + + default: + propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Steps} item key"); // throws + break; + } + } + + // Fixup the if-condition + var isDefaultScope = String.IsNullOrEmpty(scope?.Value); + ifCondition = ConvertToIfCondition(context, ifToken, false, isDefaultScope); + + if (run != null) + { + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Environment = env, + Reference = new ScriptReference(), + }; + + var inputs = new MappingToken(null, null, null); + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Script), run); + + if (workingDir != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.WorkingDirectory), workingDir); + } + + if (shell != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Shell), shell); + } + + result.Inputs = inputs; + + return result; + } + else + { + uses.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Inputs = with, + Environment = env, + }; + + if (uses.Value.StartsWith("docker://", StringComparison.Ordinal)) + { + var image = uses.Value.Substring("docker://".Length); + result.Reference = new ContainerRegistryReference { Image = image }; + } + else if (uses.Value.StartsWith("./") || uses.Value.StartsWith(".\\")) + { + result.Reference = new RepositoryPathReference + { + RepositoryType = PipelineConstants.SelfAlias, + Path = uses.Value + }; + } + else + { + var usesSegments = uses.Value.Split('@'); + var pathSegments = usesSegments[0].Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + var gitRef = usesSegments.Length == 2 ? usesSegments[1] : String.Empty; + + if (usesSegments.Length != 2 || + pathSegments.Length < 2 || + String.IsNullOrEmpty(pathSegments[0]) || + String.IsNullOrEmpty(pathSegments[1]) || + String.IsNullOrEmpty(gitRef)) + { + // todo: loc + context.Error(uses, $"Expected format {{org}}/{{repo}}[/path]@ref. Actual '{uses.Value}'"); + } + else + { + var repositoryName = $"{pathSegments[0]}/{pathSegments[1]}"; + var directoryPath = pathSegments.Length > 2 ? String.Join("/", pathSegments.Skip(2)) : String.Empty; + + result.Reference = new RepositoryPathReference + { + RepositoryType = RepositoryTypes.GitHub, + Name = repositoryName, + Ref = gitRef, + Path = directoryPath, + }; + } + } + + return result; + } + } + + /// + /// When empty, default to "success()". + /// When a status function is not referenced, format as "success() && <CONDITION>". + /// + private static String ConvertToIfCondition( + TemplateContext context, + TemplateToken token, + Boolean isJob, + Boolean isDefaultScope) + { + String condition; + if (token is null) + { + condition = null; + } + else if (token is BasicExpressionToken expressionToken) + { + condition = expressionToken.Expression; + } + else + { + var stringToken = token.AssertString($"{(isJob ? "job" : "step")} {PipelineTemplateConstants.If}"); + condition = stringToken.Value; + } + + if (String.IsNullOrWhiteSpace(condition)) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var expressionParser = new ExpressionParser(); + var functions = default(IFunctionInfo[]); + var namedValues = default(INamedValueInfo[]); + if (isJob) + { + namedValues = s_jobIfNamedValues; + // TODO: refactor into seperate functions + // functions = PhaseCondition.FunctionInfo; + } + else + { + namedValues = isDefaultScope ? s_stepNamedValues : s_stepInTemplateNamedValues; + functions = s_stepConditionFunctions; + } + + var node = default(ExpressionNode); + try + { + node = expressionParser.CreateTree(condition, null, namedValues, functions) as ExpressionNode; + } + catch (Exception ex) + { + context.Error(token, ex); + return null; + } + + if (node == null) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var hasStatusFunction = node.Traverse().Any(x => + { + if (x is Function function) + { + return String.Equals(function.Name, PipelineTemplateConstants.Always, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Cancelled, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Failure, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Success, StringComparison.OrdinalIgnoreCase); + } + + return false; + }); + + return hasStatusFunction ? condition : $"{PipelineTemplateConstants.Success}() && ({condition})"; + } + + private static readonly INamedValueInfo[] s_jobIfNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepInTemplateNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.Inputs), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly IFunctionInfo[] s_stepConditionFunctions = new IFunctionInfo[] + { + new FunctionInfo(PipelineTemplateConstants.Always, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Cancelled, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Failure, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Success, 0, 0), + new FunctionInfo(PipelineTemplateConstants.HashFiles, 1, Byte.MaxValue), + }; } } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index a36f5b7e3..55076e670 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -159,6 +159,32 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating return result; } + public List LoadCompositeSteps( + TemplateToken token + ) + { + var result = default(List); + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(null, null, setMissingContext: false); + // TODO: we might want to to have a bool to prevent it from filling in with missing context w/ dummy variables + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsInTemplate, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = PipelineTemplateConverter.ConvertToSteps(context, token); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + return result; + } + + public Dictionary EvaluateStepEnvironment( TemplateToken token, DictionaryContextData contextData, @@ -400,7 +426,8 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating private TemplateContext CreateContext( DictionaryContextData contextData, IList expressionFunctions, - IEnumerable> expressionState = null) + IEnumerable> expressionState = null, + bool setMissingContext = true) { var result = new TemplateContext { @@ -449,18 +476,21 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating // - Evaluating early when all referenced contexts are available, even though all allowed // contexts may not yet be available. For example, evaluating step display name can often // be performed early. - foreach (var name in s_expressionValueNames) + if (setMissingContext) { - if (!result.ExpressionValues.ContainsKey(name)) + foreach (var name in s_expressionValueNames) { - result.ExpressionValues[name] = null; + if (!result.ExpressionValues.ContainsKey(name)) + { + result.ExpressionValues[name] = null; + } } - } - foreach (var name in s_expressionFunctionNames) - { - if (!functionNames.Contains(name)) + foreach (var name in s_expressionFunctionNames) { - result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + if (!functionNames.Contains(name)) + { + result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + } } } diff --git a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs index 2d599dd9c..2e03671fb 100644 --- a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs @@ -94,5 +94,12 @@ namespace GitHub.DistributedTask.Pipelines public static readonly String Resources = "resources"; public static readonly String All = "all"; } + + public static class ScriptStepInputs + { + public static readonly String Script = "script"; + public static readonly String WorkingDirectory = "workingDirectory"; + public static readonly String Shell = "shell"; + } } } diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 2d7cb9fb0..1dfee2252 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -80,7 +80,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -115,7 +115,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -154,7 +154,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -208,7 +208,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -287,7 +287,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -330,7 +330,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Step.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Step.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -361,7 +361,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -391,7 +391,7 @@ namespace GitHub.Runner.Common.Tests.Worker { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -417,7 +417,7 @@ namespace GitHub.Runner.Common.Tests.Worker _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -455,7 +455,7 @@ namespace GitHub.Runner.Common.Tests.Worker _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -493,7 +493,7 @@ namespace GitHub.Runner.Common.Tests.Worker _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -524,7 +524,7 @@ namespace GitHub.Runner.Common.Tests.Worker _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object, step3.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -560,7 +560,7 @@ namespace GitHub.Runner.Common.Tests.Worker _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object, step3.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); From 121deedeb5861767b436c789d73a8faa746981d8 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 30 Jun 2020 17:25:47 -0400 Subject: [PATCH 11/11] Fix trailing '.0' for Int64 values (#572) --- .../DTPipelines/Pipelines/ContextData/NumberContextData.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs b/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs index 07d2172bc..82ad590b1 100644 --- a/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs +++ b/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs @@ -42,7 +42,12 @@ namespace GitHub.DistributedTask.Pipelines.ContextData var floored = Math.Floor(m_value); if (m_value == floored && m_value <= (Double)Int32.MaxValue && m_value >= (Double)Int32.MinValue) { - Int32 flooredInt = (Int32)floored; + var flooredInt = (Int32)floored; + return (JToken)flooredInt; + } + else if (m_value == floored && m_value <= (Double)Int64.MaxValue && m_value >= (Double)Int64.MinValue) + { + var flooredInt = (Int64)floored; return (JToken)flooredInt; } else