From e601a3f4be7d99d0ee4d9014ad3665ec7fc32a85 Mon Sep 17 00:00:00 2001 From: Ethan Chiu Date: Thu, 11 Jun 2020 17:54:24 -0400 Subject: [PATCH] Add necessary files + add functionality to convert to ActionStep object --- src/Runner.Worker/ActionManager.cs | 2 +- src/Runner.Worker/ActionManifestManager.cs | 33 +- .../Handlers/CompositeActionHandler.cs | 112 ++++-- src/Runner.Worker/Handlers/HandlerFactory.cs | 1 + src/Runner.Worker/JobExtension.cs | 8 + src/Runner.Worker/JobRunner.cs | 1 + src/Runner.Worker/action_yaml.json | 166 +-------- .../PipelineTemplateConverter.cs | 348 +++++++++++++++++- .../PipelineTemplateEvaluator.cs | 60 ++- .../Pipelines/PipelineConstants.cs | 7 + 10 files changed, 525 insertions(+), 213 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index ab4effb1b..f622be101 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -1286,7 +1286,7 @@ namespace GitHub.Runner.Worker public override bool HasPre => false; public override bool HasPost => false; - public MappingToken Steps {get; set;} + public List Steps {get; set;} // public string Script { get; set; } diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 7137bacac..8c6f6d4ac 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -14,6 +14,7 @@ using YamlDotNet.Core; using YamlDotNet.Core.Events; using System.Globalization; using System.Linq; +using GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -92,7 +93,7 @@ namespace GitHub.Runner.Worker break; case "runs": - actionDefinition.Execution = ConvertRuns(context, actionPair.Value); + actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value); break; default: Trace.Info($"Ignore action property {propertyName}."); @@ -294,8 +295,10 @@ namespace GitHub.Runner.Worker } private ActionExecutionData ConvertRuns( + IExecutionContext executionContext, TemplateContext context, - TemplateToken inputsToken) + TemplateToken inputsToken + ) { var runsMapping = inputsToken.AssertMapping("runs"); var usingToken = default(StringToken); @@ -316,7 +319,9 @@ namespace GitHub.Runner.Worker // var stepsToken = runsMapping.AssertMapping("steps"); // Actually, not sure, let's just set it to MappingToken since AssertMapping("steps") // returns a MappingToken - var stepsToken = default(MappingToken); + // var stepsToken = default(SequenceToken); + var stepsLoaded = default(List); + // It should be a array (aka sequence) foreach (var run in runsMapping) { @@ -363,13 +368,20 @@ namespace GitHub.Runner.Worker preIfToken = run.Value.AssertString("pre-if"); break; case "steps": - stepsToken = run.Value.AssertMapping("steps"); + // stepsToken = run.Value.AssertMapping("steps"); // Maybe insert a for loop here instead since MappingToken is not supposed to be used in HandlerFactory.cs - // var steps = run.Value.AssertMapping("steps"); + // Just support 1 layer of steps w/ just run + var steps = run.Value.AssertMapping("steps"); // foreach (var s in steps) { // // Create list of steps - // // + // loadS // } + // foreach (var run in runsMapping) + // stepsToken = List + // Call load steps here + // var test = new PipelineTemplateEvaluator(); + var evaluator = executionContext.ToPipelineTemplateEvaluator(); + stepsLoaded = evaluator.LoadSteps(steps, null, null); break; default: Trace.Info($"Ignore run property {runsKey}."); @@ -421,15 +433,18 @@ namespace GitHub.Runner.Worker // TODO: add composite stuff here else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase)) { - if (stepsToken.Count <= 0) - { + // if (stepsToken.Count <= 0) + // { + // throw new ArgumentNullException($"No steps provided."); + // } + if (stepsLoaded == null) { throw new ArgumentNullException($"No steps provided."); } else { return new CompositeActionExecutionData() { - Steps = stepsToken + Steps = stepsLoaded }; } } diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index c34331dc8..0be6ecbaa 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -7,6 +7,9 @@ using GitHub.DistributedTask.WebApi; using Pipelines = GitHub.DistributedTask.Pipelines; using System; using System.Linq; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using System.Collections.Generic; +using GitHub.DistributedTask.Pipelines.ContextData; namespace GitHub.Runner.Worker.Handlers { @@ -21,50 +24,103 @@ namespace GitHub.Runner.Worker.Handlers { public CompositeActionExecutionData Data { get; set; } + public override void PrintActionDetails(ActionRunStage stage) + { + + } + public async Task RunAsync(ActionRunStage stage) { - // Copied from NodEscriptActionHandler.cs + // DELETE LATER + // await Task.Yield(); + + // Copied from ScriptHandler.cs // Validate args. Trace.Entering(); - ArgUtil.NotNull(Data, nameof(Data)); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Inputs, nameof(Inputs)); - ArgUtil.Directory(ActionDirectory, nameof(ActionDirectory)); - // Update the env dictionary. - AddInputsToEnvironment(); - AddPrependPathToEnvironment(); + var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(githubContext, nameof(githubContext)); - // expose context to environment - // for example, this is how we know what OS the runner is running on - foreach (var context in ExecutionContext.ExpressionValues) + var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp); + + // Resolve steps + var target = Data.Steps; + + // For now, just assume it is 1 Run step + // We will adapt this in the future. + var runStepInputs= target[0].Inputs; + + + + // For now assume it's just a run step. + // runStep.TryGetValue("run", out var runDefaults); + string prependPath = string.Join(Path.PathSeparator.ToString(), runStep..Reverse()); + + + + + + + // Copied from ScriptHandler.cs and ScriptHandlerHelpers.cs to handle bash commands. + string argFormat; + string shellCommand; + string shellCommandPath = null; + bool validateShellOnHost = !(StepHost is ContainerStepHost); + string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse()); + string shell = null; + if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell)) { - if (context.Value is IEnvironmentContextData runtimeContext && runtimeContext != null) + // TODO: figure out how defaults interact with template later + // for now, we won't check job.defaults if we are inside a template. + if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults)) { - foreach (var env in runtimeContext.GetRuntimeEnvironmentVariables()) + runDefaults.TryGetValue("shell", out shell); + } + } + if (string.IsNullOrEmpty(shell)) + { +#if OS_WINDOWS + shellCommand = "pwsh"; + if (validateShellOnHost) + { + shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath); + if (string.IsNullOrEmpty(shellCommandPath)) { - Environment[env.Key] = env.Value; + shellCommand = "powershell"; + Trace.Info($"Defaulting to {shellCommand}"); + shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath); } } +#else + shellCommand = "sh"; + if (validateShellOnHost) + { + shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath); + } +#endif + argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand); + } + else + { + var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell); + shellCommand = parsed.shellCommand; + if (validateShellOnHost) + { + shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath); + } + + argFormat = $"{parsed.shellArgs}".TrimStart(); + if (string.IsNullOrEmpty(argFormat)) + { + argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand); + } } - // Add Actions Runtime server info - var systemConnection = ExecutionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); - Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri; - Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken]; - if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl)) - { - Environment["ACTIONS_CACHE_URL"] = cacheUrl; - } - - // Resolve steps - // How do I handle the MappingToken? - MappingToken target = null; - if (stage == ActionRunStage.Main) - { - target = Data.Steps; - } + + diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index 0a63ce2ef..e225379e9 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -68,6 +68,7 @@ namespace GitHub.Runner.Worker.Handlers } else if (data.ExecutionType == ActionExecutionType.Composite) { + // TODO // Runner plugin handler = HostContext.CreateService(); // handler = CompositeHandler; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 235cc9063..df1ad01b3 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -239,6 +239,7 @@ namespace GitHub.Runner.Worker } actionRunner.TryEvaluateDisplayName(contextData, context); + jobSteps.Add(actionRunner); if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep)) @@ -284,6 +285,13 @@ namespace GitHub.Runner.Worker intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState); actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName, intraActionState); } + + // TODO: Maybe add EvaluateStep stuff here too? + // TODO: INSERT CONVERTSTEPS FUNCTION HERE FOR EVALUATING STEPS + // Maybe we don't need to do this here do we need to initialize the job? + context.Debug("Evaluating job evaluating steps"); + var stepsEvaluation = templateEvaluator.EvaluateSteps(contextData, context, context.ExpressionFunctions); + //////// } List steps = new List(); diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 915a43fbf..3c46a8247 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -60,6 +60,7 @@ namespace GitHub.Runner.Worker { // Create the job execution context. jobContext = HostContext.CreateService(); + // HERE jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index bc36aea58..37b579f75 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -98,171 +98,7 @@ } }, "steps-item": { - "one-of": [ - "run-step", - "regular-step", - "steps-template-reference" - ] - }, - "run-step": { - "mapping": { - "properties": { - "name": "string-steps-context", - "id": "non-empty-string", - "if": "step-if", - "timeout-minutes": "number-steps-context", - "run": { - "type": "string-steps-context", - "required": true - }, - "continue-on-error": "boolean-steps-context", - "env": "step-env", - "working-directory": "string-steps-context", - "shell": "non-empty-string" - } - } - }, - "regular-step": { - "mapping": { - "properties": { - "name": "string-steps-context", - "id": "non-empty-string", - "if": "step-if", - "continue-on-error": "boolean-steps-context", - "timeout-minutes": "number-steps-context", - "uses": { - "type": "non-empty-string", - "required": true - }, - "with": "step-with", - "env": "step-env" - } - } - }, - "steps-template-reference": { - "mapping": { - "properties": { - "template": "non-empty-string", - "id": "non-empty-string", - "inputs": "steps-template-reference-inputs" - } - } - }, - "string-steps-context": { - "context": [ - "github", - "needs", - "strategy", - "matrix", - "secrets", - "steps", - "job", - "runner", - "env", - "hashFiles(1,255)" - ], - "string": {} - }, - "step-if": { - "context": [ - "github", - "needs", - "strategy", - "matrix", - "steps", - "job", - "runner", - "env", - "always(0,0)", - "failure(0,0)", - "cancelled(0,0)", - "success(0,0)", - "hashFiles(1,255)" - ], - "string": {} - }, - "number-steps-context": { - "context": [ - "github", - "needs", - "strategy", - "matrix", - "secrets", - "steps", - "job", - "runner", - "env", - "hashFiles(1,255)" - ], - "number": {} - }, - "boolean-steps-context": { - "context": [ - "github", - "needs", - "strategy", - "matrix", - "secrets", - "steps", - "job", - "runner", - "env", - "hashFiles(1,255)" - ], - "boolean": {} - }, - "step-env": { - "context": [ - "github", - "needs", - "strategy", - "matrix", - "secrets", - "steps", - "job", - "runner", - "env", - "hashFiles(1,255)" - ], - "mapping": { - "loose-key-type": "non-empty-string", - "loose-value-type": "string" - } - }, - "step-with": { - "context": [ - "github", - "needs", - "strategy", - "matrix", - "secrets", - "steps", - "job", - "runner", - "env", - "hashFiles(1,255)" - ], - "mapping": { - "loose-key-type": "non-empty-string", - "loose-value-type": "string" - } - }, - "steps-template-reference-inputs": { - "context": [ - "github", - "needs", - "strategy", - "matrix", - "secrets", - "steps", - "job", - "runner", - "env" - ], - "mapping": { - "loose-key-type": "non-empty-string", - "loose-value-type": "string" - } + "run-step": "non-empty-string" }, "container-runs-context": { "context": [ diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 43be43d33..e40df08e8 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -16,6 +16,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating { internal static class PipelineTemplateConverter { + internal static Boolean ConvertToIfResult( TemplateContext context, TemplateToken ifResult) @@ -29,7 +30,6 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult); return evaluationResult.IsTruthy; } - internal static Boolean? ConvertToStepContinueOnError( TemplateContext context, TemplateToken token, @@ -264,5 +264,351 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating return result; } + + //Note: originally was List but we need to change to List to use the "Inputs" attribute + internal static List ConvertToSteps( + TemplateContext context, + TemplateToken steps) + { + var stepsSequence = steps.AssertSequence($"job {PipelineTemplateConstants.Steps}"); + + var result = new List(); + foreach (var stepsItem in stepsSequence) + { + var step = ConvertToStep(context, stepsItem); + if (step != null) // step = null means we are hitting error during step conversion, there should be an error in context.errors + { + if (step.Enabled) + { + result.Add(step); + } + } + } + + return result; + } + + private static ActionStep ConvertToStep( + TemplateContext context, + TemplateToken stepsItem) + { + var step = stepsItem.AssertMapping($"{PipelineTemplateConstants.Steps} item"); + var continueOnError = default(ScalarToken); + var env = default(TemplateToken); + var id = default(StringToken); + var ifCondition = default(String); + var ifToken = default(ScalarToken); + var name = default(ScalarToken); + var run = default(ScalarToken); + var scope = default(StringToken); + var timeoutMinutes = default(ScalarToken); + var uses = default(StringToken); + var with = default(TemplateToken); + var workingDir = default(ScalarToken); + var path = default(ScalarToken); + var clean = default(ScalarToken); + var fetchDepth = default(ScalarToken); + var lfs = default(ScalarToken); + var submodules = default(ScalarToken); + var shell = default(ScalarToken); + + foreach (var stepProperty in step) + { + var propertyName = stepProperty.Key.AssertString($"{PipelineTemplateConstants.Steps} item key"); + + switch (propertyName.Value) + { + case PipelineTemplateConstants.Clean: + clean = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Clean}"); + break; + + case PipelineTemplateConstants.ContinueOnError: + ConvertToStepContinueOnError(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + continueOnError = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} {PipelineTemplateConstants.ContinueOnError}"); + break; + + case PipelineTemplateConstants.Env: + ConvertToStepEnvironment(context, stepProperty.Value, StringComparer.Ordinal, allowExpressions: true); // Validate early if possible + env = stepProperty.Value; + break; + + case PipelineTemplateConstants.FetchDepth: + fetchDepth = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.FetchDepth}"); + break; + + case PipelineTemplateConstants.Id: + id = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Id}"); + if (!NameValidation.IsValid(id.Value, true)) + { + context.Error(id, $"Step id {id.Value} is invalid. Ids must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'"); + } + break; + + case PipelineTemplateConstants.If: + ifToken = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.If}"); + break; + + case PipelineTemplateConstants.Lfs: + lfs = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Lfs}"); + break; + + case PipelineTemplateConstants.Name: + name = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Name}"); + break; + + case PipelineTemplateConstants.Path: + path = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Path}"); + break; + + case PipelineTemplateConstants.Run: + run = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Run}"); + break; + + case PipelineTemplateConstants.Shell: + shell = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Shell}"); + break; + + case PipelineTemplateConstants.Scope: + scope = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Scope}"); + break; + + case PipelineTemplateConstants.Submodules: + submodules = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Submodules}"); + break; + + case PipelineTemplateConstants.TimeoutMinutes: + ConvertToStepTimeout(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + timeoutMinutes = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.TimeoutMinutes}"); + break; + + case PipelineTemplateConstants.Uses: + uses = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + break; + + case PipelineTemplateConstants.With: + ConvertToStepInputs(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + with = stepProperty.Value; + break; + + case PipelineTemplateConstants.WorkingDirectory: + workingDir = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.WorkingDirectory}"); + break; + + default: + propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Steps} item key"); // throws + break; + } + } + + // Fixup the if-condition + var isDefaultScope = String.IsNullOrEmpty(scope?.Value); + ifCondition = ConvertToIfCondition(context, ifToken, false, isDefaultScope); + + if (run != null) + { + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Environment = env, + Reference = new ScriptReference(), + }; + + var inputs = new MappingToken(null, null, null); + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Script), run); + + if (workingDir != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.WorkingDirectory), workingDir); + } + + if (shell != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Shell), shell); + } + + result.Inputs = inputs; + + return result; + } + else + { + uses.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Inputs = with, + Environment = env, + }; + + if (uses.Value.StartsWith("docker://", StringComparison.Ordinal)) + { + var image = uses.Value.Substring("docker://".Length); + result.Reference = new ContainerRegistryReference { Image = image }; + } + else if (uses.Value.StartsWith("./") || uses.Value.StartsWith(".\\")) + { + result.Reference = new RepositoryPathReference + { + RepositoryType = PipelineConstants.SelfAlias, + Path = uses.Value + }; + } + else + { + var usesSegments = uses.Value.Split('@'); + var pathSegments = usesSegments[0].Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + var gitRef = usesSegments.Length == 2 ? usesSegments[1] : String.Empty; + + if (usesSegments.Length != 2 || + pathSegments.Length < 2 || + String.IsNullOrEmpty(pathSegments[0]) || + String.IsNullOrEmpty(pathSegments[1]) || + String.IsNullOrEmpty(gitRef)) + { + // todo: loc + context.Error(uses, $"Expected format {{org}}/{{repo}}[/path]@ref. Actual '{uses.Value}'"); + } + else + { + var repositoryName = $"{pathSegments[0]}/{pathSegments[1]}"; + var directoryPath = pathSegments.Length > 2 ? String.Join("/", pathSegments.Skip(2)) : String.Empty; + + result.Reference = new RepositoryPathReference + { + RepositoryType = RepositoryTypes.GitHub, + Name = repositoryName, + Ref = gitRef, + Path = directoryPath, + }; + } + } + + return result; + } + } + + /// + /// When empty, default to "success()". + /// When a status function is not referenced, format as "success() && <CONDITION>". + /// + private static String ConvertToIfCondition( + TemplateContext context, + TemplateToken token, + Boolean isJob, + Boolean isDefaultScope) + { + String condition; + if (token is null) + { + condition = null; + } + else if (token is BasicExpressionToken expressionToken) + { + condition = expressionToken.Expression; + } + else + { + var stringToken = token.AssertString($"{(isJob ? "job" : "step")} {PipelineTemplateConstants.If}"); + condition = stringToken.Value; + } + + if (String.IsNullOrWhiteSpace(condition)) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var expressionParser = new ExpressionParser(); + var functions = default(IFunctionInfo[]); + var namedValues = default(INamedValueInfo[]); + if (isJob) + { + namedValues = s_jobIfNamedValues; + // TODO: refactor into seperate functions + // functions = PhaseCondition.FunctionInfo; + } + else + { + namedValues = isDefaultScope ? s_stepNamedValues : s_stepInTemplateNamedValues; + functions = s_stepConditionFunctions; + } + + var node = default(ExpressionNode); + try + { + node = expressionParser.CreateTree(condition, null, namedValues, functions) as ExpressionNode; + } + catch (Exception ex) + { + context.Error(token, ex); + return null; + } + + if (node == null) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var hasStatusFunction = node.Traverse().Any(x => + { + if (x is Function function) + { + return String.Equals(function.Name, PipelineTemplateConstants.Always, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Cancelled, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Failure, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Success, StringComparison.OrdinalIgnoreCase); + } + + return false; + }); + + return hasStatusFunction ? condition : $"{PipelineTemplateConstants.Success}() && ({condition})"; + } + + private static readonly INamedValueInfo[] s_jobIfNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepInTemplateNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.Inputs), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly IFunctionInfo[] s_stepConditionFunctions = new IFunctionInfo[] + { + new FunctionInfo(PipelineTemplateConstants.Always, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Cancelled, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Failure, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Success, 0, 0), + new FunctionInfo(PipelineTemplateConstants.HashFiles, 1, Byte.MaxValue), + }; } } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index a36f5b7e3..d337d599d 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -159,6 +159,44 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating return result; } + // TODO: Add function here that says Evaluate Steps but it will return whatever ConvertToSteps returns. + // return: List + // used to be evaluatesteps + public List LoadSteps( + TemplateToken token, + DictionaryContextData contextData, + IList expressionFunctions + ) + { + // Similar to EvaluateStepDisplayName() except that we pass in PipelineTemplateConstants.Steps as the type for evaluation + var result = default(List); + + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(contextData, expressionFunctions, setMissingContext: false); + // This will keep it from preexpanding + // TODO: we might want to to have a bool to prevent it from filling in with missing context w/ dummy variables + // context.ExpressionFunctions.Clear(); + // context.ExpressionValues.Clear(); + + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Steps, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = PipelineTemplateConverter.ConvertToSteps(context, token); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + + return result; + } + + public Dictionary EvaluateStepEnvironment( TemplateToken token, DictionaryContextData contextData, @@ -400,7 +438,8 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating private TemplateContext CreateContext( DictionaryContextData contextData, IList expressionFunctions, - IEnumerable> expressionState = null) + IEnumerable> expressionState = null, + bool setMissingContext = true) { var result = new TemplateContext { @@ -449,18 +488,21 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating // - Evaluating early when all referenced contexts are available, even though all allowed // contexts may not yet be available. For example, evaluating step display name can often // be performed early. - foreach (var name in s_expressionValueNames) + if (setMissingContext) { - if (!result.ExpressionValues.ContainsKey(name)) + foreach (var name in s_expressionValueNames) { - result.ExpressionValues[name] = null; + if (!result.ExpressionValues.ContainsKey(name)) + { + result.ExpressionValues[name] = null; + } } - } - foreach (var name in s_expressionFunctionNames) - { - if (!functionNames.Contains(name)) + foreach (var name in s_expressionFunctionNames) { - result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + if (!functionNames.Contains(name)) + { + result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + } } } diff --git a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs index 2d599dd9c..2e03671fb 100644 --- a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs @@ -94,5 +94,12 @@ namespace GitHub.DistributedTask.Pipelines public static readonly String Resources = "resources"; public static readonly String All = "all"; } + + public static class ScriptStepInputs + { + public static readonly String Script = "script"; + public static readonly String WorkingDirectory = "workingDirectory"; + public static readonly String Shell = "shell"; + } } }