diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 347eb3d0d..5b2fa3fda 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -395,6 +395,10 @@ namespace GitHub.Runner.Worker Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}."); } } + else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + // Don't do anything for now + } else { throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString()); @@ -1101,6 +1105,11 @@ namespace GitHub.Runner.Worker Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation."); return null; } + else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation."); + return null; + } else { throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString()); @@ -1211,6 +1220,7 @@ namespace GitHub.Runner.Worker NodeJS, Plugin, Script, + Composite } public sealed class ContainerActionExecutionData : ActionExecutionData @@ -1267,6 +1277,14 @@ namespace GitHub.Runner.Worker public override bool HasPost => false; } + public sealed class CompositeActionExecutionData : ActionExecutionData + { + public override ActionExecutionType ExecutionType => ActionExecutionType.Composite; + public override bool HasPre => false; + public override bool HasPost => false; + public List Steps { get; set; } + } + public abstract class ActionExecutionData { private string _initCondition = $"{Constants.Expressions.Always}()"; diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 4e9149d26..b210d7938 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -14,6 +14,7 @@ using YamlDotNet.Core; using YamlDotNet.Core.Events; using System.Globalization; using System.Linq; +using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -92,7 +93,7 @@ namespace GitHub.Runner.Worker break; case "runs": - actionDefinition.Execution = ConvertRuns(context, actionPair.Value); + actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value); break; default: Trace.Info($"Ignore action property {propertyName}."); @@ -284,7 +285,7 @@ namespace GitHub.Runner.Worker // Add the file table if (_fileTable?.Count > 0) { - for (var i = 0 ; i < _fileTable.Count ; i++) + for (var i = 0; i < _fileTable.Count; i++) { result.GetFileId(_fileTable[i]); } @@ -294,6 +295,7 @@ namespace GitHub.Runner.Worker } private ActionExecutionData ConvertRuns( + IExecutionContext executionContext, TemplateContext context, TemplateToken inputsToken) { @@ -311,6 +313,8 @@ namespace GitHub.Runner.Worker var postToken = default(StringToken); var postEntrypointToken = default(StringToken); var postIfToken = default(StringToken); + var stepsLoaded = default(List); + foreach (var run in runsMapping) { var runsKey = run.Key.AssertString("runs key").Value; @@ -355,6 +359,15 @@ namespace GitHub.Runner.Worker case "pre-if": preIfToken = run.Value.AssertString("pre-if"); break; + case "steps": + if (!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + var steps = run.Value.AssertSequence("steps"); + var evaluator = executionContext.ToPipelineTemplateEvaluator(); + stepsLoaded = evaluator.LoadCompositeSteps(steps); + break; + } + throw new Exception("You aren't supposed to be using Composite Actions yet!"); default: Trace.Info($"Ignore run property {runsKey}."); break; @@ -402,6 +415,20 @@ namespace GitHub.Runner.Worker }; } } + else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + if (stepsLoaded == null) + { + throw new ArgumentNullException($"No steps provided."); + } + else + { + return new CompositeActionExecutionData() + { + Steps = stepsLoaded, + }; + } + } else { throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead."); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 0318974b0..a76ca3cd0 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -105,6 +105,8 @@ namespace GitHub.Runner.Worker // others void ForceTaskComplete(); void RegisterPostJobStep(IStep step); + IStep RegisterCompositeStep(IStep step, DictionaryContextData inputsData); + void EnqueueAllCompositeSteps(Queue steps); } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -169,7 +171,6 @@ namespace GitHub.Runner.Worker public bool EchoOnActionCommand { get; set; } - public TaskResult? Result { get @@ -266,6 +267,68 @@ namespace GitHub.Runner.Worker Root.PostJobSteps.Push(step); } + /* + RegisterCompositeStep is a helper function used in CompositeActionHandler::RunAsync to + add a child node, aka a step, to the current job to the front of the queue for processing. + */ + public IStep RegisterCompositeStep(IStep step, DictionaryContextData inputsData) + { + // ~Brute Force Method~ + // There is no way to put this current job in front of the queue in < O(n) time where n = # of elements in JobSteps + // Everytime we add a new step, you could requeue every item to put those steps from that stack in JobSteps which + // would result in O(n) for each time we add a composite action step where n = number of jobSteps which would compound + // to O(n*m) where m = number of composite steps + // var temp = Root.JobSteps.ToArray(); + // Root.JobSteps.Clear(); + // Root.JobSteps.Enqueue(step); + // foreach(var s in temp) + // Root.JobSteps.Enqueue(s); + + // ~Optimized Method~ + // Alterative solution: We add to another temp Queue + // After we add all the transformed composite steps to this temp queue, we requeue the whole JobSteps accordingly in EnqueueAllCompositeSteps() + // where the queued composite steps are at the front of the JobSteps Queue and the rest of the jobs maintain its order and are + // placed after the queued composite steps + // This will take only O(n+m) time which would be just as efficient if we refactored the code of JobSteps to a PriorityQueue + // This temp Queue is created in the CompositeActionHandler. + + Trace.Info("Adding Composite Action Step"); + var newGuid = Guid.NewGuid(); + step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null); + step.ExecutionContext.ExpressionValues["inputs"] = inputsData; + return step; + } + + // Add Composite Steps first and then requeue the rest of the job steps. + public void EnqueueAllCompositeSteps(Queue steps) + { + // TODO: For UI purposes, look at figuring out how to condense steps in one node + // maybe use "this" instead of "Root"? + if (Root.JobSteps != null) + { + var temp = Root.JobSteps.ToArray(); + Root.JobSteps.Clear(); + foreach (var cs in steps) + { + Trace.Info($"EnqueueAllCompositeSteps : Adding Composite action step {cs}"); + Root.JobSteps.Enqueue(cs); + } + foreach (var s in temp) + { + Root.JobSteps.Enqueue(s); + } + } + else + { + Root.JobSteps = new Queue(); + foreach (var cs in steps) + { + Trace.Info($"EnqueueAllCompositeSteps : Adding Composite action step {cs}"); + Root.JobSteps.Enqueue(cs); + } + } + } + public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null) { Trace.Entering(); diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs new file mode 100644 index 000000000..34f12270b --- /dev/null +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -0,0 +1,105 @@ +using System.IO; +using System.Text; +using System.Threading.Tasks; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; +using GitHub.DistributedTask.WebApi; +using Pipelines = GitHub.DistributedTask.Pipelines; +using System; +using System.Linq; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using System.Collections.Generic; +using GitHub.DistributedTask.Pipelines.ContextData; + +namespace GitHub.Runner.Worker.Handlers +{ + [ServiceLocator(Default = typeof(CompositeActionHandler))] + public interface ICompositeActionHandler : IHandler + { + CompositeActionExecutionData Data { get; set; } + } + public sealed class CompositeActionHandler : Handler, ICompositeActionHandler + { + public CompositeActionExecutionData Data { get; set; } + + public Task RunAsync(ActionRunStage stage) + { + // Validate args. + Trace.Entering(); + ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); + ArgUtil.NotNull(Inputs, nameof(Inputs)); + + var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(githubContext, nameof(githubContext)); + + var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp); + + // Resolve action steps + var actionSteps = Data.Steps; + if (actionSteps == null) + { + Trace.Error("Data.Steps in CompositeActionHandler is null"); + } + else + { + Trace.Info($"Data Steps Value for Composite Actions is: {actionSteps}."); + } + + // Create Context Data to reuse for each composite action step + var inputsData = new DictionaryContextData(); + foreach (var i in Inputs) + { + inputsData[i.Key] = new StringContextData(i.Value); + } + + // Add each composite action step to the front of the queue + var compositeActionSteps = new Queue(); + foreach (Pipelines.ActionStep aStep in actionSteps) + { + // Ex: + // runs: + // using: "composite" + // steps: + // - uses: example/test-composite@v2 (a) + // - run echo hello world (b) + // - run echo hello world 2 (c) + // + // ethanchewy/test-composite/action.yaml + // runs: + // using: "composite" + // steps: + // - run echo hello world 3 (d) + // - run echo hello world 4 (e) + // + // Stack (LIFO) [Bottom => Middle => Top]: + // | a | + // | a | => | d | + // (Run step d) + // | a | + // | a | => | e | + // (Run step e) + // | a | + // (Run step a) + // | b | + // (Run step b) + // | c | + // (Run step c) + // Done. + + var actionRunner = HostContext.CreateService(); + actionRunner.Action = aStep; + actionRunner.Stage = stage; + actionRunner.Condition = aStep.Condition; + actionRunner.DisplayName = aStep.DisplayName; + // TODO: Do we need to add any context data from the job message? + // (See JobExtension.cs ~line 236) + + compositeActionSteps.Enqueue(ExecutionContext.RegisterCompositeStep(actionRunner, inputsData)); + } + ExecutionContext.EnqueueAllCompositeSteps(compositeActionSteps); + + return Task.CompletedTask; + } + + } +} diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index 0f2413ef5..99311afca 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -66,6 +66,14 @@ namespace GitHub.Runner.Worker.Handlers handler = HostContext.CreateService(); (handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData; } + else if (data.ExecutionType == ActionExecutionType.Composite) + { + // TODO + // Runner plugin + handler = HostContext.CreateService(); + // handler = CompositeHandler; + (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData; + } else { // This should never happen. diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 485a4cdf9..6a0e2b694 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -67,6 +67,11 @@ namespace GitHub.Runner.Worker var step = jobContext.JobSteps.Dequeue(); var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null; + // TODO: Fix this temporary workaround for Composite Actions + if (!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + nextStep = null; + } Trace.Info($"Processing step: DisplayName='{step.DisplayName}'"); ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext)); @@ -409,7 +414,11 @@ namespace GitHub.Runner.Worker scope = scopesToInitialize.Pop(); executionContext.Debug($"Initializing scope '{scope.Name}'"); executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName); - executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + } var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); var inputs = default(DictionaryContextData); try @@ -432,7 +441,11 @@ namespace GitHub.Runner.Worker // Setup expression values var scopeName = executionContext.ScopeName; executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName); - executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + } return true; } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 7a8b847d3..da8fb79a9 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -32,7 +32,8 @@ "one-of": [ "container-runs", "node12-runs", - "plugin-runs" + "plugin-runs", + "composite-runs" ] }, "container-runs": { @@ -83,6 +84,32 @@ } } }, + "composite-runs": { + "mapping": { + "properties": { + "using": "non-empty-string", + "steps": "composite-steps" + } + } + }, + "composite-steps": { + "context": [ + "github", + "needs", + "strategy", + "matrix", + "secrets", + "steps", + "inputs", + "job", + "runner", + "env", + "hashFiles(1,255)" + ], + "sequence": { + "item-type": "any" + } + }, "container-runs-context": { "context": [ "inputs" diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index d1c886dd8..bf7d8d2e9 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -65,6 +65,8 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating public const String StepEnv = "step-env"; public const String StepIfResult = "step-if-result"; public const String Steps = "steps"; + + public const String StepsInTemplate = "steps-in-template"; public const String StepsScopeInputs = "steps-scope-inputs"; public const String StepsScopeOutputs = "steps-scope-outputs"; public const String StepsTemplateRoot = "steps-template-root"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 43be43d33..a952f58fb 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -29,7 +29,6 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult); return evaluationResult.IsTruthy; } - internal static Boolean? ConvertToStepContinueOnError( TemplateContext context, TemplateToken token, @@ -264,5 +263,351 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating return result; } + + //Note: originally was List but we need to change to List to use the "Inputs" attribute + internal static List ConvertToSteps( + TemplateContext context, + TemplateToken steps) + { + var stepsSequence = steps.AssertSequence($"job {PipelineTemplateConstants.Steps}"); + + var result = new List(); + foreach (var stepsItem in stepsSequence) + { + var step = ConvertToStep(context, stepsItem); + if (step != null) // step = null means we are hitting error during step conversion, there should be an error in context.errors + { + if (step.Enabled) + { + result.Add(step); + } + } + } + + return result; + } + + private static ActionStep ConvertToStep( + TemplateContext context, + TemplateToken stepsItem) + { + var step = stepsItem.AssertMapping($"{PipelineTemplateConstants.Steps} item"); + var continueOnError = default(ScalarToken); + var env = default(TemplateToken); + var id = default(StringToken); + var ifCondition = default(String); + var ifToken = default(ScalarToken); + var name = default(ScalarToken); + var run = default(ScalarToken); + var scope = default(StringToken); + var timeoutMinutes = default(ScalarToken); + var uses = default(StringToken); + var with = default(TemplateToken); + var workingDir = default(ScalarToken); + var path = default(ScalarToken); + var clean = default(ScalarToken); + var fetchDepth = default(ScalarToken); + var lfs = default(ScalarToken); + var submodules = default(ScalarToken); + var shell = default(ScalarToken); + + foreach (var stepProperty in step) + { + var propertyName = stepProperty.Key.AssertString($"{PipelineTemplateConstants.Steps} item key"); + + switch (propertyName.Value) + { + case PipelineTemplateConstants.Clean: + clean = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Clean}"); + break; + + case PipelineTemplateConstants.ContinueOnError: + ConvertToStepContinueOnError(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + continueOnError = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} {PipelineTemplateConstants.ContinueOnError}"); + break; + + case PipelineTemplateConstants.Env: + ConvertToStepEnvironment(context, stepProperty.Value, StringComparer.Ordinal, allowExpressions: true); // Validate early if possible + env = stepProperty.Value; + break; + + case PipelineTemplateConstants.FetchDepth: + fetchDepth = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.FetchDepth}"); + break; + + case PipelineTemplateConstants.Id: + id = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Id}"); + if (!NameValidation.IsValid(id.Value, true)) + { + context.Error(id, $"Step id {id.Value} is invalid. Ids must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'"); + } + break; + + case PipelineTemplateConstants.If: + ifToken = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.If}"); + break; + + case PipelineTemplateConstants.Lfs: + lfs = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Lfs}"); + break; + + case PipelineTemplateConstants.Name: + name = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Name}"); + break; + + case PipelineTemplateConstants.Path: + path = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Path}"); + break; + + case PipelineTemplateConstants.Run: + run = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Run}"); + break; + + case PipelineTemplateConstants.Shell: + shell = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Shell}"); + break; + + case PipelineTemplateConstants.Scope: + scope = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Scope}"); + break; + + case PipelineTemplateConstants.Submodules: + submodules = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Submodules}"); + break; + + case PipelineTemplateConstants.TimeoutMinutes: + ConvertToStepTimeout(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + timeoutMinutes = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.TimeoutMinutes}"); + break; + + case PipelineTemplateConstants.Uses: + uses = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + break; + + case PipelineTemplateConstants.With: + ConvertToStepInputs(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + with = stepProperty.Value; + break; + + case PipelineTemplateConstants.WorkingDirectory: + workingDir = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.WorkingDirectory}"); + break; + + default: + propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Steps} item key"); // throws + break; + } + } + + // Fixup the if-condition + var isDefaultScope = String.IsNullOrEmpty(scope?.Value); + ifCondition = ConvertToIfCondition(context, ifToken, false, isDefaultScope); + + if (run != null) + { + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Environment = env, + Reference = new ScriptReference(), + }; + + var inputs = new MappingToken(null, null, null); + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Script), run); + + if (workingDir != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.WorkingDirectory), workingDir); + } + + if (shell != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Shell), shell); + } + + result.Inputs = inputs; + + return result; + } + else + { + uses.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Inputs = with, + Environment = env, + }; + + if (uses.Value.StartsWith("docker://", StringComparison.Ordinal)) + { + var image = uses.Value.Substring("docker://".Length); + result.Reference = new ContainerRegistryReference { Image = image }; + } + else if (uses.Value.StartsWith("./") || uses.Value.StartsWith(".\\")) + { + result.Reference = new RepositoryPathReference + { + RepositoryType = PipelineConstants.SelfAlias, + Path = uses.Value + }; + } + else + { + var usesSegments = uses.Value.Split('@'); + var pathSegments = usesSegments[0].Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + var gitRef = usesSegments.Length == 2 ? usesSegments[1] : String.Empty; + + if (usesSegments.Length != 2 || + pathSegments.Length < 2 || + String.IsNullOrEmpty(pathSegments[0]) || + String.IsNullOrEmpty(pathSegments[1]) || + String.IsNullOrEmpty(gitRef)) + { + // todo: loc + context.Error(uses, $"Expected format {{org}}/{{repo}}[/path]@ref. Actual '{uses.Value}'"); + } + else + { + var repositoryName = $"{pathSegments[0]}/{pathSegments[1]}"; + var directoryPath = pathSegments.Length > 2 ? String.Join("/", pathSegments.Skip(2)) : String.Empty; + + result.Reference = new RepositoryPathReference + { + RepositoryType = RepositoryTypes.GitHub, + Name = repositoryName, + Ref = gitRef, + Path = directoryPath, + }; + } + } + + return result; + } + } + + /// + /// When empty, default to "success()". + /// When a status function is not referenced, format as "success() && <CONDITION>". + /// + private static String ConvertToIfCondition( + TemplateContext context, + TemplateToken token, + Boolean isJob, + Boolean isDefaultScope) + { + String condition; + if (token is null) + { + condition = null; + } + else if (token is BasicExpressionToken expressionToken) + { + condition = expressionToken.Expression; + } + else + { + var stringToken = token.AssertString($"{(isJob ? "job" : "step")} {PipelineTemplateConstants.If}"); + condition = stringToken.Value; + } + + if (String.IsNullOrWhiteSpace(condition)) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var expressionParser = new ExpressionParser(); + var functions = default(IFunctionInfo[]); + var namedValues = default(INamedValueInfo[]); + if (isJob) + { + namedValues = s_jobIfNamedValues; + // TODO: refactor into seperate functions + // functions = PhaseCondition.FunctionInfo; + } + else + { + namedValues = isDefaultScope ? s_stepNamedValues : s_stepInTemplateNamedValues; + functions = s_stepConditionFunctions; + } + + var node = default(ExpressionNode); + try + { + node = expressionParser.CreateTree(condition, null, namedValues, functions) as ExpressionNode; + } + catch (Exception ex) + { + context.Error(token, ex); + return null; + } + + if (node == null) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var hasStatusFunction = node.Traverse().Any(x => + { + if (x is Function function) + { + return String.Equals(function.Name, PipelineTemplateConstants.Always, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Cancelled, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Failure, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Success, StringComparison.OrdinalIgnoreCase); + } + + return false; + }); + + return hasStatusFunction ? condition : $"{PipelineTemplateConstants.Success}() && ({condition})"; + } + + private static readonly INamedValueInfo[] s_jobIfNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepInTemplateNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.Inputs), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly IFunctionInfo[] s_stepConditionFunctions = new IFunctionInfo[] + { + new FunctionInfo(PipelineTemplateConstants.Always, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Cancelled, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Failure, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Success, 0, 0), + new FunctionInfo(PipelineTemplateConstants.HashFiles, 1, Byte.MaxValue), + }; } } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index a36f5b7e3..55076e670 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -159,6 +159,32 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating return result; } + public List LoadCompositeSteps( + TemplateToken token + ) + { + var result = default(List); + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(null, null, setMissingContext: false); + // TODO: we might want to to have a bool to prevent it from filling in with missing context w/ dummy variables + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsInTemplate, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = PipelineTemplateConverter.ConvertToSteps(context, token); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + return result; + } + + public Dictionary EvaluateStepEnvironment( TemplateToken token, DictionaryContextData contextData, @@ -400,7 +426,8 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating private TemplateContext CreateContext( DictionaryContextData contextData, IList expressionFunctions, - IEnumerable> expressionState = null) + IEnumerable> expressionState = null, + bool setMissingContext = true) { var result = new TemplateContext { @@ -449,18 +476,21 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating // - Evaluating early when all referenced contexts are available, even though all allowed // contexts may not yet be available. For example, evaluating step display name can often // be performed early. - foreach (var name in s_expressionValueNames) + if (setMissingContext) { - if (!result.ExpressionValues.ContainsKey(name)) + foreach (var name in s_expressionValueNames) { - result.ExpressionValues[name] = null; + if (!result.ExpressionValues.ContainsKey(name)) + { + result.ExpressionValues[name] = null; + } } - } - foreach (var name in s_expressionFunctionNames) - { - if (!functionNames.Contains(name)) + foreach (var name in s_expressionFunctionNames) { - result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + if (!functionNames.Contains(name)) + { + result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + } } } diff --git a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs index 2d599dd9c..2e03671fb 100644 --- a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs @@ -94,5 +94,12 @@ namespace GitHub.DistributedTask.Pipelines public static readonly String Resources = "resources"; public static readonly String All = "all"; } + + public static class ScriptStepInputs + { + public static readonly String Script = "script"; + public static readonly String WorkingDirectory = "workingDirectory"; + public static readonly String Shell = "shell"; + } } }