Download action repos into directory

Co-authored-by: Christopher Johnson <thchrisjohnson@github.com>

releaseVersion

@@ -1 +1 @@
-2.272.0
+<Update to ./src/runnerversion when creating release>

src/Runner.Listener/Runner.cs

@@ -462,13 +462,14 @@ Options:
  --commit   Prints the runner commit
 
 Config Options:
- --unattended     Disable interactive prompts for missing arguments. Defaults will be used for missing options
- --url string     Repository to add the runner to. Required if unattended
- --token string   Registration token. Required if unattended
- --name string    Name of the runner to configure (default {Environment.MachineName ?? "myrunner"})
- --labels string  Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}'
- --work string    Relative runner work directory (default {Constants.Path.WorkDirectory})
- --replace        Replace any existing runner with the same name (default false)");
+ --unattended           Disable interactive prompts for missing arguments. Defaults will be used for missing options
+ --url string           Repository to add the runner to. Required if unattended
+ --token string         Registration token. Required if unattended
+ --name string          Name of the runner to configure (default {Environment.MachineName ?? "myrunner"})
+ --runnergroup string   Name of the runner group to add this runner to (defaults to the default runner group)
+ --labels string        Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}'
+ --work string          Relative runner work directory (default {Constants.Path.WorkDirectory})
+ --replace              Replace any existing runner with the same name (default false)");
 #if OS_WINDOWS
     _term.WriteLine($@" --runasservice   Run the runner as a service");
     _term.WriteLine($@" --windowslogonaccount string   Account to run the service as. Requires runasservice");

src/Runner.Worker/ActionManifestManager.cs

@@ -30,6 +30,8 @@ namespace GitHub.Runner.Worker
         Dictionary<string, string> EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary<string, PipelineContextData> extraExpressionValues);
 
         string EvaluateDefaultInput(IExecutionContext executionContext, string inputName, TemplateToken token);
+
+        // string EvaluateDefaultInputInsideComposite(IExecutionContext executionContext, string inputName, TemplateToken token);
     }
 
     public sealed class ActionManifestManager : RunnerService, IActionManifestManager
@@ -282,6 +284,9 @@ namespace GitHub.Runner.Worker
             if (token != null)
             {
                 var templateContext = CreateTemplateContext(executionContext);
+                Trace.Info($"Template context GitHub: {StringUtil.ConvertToJson(templateContext.ExpressionValues["github"])}");
+
+                Trace.Info($"Template context keys: {StringUtil.ConvertToJson(templateContext.ExpressionValues.Keys)}");
                 try
                 {
                     var evaluateResult = TemplateEvaluator.Evaluate(templateContext, "input-default-context", token, 0, null, omitHeader: true);
@@ -304,6 +309,83 @@ namespace GitHub.Runner.Worker
             return result;
         }
 
+        // public string EvaluateDefaultInputInsideComposite(
+        //     IExecutionContext executionContext,
+        //     string inputName,
+        //     TemplateToken token)
+        // {
+        //     string result = "";
+        //     if (token != null)
+        //     {
+        //         // Add GitHub Expression Values
+        //         var githubContext = executionContext.ExpressionValues["github"];
+        //         Trace.Info($"GitHub Context EvaluateDefaultInputInsideComposite: {StringUtil.ConvertToJson(githubContext)}");
+
+        //         var temp = new Dictionary<string, PipelineContextData>();
+        //         temp["github"] = githubContext;
+
+        //         // Trace.Info($"Token: {StringUtil.ConvertToJson(token)}");
+
+        //         var templateContext = CreateTemplateContext(executionContext, temp);
+        //         try
+        //         {
+
+        //             var evaluateResult = TemplateEvaluator.Evaluate(templateContext, "uses-input", token, 0, null, omitHeader: true);
+
+        //             Trace.Info($"Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
+        //             templateContext.Errors.Check();
+
+        //             // Can't evaluate the default github.respository, etc.
+
+
+        //             // TODO: restrict to only be used for composite "uses" steps
+        //             // Find better way to isolate only 
+        //             // We could create a whitelist for just checkout?
+        //             // (ex: "repo", "token", etc.)
+        //             // if (evaluateResult is BasicExpressionToken)
+        //             // {
+                        
+
+        //             //     // var evaluateResult2 = TemplateEvaluator.Evaluate(templateContext, "step-with", token, 0, null, omitHeader: true);
+        //             //     // templateContext.Errors.Check();
+
+        //             //     // Trace.Info($"Test2 Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult2)}");
+        //             //     // var result2 = evaluateResult2.AssertString($"default value for input '{inputName}'").Value;
+
+        //             //     // TODO 6/28 => Try just getting it from the getgithubcontext lmao.
+
+        //             //     // Trace.Info($"Basic expr token: {evaluateResult}");
+
+        //             //     var stringVersion = evaluateResult.AssertString($"default value for input '{inputName}'").Value;
+
+        //             //     // no we have to use the template evaluator since it's a
+
+        //             //     // var githubTokenSplit = 
+
+        //             //     // // Evaluate it
+        //             //     // var evaluateResult = executionContext.GetGitHubContext("");
+
+        //             //     return result2;
+        //             // }
+
+        //             // Trace.Info($"Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
+
+        //             // String
+        //             result = evaluateResult.AssertString($"default value for input '{inputName}'").Value;
+        //         }
+        //         catch (Exception ex) when (!(ex is TemplateValidationException))
+        //         {
+        //             Trace.Error(ex);
+        //             templateContext.Errors.Add(ex);
+        //         }
+
+        //         templateContext.Errors.Check();
+        //     }
+
+        //     return result;
+
+        // }
+
         private TemplateContext CreateTemplateContext(
             IExecutionContext executionContext,
             IDictionary<string, PipelineContextData> extraExpressionValues = null)

src/Runner.Worker/ActionRunner.cs

@@ -169,6 +169,25 @@ namespace GitHub.Runner.Worker
                 validInputs.Add("entryPoint");
                 validInputs.Add("args");
             }
+
+            Trace.Info($"Repo: {ExecutionContext.GetGitHubContext("repository")}");
+
+            // Since we don't pass the GitHub Context attributes to the composite action,
+            // We need to explitly set the default values of certain things we need like the 
+            // default repository 
+
+            // if (ExecutionContext.GetGitHubContext("repository") == null)
+            // {
+            //     ExecutionContext.SetGitHubContext("repository", definition.Directory);
+            // }
+
+            // var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
+
+            // foreach (var pair in githubContext)
+            // {
+            //     ExecutionContext.SetGitHubContext(pair.Key, pair.Value as StringContextData);
+            // }
+            
             // Merge the default inputs from the definition
             if (definition.Data?.Inputs != null)
             {
@@ -179,7 +198,17 @@ namespace GitHub.Runner.Worker
                     validInputs.Add(key);
                     if (!inputs.ContainsKey(key))
                     {
+                        Trace.Info($"Definition Input Key: {key}");
+                        // if (ExecutionContext.InsideComposite)
+                        // {
+                        //     inputs[key] = manifestManager.EvaluateDefaultInputInsideComposite(ExecutionContext, key, input.Value);
+                        // }
+                        // else
+                        // {
+                        //     inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value);
+                        // }
                         inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value);
+                        Trace.Info($"Definition Input Value: {inputs[key]}");
                     }
                 }
             }

src/Runner.Worker/ExecutionContext.cs

@@ -53,7 +53,7 @@ namespace GitHub.Runner.Worker
         JobContext JobContext { get; }
 
         // Only job level ExecutionContext has JobSteps
-        List<IStep> JobSteps { get; }
+        Queue<IStep> JobSteps { get; }
 
         // Only job level ExecutionContext has PostJobSteps
         Stack<IStep> PostJobSteps { get; }
@@ -144,7 +144,7 @@ namespace GitHub.Runner.Worker
         public GlobalContext Global { get; private set; }
 
         // Only job level ExecutionContext has JobSteps
-        public List<IStep> JobSteps { get; private set; }
+        public Queue<IStep> JobSteps { get; private set; }
 
         // Only job level ExecutionContext has PostJobSteps
         public Stack<IStep> PostJobSteps { get; private set; }
@@ -663,7 +663,7 @@ namespace GitHub.Runner.Worker
             Global.PrependPath = new List<string>();
 
             // JobSteps for job ExecutionContext
-            JobSteps = new List<IStep>();
+            JobSteps = new Queue<IStep>();
 
             // PostJobSteps for job ExecutionContext
             PostJobSteps = new Stack<IStep>();

src/Runner.Worker/Handlers/CompositeActionHandler.cs

@@ -5,11 +5,14 @@ using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Expressions2;
 using GitHub.DistributedTask.ObjectTemplating.Tokens;
 using GitHub.DistributedTask.Pipelines.ContextData;
+using GitHub.DistributedTask.Pipelines.ObjectTemplating;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Sdk;
+using GitHub.Runner.Worker.Expressions;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 
@@ -35,6 +38,8 @@ namespace GitHub.Runner.Worker.Handlers
             var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
             ArgUtil.NotNull(githubContext, nameof(githubContext));
 
+            // Trace.Info($"Github Context: {StringUtil.ConvertToJson(githubContext)}");
+
             // Resolve action steps
             var actionSteps = Data.Steps;
 
@@ -45,6 +50,8 @@ namespace GitHub.Runner.Worker.Handlers
                 inputsData[i.Key] = new StringContextData(i.Value);
             }
 
+            Trace.Info($"Composite Actions Inputs {StringUtil.ConvertToJson(inputsData)}");
+
             // Initialize Composite Steps List of Steps
             var compositeSteps = new List<IStep>();
 
@@ -64,6 +71,10 @@ namespace GitHub.Runner.Worker.Handlers
                 compositeGitHubContext[pair.Key] = pair.Value;
             }
 
+            // Download all data
+            var actionManager = HostContext.GetService<IActionManager>();
+            await actionManager.PrepareActionsAsync(ExecutionContext, actionSteps);
+
             foreach (Pipelines.ActionStep actionStep in actionSteps)
             {
                 var actionRunner = HostContext.CreateService<IActionRunner>();
@@ -76,6 +87,7 @@ namespace GitHub.Runner.Worker.Handlers
                 // Set GITHUB_ACTION_PATH
                 step.ExecutionContext.ExpressionValues["github"] = compositeGitHubContext;
                 step.ExecutionContext.SetGitHubContext("action_path", ActionDirectory);
+                step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo<HashFilesFunction>(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue));
 
                 compositeSteps.Add(step);
             }
@@ -150,6 +162,7 @@ namespace GitHub.Runner.Worker.Handlers
         {
             ArgUtil.NotNull(compositeSteps, nameof(compositeSteps));
 
+
             // The parent StepsRunner of the whole Composite Action Step handles the cancellation stuff already. 
             foreach (IStep step in compositeSteps)
             {

src/Runner.Worker/JobRunner.cs

@@ -152,7 +152,7 @@ namespace GitHub.Runner.Worker
                 {
                     foreach (var step in jobSteps)
                     {
-                        jobContext.JobSteps.Add(step);
+                        jobContext.JobSteps.Enqueue(step);
                     }
 
                     await stepsRunner.RunAsync(jobContext);

src/Runner.Worker/StepsRunner.cs

@@ -59,14 +59,13 @@ namespace GitHub.Runner.Worker
                     checkPostJobActions = true;
                     while (jobContext.PostJobSteps.TryPop(out var postStep))
                     {
-                        jobContext.JobSteps.Add(postStep);
+                        jobContext.JobSteps.Enqueue(postStep);
                     }
 
                     continue;
                 }
 
-                var step = jobContext.JobSteps[0];
-                jobContext.JobSteps.RemoveAt(0);
+                var step = jobContext.JobSteps.Dequeue();
 
                 Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
                 ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));

src/Runner.Worker/action_yaml.json

@@ -109,9 +109,15 @@
         },
         "composite-steps": {
             "sequence": {
-                "item-type": "composite-step"
+                "item-type": "composite-step-types"
             }
         },
+        "composite-step-types": {
+            "one-of": [
+                "composite-step",
+                "uses-step"
+            ]
+        },
         "composite-step": {
             "mapping": {
                 "properties": {
@@ -130,6 +136,20 @@
                 }
             }
         },
+        "uses-step": {
+            "mapping": {
+                "properties": {
+                    "name": "string-steps-context",
+                    "id": "non-empty-string",
+                    "uses": {
+                        "type": "non-empty-string",
+                        "required": true
+                    },
+                    "with": "step-with",
+                    "env": "step-env"
+                }
+            }
+        },
         "container-runs-context": {
             "context": [
                 "inputs"
@@ -195,6 +215,37 @@
                 "loose-key-type": "non-empty-string",
                 "loose-value-type": "string"
             }
+        },
+        "step-with": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "mapping": {
+                "loose-key-type": "non-empty-string",
+                "loose-value-type": "string"
+            }
+        },
+        "uses-input": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "string": {}
         }
     }
-}
+}

src/Test/L0/Worker/StepsRunnerL0.cs

@@ -82,7 +82,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -117,7 +117,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -156,7 +156,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -210,7 +210,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -289,7 +289,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -332,7 +332,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Step.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Step.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -363,7 +363,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -393,7 +393,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -419,7 +419,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -457,7 +457,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -495,7 +495,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -526,7 +526,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -562,7 +562,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);

src/runnerversion

@@ -1 +1 @@
-2.272.0
+2.274.0