Fix GITHUB_ACTION not being set

Co-authored-by: Christopher Johnson <thchrisjohnson@github.com>

.github/workflows/release.yml

@@ -1,6 +1,7 @@
 name: Runner CD
 
 on:
+  workflow_dispatch:
   push:
     paths:
     - releaseVersion

releaseNote.md

@@ -1,18 +1,15 @@
 ## Features
-  - Resolve action download info from server (#508, #515, #550)
-  - Print runner and machine name to log. (#539)
+  - Composite Actions Support for Multiple Run Steps (#549, #557, #564, #568, #569, #578, #591, #599, #605, #609, #610, #615, #624) 
+  - Prepare to switch GITHUB_ACTION to use ContextName instead of refname (#593)
+  - Fold logs for intermediate docker commands (#608)
+  - Add ability to register a runner to the non-default self-hosted runner group (#613)
+  
 ## Bugs
-  - Reduce input validation warnings (#506) 
-  - Fix null ref exception in SecretMasker caused by `hashfiles` timeout. (#516)
-  - Add libicu66 to `./installDependencies.sh` for Ubuntu 20.04 (#535)
-  - Fix DataContract with Token service (#532)
-  - Skip search $PATH on command with fully qualified path (#526)
-  - Restore SELinux context on service file when SELinux is enabled (#525)
+  - Double quotes around variable so CD works if path contains spaces (#602)
+  - Bump lodash in /src/Misc/expressionFunc/hashFiles (#603) 
+  - Fix poor performance of process spawned from svc daemon (#614)
 ## Misc
-  - Remove SPS/Token migration code. Remove GHES url manipulate code. (#513) 
-  - Add sub-step for developer flow for clarity (#523)
-  - Update Links and Language to Git + VSCode (#522)
-  - Update runner configuration exception message (#540)
+  - Move shared ExecutionContext properties under .Global (#594)
 
 ## Windows x64
 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.

src/Misc/layoutbin/actions.runner.plist.template

@@ -23,5 +23,7 @@
       <key>ACTIONS_RUNNER_SVC</key>
       <string>1</string>
     </dict>
+    <key>ProcessType</key>
+    <string>Interactive</string>
   </dict>
 </plist>

src/Runner.Common/Constants.cs

@@ -90,7 +90,7 @@ namespace GitHub.Runner.Common
                     public static readonly string Labels = "labels";
                     public static readonly string MonitorSocketAddress = "monitorsocketaddress";
                     public static readonly string Name = "name";
-                    public static readonly string Pool = "pool";
+                    public static readonly string RunnerGroup = "runnergroup";
                     public static readonly string StartupType = "startuptype";
                     public static readonly string Url = "url";
                     public static readonly string UserName = "username";

src/Runner.Listener/CommandSettings.cs

@@ -42,7 +42,7 @@ namespace GitHub.Runner.Listener
             Constants.Runner.CommandLine.Args.Labels,
             Constants.Runner.CommandLine.Args.MonitorSocketAddress,
             Constants.Runner.CommandLine.Args.Name,
-            Constants.Runner.CommandLine.Args.Pool,
+            Constants.Runner.CommandLine.Args.RunnerGroup,
             Constants.Runner.CommandLine.Args.StartupType,
             Constants.Runner.CommandLine.Args.Token,
             Constants.Runner.CommandLine.Args.Url,
@@ -169,6 +169,15 @@ namespace GitHub.Runner.Listener
                 validator: Validators.NonEmptyValidator);
         }
 
+        public string GetRunnerGroupName(string defaultPoolName = null)
+        {
+            return GetArgOrPrompt(
+                name: Constants.Runner.CommandLine.Args.RunnerGroup,
+                description: "Enter the name of the runner group to add this runner to:",
+                defaultValue: defaultPoolName ?? "default",
+                validator: Validators.NonEmptyValidator);
+        }
+
         public string GetToken()
         {
             return GetArgOrPrompt(

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -159,17 +159,34 @@ namespace GitHub.Runner.Listener.Configuration
 
             _term.WriteSection("Runner Registration");
 
-            //Get all the agent pools, and select the first private pool
+            // If we have more than one runner group available, allow the user to specify which one to be added into
+            string poolName = null;
+            TaskAgentPool agentPool = null;
             List<TaskAgentPool> agentPools = await _runnerServer.GetAgentPoolsAsync();
-            TaskAgentPool agentPool = agentPools?.Where(x => x.IsHosted == false).FirstOrDefault();
+            TaskAgentPool defaultPool = agentPools?.Where(x => x.IsInternal).FirstOrDefault();
 
-            if (agentPool == null)
+            if (agentPools?.Where(x => !x.IsHosted).Count() > 1)
             {
-                throw new TaskAgentPoolNotFoundException($"Could not find any private pool. Contact support.");
+                poolName = command.GetRunnerGroupName(defaultPool?.Name);
+                _term.WriteLine();
+                agentPool = agentPools.Where(x => string.Equals(poolName, x.Name, StringComparison.OrdinalIgnoreCase) && !x.IsHosted).FirstOrDefault();
             }
             else
             {
-                Trace.Info("Found a private pool with id {1} and name {2}", agentPool.Id, agentPool.Name);
+                agentPool = defaultPool;
+            }
+
+            if (agentPool == null && poolName == null)
+            {
+                throw new TaskAgentPoolNotFoundException($"Could not find any self-hosted runner groups. Contact support.");
+            }
+            else if (agentPool == null && poolName != null)
+            {
+                throw new TaskAgentPoolNotFoundException($"Could not find any self-hosted runner group named \"{poolName}\".");
+            }
+            else
+            {
+                Trace.Info("Found a self-hosted runner group with id {1} and name {2}", agentPool.Id, agentPool.Name);
                 runnerSettings.PoolId = agentPool.Id;
                 runnerSettings.PoolName = agentPool.Name;
             }

src/Runner.Listener/Runner.cs

@@ -462,13 +462,14 @@ Options:
  --commit   Prints the runner commit
 
 Config Options:
- --unattended     Disable interactive prompts for missing arguments. Defaults will be used for missing options
- --url string     Repository to add the runner to. Required if unattended
- --token string   Registration token. Required if unattended
- --name string    Name of the runner to configure (default {Environment.MachineName ?? "myrunner"})
- --labels string  Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}'
- --work string    Relative runner work directory (default {Constants.Path.WorkDirectory})
- --replace        Replace any existing runner with the same name (default false)");
+ --unattended           Disable interactive prompts for missing arguments. Defaults will be used for missing options
+ --url string           Repository to add the runner to. Required if unattended
+ --token string         Registration token. Required if unattended
+ --name string          Name of the runner to configure (default {Environment.MachineName ?? "myrunner"})
+ --runnergroup string   Name of the runner group to add this runner to (defaults to the default runner group)
+ --labels string        Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}'
+ --work string          Relative runner work directory (default {Constants.Path.WorkDirectory})
+ --replace              Replace any existing runner with the same name (default false)");
 #if OS_WINDOWS
     _term.WriteLine($@" --runasservice   Run the runner as a service");
     _term.WriteLine($@" --windowslogonaccount string   Account to run the service as. Requires runasservice");

src/Runner.Worker/ActionManager.cs

@@ -395,7 +395,7 @@ namespace GitHub.Runner.Worker
                             Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
                         }
                     }
-                    else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                    else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite)
                     {
                         var compositeAction = definition.Data.Execution as CompositeActionExecutionData;
                         Trace.Info($"Load {compositeAction.Steps?.Count ?? 0} action steps.");
@@ -1048,7 +1048,7 @@ namespace GitHub.Runner.Worker
                     Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
                     return null;
                 }
-                else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite)
                 {
                     Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation.");
                     return null;

src/Runner.Worker/ActionManifestManager.cs

@@ -30,6 +30,8 @@ namespace GitHub.Runner.Worker
         Dictionary<string, string> EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary<string, PipelineContextData> extraExpressionValues);
 
         string EvaluateDefaultInput(IExecutionContext executionContext, string inputName, TemplateToken token);
+
+        // string EvaluateDefaultInputInsideComposite(IExecutionContext executionContext, string inputName, TemplateToken token);
     }
 
     public sealed class ActionManifestManager : RunnerService, IActionManifestManager
@@ -105,12 +107,7 @@ namespace GitHub.Runner.Worker
                             break;
 
                         case "outputs":
-                            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
-                            {
-                                actionOutputs = actionPair.Value.AssertMapping("outputs");
-                                break;
-                            }
-                            Trace.Info($"Ignore action property outputs. Outputs for a whole action is not supported yet.");
+                            actionOutputs = actionPair.Value.AssertMapping("outputs");
                             break;
 
                         case "description":
@@ -287,6 +284,9 @@ namespace GitHub.Runner.Worker
             if (token != null)
             {
                 var templateContext = CreateTemplateContext(executionContext);
+                Trace.Info($"Template context GitHub: {StringUtil.ConvertToJson(templateContext.ExpressionValues["github"])}");
+
+                Trace.Info($"Template context keys: {StringUtil.ConvertToJson(templateContext.ExpressionValues.Keys)}");
                 try
                 {
                     var evaluateResult = TemplateEvaluator.Evaluate(templateContext, "input-default-context", token, 0, null, omitHeader: true);
@@ -309,6 +309,83 @@ namespace GitHub.Runner.Worker
             return result;
         }
 
+        // public string EvaluateDefaultInputInsideComposite(
+        //     IExecutionContext executionContext,
+        //     string inputName,
+        //     TemplateToken token)
+        // {
+        //     string result = "";
+        //     if (token != null)
+        //     {
+        //         // Add GitHub Expression Values
+        //         var githubContext = executionContext.ExpressionValues["github"];
+        //         Trace.Info($"GitHub Context EvaluateDefaultInputInsideComposite: {StringUtil.ConvertToJson(githubContext)}");
+
+        //         var temp = new Dictionary<string, PipelineContextData>();
+        //         temp["github"] = githubContext;
+
+        //         // Trace.Info($"Token: {StringUtil.ConvertToJson(token)}");
+
+        //         var templateContext = CreateTemplateContext(executionContext, temp);
+        //         try
+        //         {
+
+        //             var evaluateResult = TemplateEvaluator.Evaluate(templateContext, "uses-input", token, 0, null, omitHeader: true);
+
+        //             Trace.Info($"Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
+        //             templateContext.Errors.Check();
+
+        //             // Can't evaluate the default github.respository, etc.
+
+
+        //             // TODO: restrict to only be used for composite "uses" steps
+        //             // Find better way to isolate only 
+        //             // We could create a whitelist for just checkout?
+        //             // (ex: "repo", "token", etc.)
+        //             // if (evaluateResult is BasicExpressionToken)
+        //             // {
+                        
+
+        //             //     // var evaluateResult2 = TemplateEvaluator.Evaluate(templateContext, "step-with", token, 0, null, omitHeader: true);
+        //             //     // templateContext.Errors.Check();
+
+        //             //     // Trace.Info($"Test2 Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult2)}");
+        //             //     // var result2 = evaluateResult2.AssertString($"default value for input '{inputName}'").Value;
+
+        //             //     // TODO 6/28 => Try just getting it from the getgithubcontext lmao.
+
+        //             //     // Trace.Info($"Basic expr token: {evaluateResult}");
+
+        //             //     var stringVersion = evaluateResult.AssertString($"default value for input '{inputName}'").Value;
+
+        //             //     // no we have to use the template evaluator since it's a
+
+        //             //     // var githubTokenSplit = 
+
+        //             //     // // Evaluate it
+        //             //     // var evaluateResult = executionContext.GetGitHubContext("");
+
+        //             //     return result2;
+        //             // }
+
+        //             // Trace.Info($"Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
+
+        //             // String
+        //             result = evaluateResult.AssertString($"default value for input '{inputName}'").Value;
+        //         }
+        //         catch (Exception ex) when (!(ex is TemplateValidationException))
+        //         {
+        //             Trace.Error(ex);
+        //             templateContext.Errors.Add(ex);
+        //         }
+
+        //         templateContext.Errors.Check();
+        //     }
+
+        //     return result;
+
+        // }
+
         private TemplateContext CreateTemplateContext(
             IExecutionContext executionContext,
             IDictionary<string, PipelineContextData> extraExpressionValues = null)
@@ -423,14 +500,10 @@ namespace GitHub.Runner.Worker
                         preIfToken = run.Value.AssertString("pre-if");
                         break;
                     case "steps":
-                        if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
-                        {
-                            var stepsToken = run.Value.AssertSequence("steps");
-                            steps = PipelineTemplateConverter.ConvertToSteps(templateContext, stepsToken);
-                            templateContext.Errors.Check();
-                            break;
-                        }
-                        throw new Exception("You aren't supposed to be using Composite Actions yet!");
+                        var stepsToken = run.Value.AssertSequence("steps");
+                        steps = PipelineTemplateConverter.ConvertToSteps(templateContext, stepsToken);
+                        templateContext.Errors.Check();
+                        break;
                     default:
                         Trace.Info($"Ignore run property {runsKey}.");
                         break;
@@ -478,7 +551,7 @@ namespace GitHub.Runner.Worker
                         };
                     }
                 }
-                else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase))
                 {
                     if (steps == null)
                     {

src/Runner.Worker/ActionRunner.cs

@@ -169,6 +169,25 @@ namespace GitHub.Runner.Worker
                 validInputs.Add("entryPoint");
                 validInputs.Add("args");
             }
+
+            Trace.Info($"Repo: {ExecutionContext.GetGitHubContext("repository")}");
+
+            // Since we don't pass the GitHub Context attributes to the composite action,
+            // We need to explitly set the default values of certain things we need like the 
+            // default repository 
+
+            // if (ExecutionContext.GetGitHubContext("repository") == null)
+            // {
+            //     ExecutionContext.SetGitHubContext("repository", definition.Directory);
+            // }
+
+            // var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
+
+            // foreach (var pair in githubContext)
+            // {
+            //     ExecutionContext.SetGitHubContext(pair.Key, pair.Value as StringContextData);
+            // }
+            
             // Merge the default inputs from the definition
             if (definition.Data?.Inputs != null)
             {
@@ -179,7 +198,17 @@ namespace GitHub.Runner.Worker
                     validInputs.Add(key);
                     if (!inputs.ContainsKey(key))
                     {
+                        Trace.Info($"Definition Input Key: {key}");
+                        // if (ExecutionContext.InsideComposite)
+                        // {
+                        //     inputs[key] = manifestManager.EvaluateDefaultInputInsideComposite(ExecutionContext, key, input.Value);
+                        // }
+                        // else
+                        // {
+                        //     inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value);
+                        // }
                         inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value);
+                        Trace.Info($"Definition Input Value: {inputs[key]}");
                     }
                 }
             }

src/Runner.Worker/ExecutionContext.cs

@@ -53,19 +53,21 @@ namespace GitHub.Runner.Worker
         JobContext JobContext { get; }
 
         // Only job level ExecutionContext has JobSteps
-        List<IStep> JobSteps { get; }
+        Queue<IStep> JobSteps { get; }
 
         // Only job level ExecutionContext has PostJobSteps
         Stack<IStep> PostJobSteps { get; }
 
         bool EchoOnActionCommand { get; set; }
 
+        bool InsideComposite { get; }
+
         ExecutionContext Root { get; }
 
         // Initialize
         void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token);
         void CancelToken();
-        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, CancellationTokenSource cancellationTokenSource = null);
+        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool insideComposite = false, CancellationTokenSource cancellationTokenSource = null);
 
         // logging
         long Write(string tag, string message);
@@ -142,7 +144,7 @@ namespace GitHub.Runner.Worker
         public GlobalContext Global { get; private set; }
 
         // Only job level ExecutionContext has JobSteps
-        public List<IStep> JobSteps { get; private set; }
+        public Queue<IStep> JobSteps { get; private set; }
 
         // Only job level ExecutionContext has PostJobSteps
         public Stack<IStep> PostJobSteps { get; private set; }
@@ -152,6 +154,8 @@ namespace GitHub.Runner.Worker
 
         public bool EchoOnActionCommand { get; set; }
 
+        public bool InsideComposite { get; private set; }
+
         public TaskResult? Result
         {
             get
@@ -256,7 +260,7 @@ namespace GitHub.Runner.Worker
             DictionaryContextData inputsData,
             Dictionary<string, string> envData)
         {
-            step.ExecutionContext = Root.CreateChild(_record.Id, step.DisplayName, _record.Id.ToString("N"), scopeName, step.Action.ContextName, logger: _logger, cancellationTokenSource: CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token));
+            step.ExecutionContext = Root.CreateChild(_record.Id, step.DisplayName, _record.Id.ToString("N"), scopeName, step.Action.ContextName, logger: _logger, insideComposite: true, cancellationTokenSource: CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token));
             step.ExecutionContext.ExpressionValues["inputs"] = inputsData;
             step.ExecutionContext.ExpressionValues["steps"] = Global.StepsContext.GetScope(step.ExecutionContext.GetFullyQualifiedContextName());
 
@@ -275,7 +279,7 @@ namespace GitHub.Runner.Worker
             return step;
         }
 
-        public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, CancellationTokenSource cancellationTokenSource = null)
+        public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool insideComposite = false, CancellationTokenSource cancellationTokenSource = null)
         {
             Trace.Entering();
 
@@ -322,6 +326,8 @@ namespace GitHub.Runner.Worker
                 child._logger.Setup(_mainTimelineId, recordId);
             }
 
+            child.InsideComposite = insideComposite;
+
             return child;
         }
 
@@ -657,7 +663,7 @@ namespace GitHub.Runner.Worker
             Global.PrependPath = new List<string>();
 
             // JobSteps for job ExecutionContext
-            JobSteps = new List<IStep>();
+            JobSteps = new Queue<IStep>();
 
             // PostJobSteps for job ExecutionContext
             PostJobSteps = new Stack<IStep>();

src/Runner.Worker/GitHubContext.cs

@@ -9,6 +9,7 @@ namespace GitHub.Runner.Worker
         private readonly HashSet<string> _contextEnvWhitelist = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
         {
             "action",
+            "action_path",
             "actor",
             "api_url",
             "base_ref",

src/Runner.Worker/Handlers/CompositeActionHandler.cs

@@ -5,11 +5,14 @@ using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Expressions2;
 using GitHub.DistributedTask.ObjectTemplating.Tokens;
 using GitHub.DistributedTask.Pipelines.ContextData;
+using GitHub.DistributedTask.Pipelines.ObjectTemplating;
 using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Sdk;
+using GitHub.Runner.Worker.Expressions;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 
@@ -35,6 +38,8 @@ namespace GitHub.Runner.Worker.Handlers
             var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
             ArgUtil.NotNull(githubContext, nameof(githubContext));
 
+            // Trace.Info($"Github Context: {StringUtil.ConvertToJson(githubContext)}");
+
             // Resolve action steps
             var actionSteps = Data.Steps;
 
@@ -45,6 +50,8 @@ namespace GitHub.Runner.Worker.Handlers
                 inputsData[i.Key] = new StringContextData(i.Value);
             }
 
+            Trace.Info($"Composite Actions Inputs {StringUtil.ConvertToJson(inputsData)}");
+
             // Initialize Composite Steps List of Steps
             var compositeSteps = new List<IStep>();
 
@@ -56,6 +63,14 @@ namespace GitHub.Runner.Worker.Handlers
                 childScopeName = $"__{Guid.NewGuid()}";
             }
 
+            // Copy the github context so that we don't modify the original pointer
+            // We can't use PipelineContextData.Clone() since that creates a null pointer exception for copying a GitHubContext
+            var compositeGitHubContext = new GitHubContext();
+            foreach (var pair in githubContext)
+            {
+                compositeGitHubContext[pair.Key] = pair.Value.Clone();
+            }
+
             foreach (Pipelines.ActionStep actionStep in actionSteps)
             {
                 var actionRunner = HostContext.CreateService<IActionRunner>();
@@ -64,6 +79,12 @@ namespace GitHub.Runner.Worker.Handlers
                 actionRunner.Condition = actionStep.Condition;
 
                 var step = ExecutionContext.CreateCompositeStep(childScopeName, actionRunner, inputsData, Environment);
+
+                // Set GITHUB_ACTION_PATH
+                step.ExecutionContext.ExpressionValues["github"] = compositeGitHubContext;
+                step.ExecutionContext.SetGitHubContext("action_path", ActionDirectory);
+                step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo<HashFilesFunction>(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue));
+
                 compositeSteps.Add(step);
             }
 
@@ -176,9 +197,6 @@ namespace GitHub.Runner.Worker.Handlers
 
                 var actionStep = step as IActionRunner;
 
-                // Set GITHUB_ACTION
-                step.ExecutionContext.SetGitHubContext("action", step.ExecutionContext.GetFullyQualifiedContextName());
-
                 try
                 {
                     // Evaluate and merge action's env block to env context
@@ -215,12 +233,6 @@ namespace GitHub.Runner.Worker.Handlers
 
         private async Task RunStepAsync(IStep step)
         {
-            // Try to evaluate the display name
-            if (step is IActionRunner actionRunner && actionRunner.Stage == ActionRunStage.Main)
-            {
-                actionRunner.TryEvaluateDisplayName(step.ExecutionContext.ExpressionValues, step.ExecutionContext);
-            }
-
             // Start the step.
             Trace.Info("Starting the step.");
             step.ExecutionContext.Debug($"Starting: {step.DisplayName}");

src/Runner.Worker/Handlers/ScriptHandler.cs

@@ -23,6 +23,19 @@ namespace GitHub.Runner.Worker.Handlers
 
         public override void PrintActionDetails(ActionRunStage stage)
         {
+            // We don't want to display the internal workings if composite (similar/equivalent information can be found in debug)
+            void writeDetails(string message)
+            {
+                if (ExecutionContext.InsideComposite)
+                {
+                    ExecutionContext.Debug(message);
+                }
+                else
+                {
+                    ExecutionContext.Output(message);
+                }
+            }
+
             if (stage == ActionRunStage.Post)
             {
                 throw new NotSupportedException("Script action should not have 'Post' job action.");
@@ -39,7 +52,7 @@ namespace GitHub.Runner.Worker.Handlers
                     firstLine = firstLine.Substring(0, firstNewLine);
                 }
 
-                ExecutionContext.Output($"##[group]Run {firstLine}");
+                writeDetails(ExecutionContext.InsideComposite ? $"Run {firstLine}" : $"##[group]Run {firstLine}");
             }
             else
             {
@@ -50,7 +63,7 @@ namespace GitHub.Runner.Worker.Handlers
             foreach (var line in multiLines)
             {
                 // Bright Cyan color
-                ExecutionContext.Output($"\x1b[36;1m{line}\x1b[0m");
+                writeDetails($"\x1b[36;1m{line}\x1b[0m");
             }
 
             string argFormat;
@@ -109,23 +122,23 @@ namespace GitHub.Runner.Worker.Handlers
 
             if (!string.IsNullOrEmpty(shellCommandPath))
             {
-                ExecutionContext.Output($"shell: {shellCommandPath} {argFormat}");
+                writeDetails($"shell: {shellCommandPath} {argFormat}");
             }
             else
             {
-                ExecutionContext.Output($"shell: {shellCommand} {argFormat}");
+                writeDetails($"shell: {shellCommand} {argFormat}");
             }
 
             if (this.Environment?.Count > 0)
             {
-                ExecutionContext.Output("env:");
+                writeDetails("env:");
                 foreach (var env in this.Environment)
                 {
-                    ExecutionContext.Output($"  {env.Key}: {env.Value}");
+                    writeDetails($"  {env.Key}: {env.Value}");
                 }
             }
 
-            ExecutionContext.Output("##[endgroup]");
+            writeDetails(ExecutionContext.InsideComposite ? "" : "##[endgroup]");
         }
 
         public async Task RunAsync(ActionRunStage stage)
@@ -151,8 +164,6 @@ namespace GitHub.Runner.Worker.Handlers
             string workingDirectory = null;
             if (!Inputs.TryGetValue("workingDirectory", out workingDirectory))
             {
-                // TODO: figure out how defaults interact with template later
-                // for now, we won't check job.defaults if we are inside a template.
                 if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
                 {
                     if (runDefaults.TryGetValue("working-directory", out workingDirectory))
@@ -167,8 +178,6 @@ namespace GitHub.Runner.Worker.Handlers
             string shell = null;
             if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell))
             {
-                // TODO: figure out how defaults interact with template later
-                // for now, we won't check job.defaults if we are inside a template.
                 if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
                 {
                     if (runDefaults.TryGetValue("shell", out shell))

src/Runner.Worker/JobRunner.cs

@@ -152,7 +152,7 @@ namespace GitHub.Runner.Worker
                 {
                     foreach (var step in jobSteps)
                     {
-                        jobContext.JobSteps.Add(step);
+                        jobContext.JobSteps.Enqueue(step);
                     }
 
                     await stepsRunner.RunAsync(jobContext);

src/Runner.Worker/StepsRunner.cs

@@ -59,14 +59,13 @@ namespace GitHub.Runner.Worker
                     checkPostJobActions = true;
                     while (jobContext.PostJobSteps.TryPop(out var postStep))
                     {
-                        jobContext.JobSteps.Add(postStep);
+                        jobContext.JobSteps.Enqueue(postStep);
                     }
 
                     continue;
                 }
 
-                var step = jobContext.JobSteps[0];
-                jobContext.JobSteps.RemoveAt(0);
+                var step = jobContext.JobSteps.Dequeue();
 
                 Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
                 ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));

src/Runner.Worker/action_yaml.json

@@ -108,19 +108,46 @@
             }
         },
         "composite-steps": {
-            "context": [
-                "github",
-                "strategy",
-                "matrix",
-                "steps",
-                "inputs",
-                "job",
-                "runner",
-                "env",
-                "hashFiles(1,255)"
-            ],
             "sequence": {
-                "item-type": "any"
+                "item-type": "composite-step-types"
+            }
+        },
+        "composite-step-types": {
+            "one-of": [
+                "composite-step",
+                "uses-step"
+            ]
+        },
+        "composite-step": {
+            "mapping": {
+                "properties": {
+                    "name": "string-steps-context",
+                    "id": "non-empty-string",
+                    "run": {
+                        "type": "string-steps-context",
+                        "required": true
+                    },
+                    "env": "step-env",
+                    "working-directory": "string-steps-context",
+                    "shell": {
+                        "type": "non-empty-string",
+                        "required": true
+                    }
+                }
+            }
+        },
+        "uses-step": {
+            "mapping": {
+                "properties": {
+                    "name": "string-steps-context",
+                    "id": "non-empty-string",
+                    "uses": {
+                        "type": "non-empty-string",
+                        "required": true
+                    },
+                    "with": "step-with",
+                    "env": "step-env"
+                }
             }
         },
         "container-runs-context": {
@@ -157,6 +184,68 @@
             "string": {
                 "require-non-empty": true
             }
+        },
+        "string-steps-context": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "string": {}
+        },
+        "step-env": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "mapping": {
+                "loose-key-type": "non-empty-string",
+                "loose-value-type": "string"
+            }
+        },
+        "step-with": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "mapping": {
+                "loose-key-type": "non-empty-string",
+                "loose-value-type": "string"
+            }
+        },
+        "uses-input": {
+            "context": [
+                "github",
+                "inputs",
+                "strategy",
+                "matrix",
+                "steps",
+                "job",
+                "runner",
+                "env",
+                "hashFiles(1,255)"
+            ],
+            "string": {}
         }
     }
 }

src/Sdk/DTWebApi/WebApi/TaskAgentPoolReference.cs

@@ -29,6 +29,7 @@ namespace GitHub.DistributedTask.WebApi
             this.PoolType = referenceToBeCloned.PoolType;
             this.Size = referenceToBeCloned.Size;
             this.IsLegacy = referenceToBeCloned.IsLegacy;
+            this.IsInternal = referenceToBeCloned.IsInternal;
         }
 
         public TaskAgentPoolReference Clone()
@@ -67,6 +68,16 @@ namespace GitHub.DistributedTask.WebApi
             set;
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether or not this pool is internal and can't be modified by users
+        /// </summary>
+        [DataMember]
+        public bool IsInternal
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets or sets the type of the pool
         /// </summary>

src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs

@@ -39,10 +39,12 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration
         private string _expectedToken = "expectedToken";
         private string _expectedServerUrl = "https://codedev.ms";
         private string _expectedAgentName = "expectedAgentName";
-        private string _expectedPoolName = "poolName";
+        private string _defaultRunnerGroupName = "defaultRunnerGroup";
+        private string _secondRunnerGroupName = "secondRunnerGroup";
         private string _expectedAuthType = "pat";
         private string _expectedWorkFolder = "_work";
-        private int _expectedPoolId = 1;
+        private int _defaultRunnerGroupId = 1;
+        private int _secondRunnerGroupId = 2;
         private RSACryptoServiceProvider rsa = null;
         private RunnerSettings _configMgrAgentSettings = new RunnerSettings();
 
@@ -97,7 +99,7 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration
             _serviceControlManager.Setup(x => x.GenerateScripts(It.IsAny<RunnerSettings>()));
 #endif
 
-            var expectedPools = new List<TaskAgentPool>() { new TaskAgentPool(_expectedPoolName) { Id = _expectedPoolId } };
+            var expectedPools = new List<TaskAgentPool>() { new TaskAgentPool(_defaultRunnerGroupName) { Id = _defaultRunnerGroupId, IsInternal = true }, new TaskAgentPool(_secondRunnerGroupName) { Id = _secondRunnerGroupId } };
             _runnerServer.Setup(x => x.GetAgentPoolsAsync(It.IsAny<string>(), It.IsAny<TaskAgentPoolType>())).Returns(Task.FromResult(expectedPools));
 
             var expectedAgents = new List<TaskAgent>();
@@ -155,7 +157,7 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration
                        "configure",                
                        "--url", _expectedServerUrl,
                        "--name", _expectedAgentName,
-                       "--pool", _expectedPoolName,
+                       "--runnergroup", _secondRunnerGroupName,
                        "--work", _expectedWorkFolder,
                        "--auth", _expectedAuthType,
                        "--token", _expectedToken,
@@ -175,7 +177,7 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration
                 Assert.NotNull(s);
                 Assert.True(s.ServerUrl.Equals(_expectedServerUrl));
                 Assert.True(s.AgentName.Equals(_expectedAgentName));
-                Assert.True(s.PoolId.Equals(_expectedPoolId));
+                Assert.True(s.PoolId.Equals(_secondRunnerGroupId));
                 Assert.True(s.WorkFolder.Equals(_expectedWorkFolder));
 
                 // validate GetAgentPoolsAsync gets called twice with automation pool type

src/Test/L0/Worker/StepsRunnerL0.cs

@@ -82,7 +82,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -117,7 +117,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -156,7 +156,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -210,7 +210,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -289,7 +289,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Steps.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -332,7 +332,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Step.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Step.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -363,7 +363,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -393,7 +393,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 {
                     _ec.Object.Result = null;
 
-                    _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(variableSet.Select(x => x.Object).ToList()));
+                    _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(variableSet.Select(x => x.Object).ToList()));
 
                     // Act.
                     await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -419,7 +419,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -457,7 +457,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -495,7 +495,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -526,7 +526,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);
@@ -562,7 +562,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 _ec.Object.Result = null;
 
-                _ec.Setup(x => x.JobSteps).Returns(new List<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
+                _ec.Setup(x => x.JobSteps).Returns(new Queue<IStep>(new[] { step1.Object, step2.Object, step3.Object }));
 
                 // Act.
                 await _stepsRunner.RunAsync(jobContext: _ec.Object);

src/runnerversion

@@ -1 +1 @@
-2.267.0
+2.274.0