Update src/Runner.Worker/ActionManager.cs

Co-Authored-By: Josh Soref <jsoref@users.noreply.github.com>

docs/adrs/0397-runner-registration-labels.md

@@ -0,0 +1,56 @@
+# ADR 0397: Support adding custom labels during runner config
+**Date**: 2020-03-30
+
+**Status**: Approved
+
+## Context
+
+Since configuring self-hosted runners is commonly automated via scripts, the labels need to be able to be created during configuration.  The runner currently registers the built-in labels (os, arch) during registration but does not accept labels via command line args to extend the set registered.
+
+See Issue: https://github.com/actions/runner/issues/262
+
+This is another version of [ADR275](https://github.com/actions/runner/pull/275)
+
+## Decision
+
+This ADR proposes that we add a `--labels` option to `config`, which could be used to add custom additional labels to the configured runner.
+
+For example, to add a single extra label the operator could run:
+```bash
+./config.sh --labels mylabel
+```
+> Note: the current runner command line parsing and envvar override algorithm only supports a single argument (key).
+
+This would add the label `mylabel` to the runner, and enable users to select the runner in their workflow using this label:
+```yaml
+runs-on: [self-hosted, mylabel]
+```
+
+To add multiple labels the operator could run:
+```bash
+./config.sh --labels mylabel,anotherlabel
+```
+> Note: the current runner command line parsing and envvar override algorithm only supports a single argument (key).
+
+This would add the label `mylabel` and `anotherlabel` to the runner, and enable users to select the runner in their workflow using this label:
+```yaml
+runs-on: [self-hosted, mylabel, anotherlabel]
+```
+
+It would not be possible to remove labels from an existing runner using `config.sh`, instead labels would have to be removed using the GitHub UI.
+
+The labels argument will split on commas, trim and discard empty strings.  That effectively means don't use commans in unattended config label names.  Alternatively we could choose to escape commans but it's a nice to have.
+
+## Replace
+
+If an existing runner exists and the option to replace is chosen (interactively of via unattend as in this scenario), then the labels will be replaced / overwritten (not merged).
+
+## Overriding built-in labels
+
+Note that it is possible to register "built-in" hosted labels like `ubuntu-latest` and is not considered an error.  This is an effective way for the org / runner admin to dictate by policy through registration that this set of runners will be used without having to edit all the workflow files now and in the future.
+
+We will also not make other restrictions such as limiting explicitly adding os / arch labels and validating.  We will assume that explicit labels were added for a reason and not restricting offers the most flexibility and future proofing / compat.
+
+## Consequences
+
+The ability to add custom labels to a self-hosted runner would enable most scenarios where job runner selection based on runner capabilities or characteristics are required.

src/Runner.Common/ConfigurationStore.cs

@@ -15,6 +15,9 @@ namespace GitHub.Runner.Common
     [DataContract]
     public sealed class RunnerSettings
     {
+        [DataMember(Name = "IsHostedServer", EmitDefaultValue = false)]
+        private bool? _isHostedServer;
+
         [DataMember(EmitDefaultValue = false)]
         public int AgentId { get; set; }
 
@@ -42,6 +45,21 @@ namespace GitHub.Runner.Common
         [DataMember(EmitDefaultValue = false)]
         public string MonitorSocketAddress { get; set; }
 
+        [IgnoreDataMember]
+        public bool IsHostedServer
+        {
+            get
+            {
+                // Old runners do not have this property. Hosted runners likely don't have this property either.
+                return _isHostedServer ?? true;
+            }
+
+            set
+            {
+                _isHostedServer = value;
+            }
+        }
+
         /// <summary>
         // Computed property for convenience. Can either return:
         // 1. If runner was configured at the repo level, returns something like: "myorg/myrepo"
@@ -69,6 +87,15 @@ namespace GitHub.Runner.Common
                 return repoOrOrgName;
             }
         }
+
+        [OnSerializing]
+        private void OnSerializing(StreamingContext context)
+        {
+            if (_isHostedServer.HasValue && _isHostedServer.Value)
+            {
+                _isHostedServer = null;
+            }
+        }
     }
 
     [ServiceLocator(Default = typeof(ConfigurationStore))]

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -86,7 +86,6 @@ namespace GitHub.Runner.Listener.Configuration
 
             RunnerSettings runnerSettings = new RunnerSettings();
 
-            bool isHostedServer = false;
             // Loop getting url and creds until you can connect
             ICredentialProvider credProvider = null;
             VssCredentials creds = null;
@@ -95,8 +94,7 @@ namespace GitHub.Runner.Listener.Configuration
             {
                 // Get the URL
                 var inputUrl = command.GetUrl();
-                if (!inputUrl.Contains("github.com", StringComparison.OrdinalIgnoreCase) &&
-                    !inputUrl.Contains("github.localhost", StringComparison.OrdinalIgnoreCase))
+                if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase))
                 {
                     runnerSettings.ServerUrl = inputUrl;
                     // Get the credentials
@@ -117,7 +115,7 @@ namespace GitHub.Runner.Listener.Configuration
                 try
                 {
                     // Determine the service deployment type based on connection data. (Hosted/OnPremises)
-                    isHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds);
+                    runnerSettings.IsHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds);
 
                     // Validate can connect.
                     await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds);
@@ -199,7 +197,7 @@ namespace GitHub.Runner.Listener.Configuration
                 }
                 else
                 {
-                    // Create a new agent. 
+                    // Create a new agent.
                     agent = CreateNewAgent(runnerSettings.AgentName, publicKey);
 
                     try
@@ -248,7 +246,7 @@ namespace GitHub.Runner.Listener.Configuration
             {
                 UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl);
                 UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl);
-                if (!isHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
+                if (!runnerSettings.IsHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
                 {
                     oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme;
                     oauthEndpointUrlBuilder.Host = configServerUrl.Host;
@@ -291,7 +289,7 @@ namespace GitHub.Runner.Listener.Configuration
             {
                 // there are two exception messages server send that indicate clock skew.
                 // 1. The bearer token expired on {jwt.ValidTo}. Current server time is {DateTime.UtcNow}.
-                // 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}.                
+                // 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}.
                 Trace.Error("Catch exception during test agent connection.");
                 Trace.Error(ex);
                 throw new Exception("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again.");
@@ -381,7 +379,6 @@ namespace GitHub.Runner.Listener.Configuration
                     }
 
                     // Determine the service deployment type based on connection data. (Hosted/OnPremises)
-                    bool isHostedServer = await IsHostedServer(settings.ServerUrl, creds);
                     await _runnerServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
 
                     var agents = await _runnerServer.GetAgentsAsync(settings.PoolId, settings.AgentName);
@@ -404,7 +401,7 @@ namespace GitHub.Runner.Listener.Configuration
                     _term.WriteLine("Cannot connect to server, because config files are missing. Skipping removing runner from the server.");
                 }
 
-                //delete credential config files               
+                //delete credential config files
                 currentAction = "Removing .credentials";
                 if (hasCredentials)
                 {
@@ -418,7 +415,7 @@ namespace GitHub.Runner.Listener.Configuration
                     _term.WriteLine("Does not exist. Skipping " + currentAction);
                 }
 
-                //delete settings config file                
+                //delete settings config file
                 currentAction = "Removing .runner";
                 if (isConfigured)
                 {
@@ -521,8 +518,19 @@ namespace GitHub.Runner.Listener.Configuration
 
         private async Task<GitHubAuthResult> GetTenantCredential(string githubUrl, string githubToken, string runnerEvent)
         {
+            var githubApiUrl = "";
             var gitHubUrlBuilder = new UriBuilder(githubUrl);
-            var githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration";
+            if (string.Equals(gitHubUrlBuilder.Host, "github.com", StringComparison.OrdinalIgnoreCase) ||
+                string.Equals(gitHubUrlBuilder.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) ||
+                string.Equals(gitHubUrlBuilder.Host, "github.localhost", StringComparison.OrdinalIgnoreCase))
+            {
+                githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration";
+            }
+            else
+            {
+                githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/actions/runner-registration";
+            }
+
             using (var httpClientHandler = HostContext.CreateHttpClientHandler())
             using (var httpClient = new HttpClient(httpClientHandler))
             {

src/Runner.Worker/ActionManager.cs

@@ -51,15 +51,21 @@ namespace GitHub.Runner.Worker
             List<JobExtensionRunner> containerSetupSteps = new List<JobExtensionRunner>();
             IEnumerable<Pipelines.ActionStep> actions = steps.OfType<Pipelines.ActionStep>();
 
-            // TODO: Depreciate the PREVIEW_ACTION_TOKEN
+            // TODO: Deprecate the PREVIEW_ACTION_TOKEN
             // Log even if we aren't using it to ensure users know.
             if (!string.IsNullOrEmpty(executionContext.Variables.Get("PREVIEW_ACTION_TOKEN")))
             {
-                executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is depreciated. Please remove it from the repository's secrets");
+                executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets");
             }
 
-            // Clear the cache (local runner)
-            IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
+            // Clear the cache (for self-hosted runners)
+            // Note, temporarily avoid this step for the on-premises product, to avoid rate limiting.
+            var configurationStore = HostContext.GetService<IConfigurationStore>();
+            var isHostedServer = configurationStore.GetSettings().IsHostedServer;
+            if (isHostedServer)
+            {
+                IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
+            }
 
             foreach (var action in actions)
             {
@@ -448,7 +454,8 @@ namespace GitHub.Runner.Worker
             ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
 
             string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
-            if (File.Exists(destDirectory + ".completed"))
+            string watermarkFile = destDirectory + ".completed";
+            if (File.Exists(watermarkFile))
             {
                 executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
                 return;
@@ -458,7 +465,7 @@ namespace GitHub.Runner.Worker
                 // make sure we get a clean folder ready to use.
                 IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
                 Directory.CreateDirectory(destDirectory);
-                executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
+                executionContext.Debug($"Downloading action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
             }
 
 #if OS_WINDOWS
@@ -498,24 +505,33 @@ namespace GitHub.Runner.Worker
                             using (var httpClientHandler = HostContext.CreateHttpClientHandler())
                             using (var httpClient = new HttpClient(httpClientHandler))
                             {
-                                var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
-                                if (string.IsNullOrEmpty(authToken))
+                                var configurationStore = HostContext.GetService<IConfigurationStore>();
+                                var isHostedServer = configurationStore.GetSettings().IsHostedServer;
+                                if (isHostedServer)
                                 {
-                                    // TODO: Depreciate the PREVIEW_ACTION_TOKEN
-                                    authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
-                                }
+                                    var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
+                                    if (string.IsNullOrEmpty(authToken))
+                                    {
+                                        // TODO: Deprecate the PREVIEW_ACTION_TOKEN
+                                        authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
+                                    }
 
-                                if (!string.IsNullOrEmpty(authToken))
-                                {
-                                    HostContext.SecretMasker.AddValue(authToken);
-                                    var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
-                                    httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
+                                    if (!string.IsNullOrEmpty(authToken))
+                                    {
+                                        HostContext.SecretMasker.AddValue(authToken);
+                                        var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
+                                        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
+                                    }
+                                    else
+                                    {
+                                        var accessToken = executionContext.GetGitHubContext("token");
+                                        var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
+                                        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
+                                    }
                                 }
                                 else
                                 {
-                                    var accessToken = executionContext.GetGitHubContext("token");
-                                    var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
-                                    httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
+                                    // Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated.
                                 }
 
                                 httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
@@ -597,7 +613,10 @@ namespace GitHub.Runner.Worker
                 }
 #endif
 
-                // repository archive from github always contains a nested folder
+                // repository archive from github always contains a nested folder, the nested folder contains the short SHA for the ref
+                // ex: actions/checkout@master the nested folder looks like actions-checkout-01aeccc
+                // However, when you reference action using tag, the SHA in the folder name is not really the commit SHA, it's the SHA for the tag itself.
+                var shortSha = "";
                 var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
                 if (subDirectories.Length != 1)
                 {
@@ -605,15 +624,42 @@ namespace GitHub.Runner.Worker
                 }
                 else
                 {
+                    shortSha = subDirectories[0].Name;
+                    var splitIndex = shortSha.LastIndexOf('-');
+                    if (splitIndex > 0 && splitIndex < shortSha.Length - 1)
+                    {
+                        shortSha = shortSha.Substring(splitIndex + 1);
+
+                        // we will print the short SHA when action is referenced via branch/tag
+                        if (repositoryReference.Ref.StartsWith(shortSha, StringComparison.OrdinalIgnoreCase))
+                        {
+                            // action is already referenced by SHA
+                            shortSha = null;
+                        }
+                    }
+                    else
+                    {
+                        shortSha = null;
+                    }
+
                     executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
                     IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
                 }
 
                 Trace.Verbose("Create watermark file indicate action download succeed.");
-                File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());
+                File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
 
                 executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
                 Trace.Info("Finished getting action repository.");
+
+                if (string.IsNullOrEmpty(shortSha))
+                {
+                    executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
+                }
+                else
+                {
+                    executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}' (SHA: {shortSha})");
+                }
             }
             finally
             {
@@ -863,4 +909,3 @@ namespace GitHub.Runner.Worker
         public string ActionRepository { get; set; }
     }
 }
-

src/Runner.Worker/GitHubContext.cs

@@ -10,6 +10,7 @@ namespace GitHub.Runner.Worker
         {
             "action",
             "actor",
+            "api_url", // temp for GHES alpha release
             "base_ref",
             "event_name",
             "event_path",
@@ -21,6 +22,7 @@ namespace GitHub.Runner.Worker
             "run_id",
             "run_number",
             "sha",
+            "url", // temp for GHES alpha release
             "workflow",
             "workspace",
         };

src/Runner.Worker/JobExtension.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Runtime.Serialization;
@@ -127,6 +128,17 @@ namespace GitHub.Runner.Worker
                     context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
                     context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));
 
+                    // Temporary hack for GHES alpha
+                    var configurationStore = HostContext.GetService<IConfigurationStore>();
+                    var runnerSettings = configurationStore.GetSettings();
+                    if (!runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
+                    {
+                        var url = new Uri(runnerSettings.GitHubUrl);
+                        var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
+                        context.SetGitHubContext("url", $"{url.Scheme}://{url.Host}{portInfo}");
+                        context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
+                    }
+
                     // Evaluate the job-level environment variables
                     context.Debug("Evaluating job-level environment variables");
                     var templateEvaluator = context.ToPipelineTemplateEvaluator();

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs

@@ -8,6 +8,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
     {
         public const String Always = "always";
         public const String BooleanStepsContext = "boolean-steps-context";
+        public const String BooleanStrategyContext = "boolean-strategy-context";
         public const String CancelTimeoutMinutes = "cancel-timeout-minutes";
         public const String Cancelled = "cancelled";
         public const String Checkout = "checkout";

src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -12,7 +12,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
     /// <summary>
     /// Converts a YAML file into a TemplateToken
     /// </summary>
-    public sealed class YamlObjectReader : IObjectReader
+    internal sealed class YamlObjectReader : IObjectReader
     {
         internal YamlObjectReader(
             Int32? fileId,

src/Sdk/DTPipelines/workflow-v1.0.json

@@ -739,7 +739,7 @@
     "container-env": {
       "mapping": {
         "loose-key-type": "non-empty-string",
-        "loose-value-type": "string"
+        "loose-value-type": "string-runner-context"
       }
     },
 

src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs

@@ -37,7 +37,7 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration
 
         private Mock<IRSAKeyManager> _rsaKeyManager;
         private string _expectedToken = "expectedToken";
-        private string _expectedServerUrl = "https://localhost";
+        private string _expectedServerUrl = "https://codedev.ms";
         private string _expectedAgentName = "expectedAgentName";
         private string _expectedPoolName = "poolName";
         private string _expectedAuthType = "pat";

src/Test/L0/Worker/ActionManagerL0.cs

@@ -111,6 +111,57 @@ namespace GitHub.Runner.Common.Tests.Worker
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public async void PrepareActions_SkipDownloadActionFromGraphWhenCached_OnPremises()
+        {
+            try
+            {
+                // Arrange
+                Setup();
+                var actionId = Guid.NewGuid();
+                var actions = new List<Pipelines.ActionStep>
+                {
+                    new Pipelines.ActionStep()
+                    {
+                        Name = "action",
+                        Id = actionId,
+                        Reference = new Pipelines.RepositoryPathReference()
+                        {
+                            Name = "actions/no-such-action",
+                            Ref = "master",
+                            RepositoryType = "GitHub"
+                        }
+                    }
+                };
+                _configurationStore.Object.GetSettings().IsHostedServer = false;
+                var actionDirectory = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/no-such-action", "master");
+                Directory.CreateDirectory(actionDirectory);
+                var watermarkFile = $"{actionDirectory}.completed";
+                File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
+                var actionFile = Path.Combine(actionDirectory, "action.yml");
+                File.WriteAllText(actionFile, @"
+name: ""no-such-action""
+runs:
+  using: node12
+  main: no-such-action.js
+");
+                var testFile = Path.Combine(actionDirectory, "test-file");
+                File.WriteAllText(testFile, "asdf");
+
+                // Act
+                await _actionManager.PrepareActionsAsync(_ec.Object, actions);
+
+                // Assert
+                Assert.True(File.Exists(testFile));
+            }
+            finally
+            {
+                Teardown();
+            }
+        }
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]