temporary local changes only for testing against GHES

* Add README

* Adapt create-latest-svc to work with GHES

* Fix inverted conditions

scripts/README.md

@@ -0,0 +1,4 @@
+# Sample scripts for self-hosted runners
+
+Here are some examples to work from if you'd like to automate your use of self-hosted runners.
+See the docs [here](../docs/automate.md).

scripts/create-latest-svc.sh

@@ -7,27 +7,29 @@ set -e
 # Configures as a service
 #
 # Examples:
-# RUNNER_CFG_PAT=<yourPAT> ./create-latest-svc.sh myuser/myrepo
-# RUNNER_CFG_PAT=<yourPAT> ./create-latest-svc.sh myorg
+# RUNNER_CFG_PAT=<yourPAT> ./create-latest-svc.sh myuser/myrepo my.ghe.deployment.net
+# RUNNER_CFG_PAT=<yourPAT> ./create-latest-svc.sh myorg my.ghe.deployment.net
 #
 # Usage:
 #     export RUNNER_CFG_PAT=<yourPAT>
-#     ./create-latest-svc scope [name] [user]
+#     ./create-latest-svc scope [ghe_domain] [name] [user]
+#
+#      scope       required  repo (:owner/:repo) or org (:organization)
+#      ghe_domain  optional  the fully qualified domain name of your GitHub Enterprise Server deployment
+#      name        optional  defaults to hostname
+#      user        optional  user svc will run as. defaults to current
 #
-#      scope required  repo (:owner/:repo) or org (:organization)
-#      name  optional  defaults to hostname
-#      user  optional  user svc will run as. defaults to current
-# 
 # Notes:
 # PATS over envvars are more secure
 # Should be used on VMs and not containers
-# Works on OSX and Linux 
+# Works on OSX and Linux
 # Assumes x64 arch
 #
 
 runner_scope=${1}
-runner_name=${2:-$(hostname)}
-svc_user=${3:-$USER}
+ghe_hostname=${2}
+runner_name=${3:-$(hostname)}
+svc_user=${4:-$USER}
 
 echo "Configuring runner @ ${runner_scope}"
 sudo echo
@@ -51,9 +53,9 @@ which curl || fatal "curl required.  Please install in PATH with apt-get, brew,
 which jq || fatal "jq required.  Please install in PATH with apt-get, brew, etc"
 
 # bail early if there's already a runner there. also sudo early
-if [ -d ./runner ]; then 
+if [ -d ./runner ]; then
     fatal "Runner already exists.  Use a different directory or delete ./runner"
-fi 
+fi
 
 sudo -u ${svc_user} mkdir runner
 
@@ -66,15 +68,20 @@ sudo -u ${svc_user} mkdir runner
 echo
 echo "Generating a registration token..."
 
-# if the scope has a slash, it's an repo runner
-base_api_url="https://api.github.com/orgs"
-if [[ "$runner_scope" == *\/* ]]; then
-    base_api_url="https://api.github.com/repos"
+base_api_url="https://api.github.com"
+if [ -n "${ghe_hostname}" ]; then
+    base_api_url="https://${ghe_hostname}/api/v3"
 fi
 
-export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token')
+# if the scope has a slash, it's a repo runner
+orgs_or_repos="orgs"
+if [[ "$runner_scope" == *\/* ]]; then
+    orgs_or_repos="repos"
+fi
 
-if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi 
+export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${orgs_or_repos}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token')
+
+if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi
 
 #---------------------------------------
 # Download latest released and extract
@@ -82,6 +89,7 @@ if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi
 echo
 echo "Downloading latest runner ..."
 
+# For the GHES Alpha, download the runner from github.com
 latest_version_label=$(curl -s -X GET 'https://api.github.com/repos/actions/runner/releases/latest' | jq -r '.tag_name')
 latest_version=$(echo ${latest_version_label:1})
 runner_file="actions-runner-${runner_plat}-x64-${latest_version}.tar.gz"
@@ -116,6 +124,10 @@ pushd ./runner
 # Unattend config
 #---------------------------------------
 runner_url="https://github.com/${runner_scope}"
+if [ -n "${ghe_hostname}" ]; then
+    runner_url="https://${ghe_hostname}/${runner_scope}"
+fi
+
 echo
 echo "Configuring ${runner_name} @ $runner_url"
 echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name"
@@ -127,9 +139,9 @@ sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNE
 echo
 echo "Configuring as a service ..."
 prefix=""
-if [ "${runner_plat}" == "linux" ]; then 
-    prefix="sudo "
-fi 
+if [ "${runner_plat}" == "linux" ]; then
+prefix="sudo "
+fi
 
 ${prefix}./svc.sh install ${svc_user}
 ${prefix}./svc.sh start

src/Runner.Common/Constants.cs

@@ -137,6 +137,9 @@ namespace GitHub.Runner.Common
                 public const int RunnerUpdating = 3;
                 public const int RunOnceRunnerUpdating = 4;
             }
+
+            public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry";
+            public static readonly string WorkerCrash = "WORKER_CRASH";
         }
 
         public static class RunnerEvent

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -160,11 +160,8 @@ namespace GitHub.Runner.Listener.Configuration
             }
 
             TaskAgent agent;
-            int attemptCount = 5;
-
             while (true)
             {
-                attemptCount--;
                 runnerSettings.AgentName = command.GetRunnerName();
 
                 _term.WriteLine();
@@ -195,10 +192,10 @@ namespace GitHub.Runner.Listener.Configuration
                             _term.WriteError("Failed to replace the runner.  Try again or ctrl-c to quit");
                         }
                     }
-                    else if (command.Unattended || attemptCount <= 0)
+                    else if (command.Unattended)
                     {
                         // if not replace and it is unattended config.
-                        throw new TaskAgentExistsException($"A runner already exists with the name {runnerSettings.AgentName}.");
+                        throw new TaskAgentExistsException($"Pool {runnerSettings.PoolId} already contains a runner with name {runnerSettings.AgentName}.");
                     }
                 }
                 else
@@ -219,7 +216,6 @@ namespace GitHub.Runner.Listener.Configuration
                     }
                 }
             }
-
             // Add Agent Id to settings
             runnerSettings.AgentId = agent.Id;
 

src/Runner.Listener/JobDispatcher.cs

@@ -858,7 +858,6 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        // TODO: We need send detailInfo back to DT in order to add an issue for the job
         private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null)
         {
             Trace.Entering();
@@ -952,8 +951,10 @@ namespace GitHub.Runner.Listener
                 ArgUtil.NotNull(timeline, nameof(timeline));
                 TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job");
                 ArgUtil.NotNull(jobRecord, nameof(jobRecord));
+                var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = errorMessage };
+                unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
                 jobRecord.ErrorCount++;
-                jobRecord.Issues.Add(new Issue() { Type = IssueType.Error, Message = errorMessage });
+                jobRecord.Issues.Add(unhandledExceptionIssue);
                 await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
             }
             catch (Exception ex)

src/Runner.Listener/MessageListener.cs

@@ -150,6 +150,20 @@ namespace GitHub.Runner.Listener
                     Trace.Error("Catch exception during create session.");
                     Trace.Error(ex);
 
+                    if (ex is VssOAuthTokenRequestException && creds.Federated is VssOAuthCredential vssOAuthCred)
+                    {
+                        // Check whether we get 401 because the runner registration already removed by the service.
+                        // If the runner registration get deleted, we can't exchange oauth token.
+                        Trace.Error("Test oauth app registration.");
+                        var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrl));
+                        var authError = await oauthTokenProvider.ValidateCredentialAsync(token);
+                        if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase))
+                        {
+                            _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure.");
+                            return false;
+                        }
+                    }
+
                     if (!IsSessionCreationExceptionRetriable(ex))
                     {
                         if (_useMigratedCredentials)

src/Runner.Worker/ActionCommandManager.cs

@@ -486,7 +486,10 @@ namespace GitHub.Runner.Worker
 
             foreach (var property in command.Properties)
             {
-                issue.Data[property.Key] = property.Value;
+                if (!string.Equals(property.Key, Constants.Runner.InternalTelemetryIssueDataKey, StringComparison.OrdinalIgnoreCase))
+                {
+                    issue.Data[property.Key] = property.Value;
+                }
             }
 
             context.AddIssue(issue);

src/Runner.Worker/ActionManager.cs

@@ -70,8 +70,8 @@ namespace GitHub.Runner.Worker
                 executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets");
             }
 
-            // Clear the cache (for self-hosted runners)
-            IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
+            // // Clear the cache (for self-hosted runners)
+            // IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
 
             foreach (var action in actions)
             {
@@ -516,9 +516,9 @@ namespace GitHub.Runner.Worker
                     // Example:  https://my-ghes/api/v3/repos/my-org/my-action/tarball/v1
                     BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref),
 
-                    // A community action, synced to their GHES instance
-                    // Example:  https://my-ghes/api/v3/repos/actions-community/some-org-some-action/tarball/v1
-                    BuildLinkToActionArchive(apiUrl, $"actions-community/{repositoryReference.Name.Replace("/", "-")}", repositoryReference.Ref)
+                    // // A community action, synced to their GHES instance
+                    // // Example:  https://my-ghes/api/v3/repos/actions-community/some-org-some-action/tarball/v1
+                    // BuildLinkToActionArchive(apiUrl, $"actions-community/{repositoryReference.Name.Replace("/", "-")}", repositoryReference.Ref)
                 };
 
                 foreach (var archiveLink in archiveLinks)
@@ -553,9 +553,9 @@ namespace GitHub.Runner.Worker
         private static string BuildLinkToActionArchive(string apiUrl, string repository, string @ref)
         {
 #if OS_WINDOWS
-            return $"{apiUrl}/repos/{repository}/zipball/{@ref}";
+            return $"https://github.com/{repository}/zipball/{@ref}";
 #else
-            return $"{apiUrl}/repos/{repository}/tarball/{@ref}";
+            return $"https://github.com/{repository}/tarball/{@ref}";
 #endif
         }
 
@@ -590,25 +590,25 @@ namespace GitHub.Runner.Worker
                             using (var httpClientHandler = HostContext.CreateHttpClientHandler())
                             using (var httpClient = new HttpClient(httpClientHandler))
                             {
-                                var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
-                                if (string.IsNullOrEmpty(authToken))
-                                {
-                                    // TODO: Deprecate the PREVIEW_ACTION_TOKEN
-                                    authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
-                                }
+                                // var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
+                                // if (string.IsNullOrEmpty(authToken))
+                                // {
+                                //     // TODO: Deprecate the PREVIEW_ACTION_TOKEN
+                                //     authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
+                                // }
 
-                                if (!string.IsNullOrEmpty(authToken))
-                                {
-                                    HostContext.SecretMasker.AddValue(authToken);
-                                    var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
-                                    httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
-                                }
-                                else
-                                {
-                                    var accessToken = executionContext.GetGitHubContext("token");
-                                    var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
-                                    httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
-                                }
+                                // if (!string.IsNullOrEmpty(authToken))
+                                // {
+                                //     HostContext.SecretMasker.AddValue(authToken);
+                                //     var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
+                                //     httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
+                                // }
+                                // else
+                                // {
+                                //     var accessToken = executionContext.GetGitHubContext("token");
+                                //     var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
+                                //     httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
+                                // }
 
                                 httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
                                 using (var response = await httpClient.GetAsync(link))

src/Runner.Worker/GitHubContext.cs

@@ -10,10 +10,11 @@ namespace GitHub.Runner.Worker
         {
             "action",
             "actor",
-            "api_url", // temp for GHES alpha release
+            "api_url",
             "base_ref",
             "event_name",
             "event_path",
+            "graphql_url",
             "head_ref",
             "job",
             "ref",
@@ -22,7 +23,7 @@ namespace GitHub.Runner.Worker
             "run_id",
             "run_number",
             "sha",
-            "url", // temp for GHES alpha release
+            "url",
             "workflow",
             "workspace",
         };

src/Runner.Worker/JobExtension.cs

@@ -131,12 +131,13 @@ namespace GitHub.Runner.Worker
                     // Temporary hack for GHES alpha
                     var configurationStore = HostContext.GetService<IConfigurationStore>();
                     var runnerSettings = configurationStore.GetSettings();
-                    if (!runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
+                    if (string.IsNullOrEmpty(context.GetGitHubContext("url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
                     {
                         var url = new Uri(runnerSettings.GitHubUrl);
                         var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
                         context.SetGitHubContext("url", $"{url.Scheme}://{url.Host}{portInfo}");
                         context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
+                        context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql");
                     }
 
                     // Evaluate the job-level environment variables

src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenProvider.cs

@@ -119,6 +119,15 @@ namespace GitHub.Services.OAuth
             }
         }
 
+        public async Task<string> ValidateCredentialAsync(CancellationToken cancellationToken)
+        {
+            var tokenHttpClient = new VssOAuthTokenHttpClient(this.SignInUrl);
+            var tokenResponse = await tokenHttpClient.GetTokenAsync(this.Grant, this.ClientCredential, this.TokenParameters, cancellationToken);
+
+            // return the underlying authentication error
+            return tokenResponse.Error;
+        }
+
         /// <summary>
         /// Issues a token request to the configured secure token service. On success, the access token issued by the 
         /// token service is returned to the caller
@@ -131,7 +140,7 @@ namespace GitHub.Services.OAuth
             CancellationToken cancellationToken)
         {
             if (this.SignInUrl == null ||
-                this.Grant == null || 
+                this.Grant == null ||
                 this.ClientCredential == null)
             {
                 return null;