Change the auth challenge 401 to be verbose trace. (#2021)

* fix: GITHUB_ENV in composite

* fix L0 Test

docs/checks/actions.md

@@ -15,7 +15,7 @@ Make sure the runner has access to actions service for GitHub.com or GitHub Ente
     ```
     curl -v https://api.github.com/api/v3/zen
     curl -v https://vstoken.actions.githubusercontent.com/_apis/health
-    curl -v https://pipelines.actions.githubusercontent/_apis/health
+    curl -v https://pipelines.actions.githubusercontent.com/_apis/health
     ```
 
 - For GitHub Enterprise Server

docs/checks/git.md

@@ -20,11 +20,30 @@ The test also set environment variable `GIT_TRACE=1` and `GIT_CURL_VERBOSE=1` be
 
 ## How to fix the issue?
 
-### 1. Check the common network issue
+### 1. Check global and system git config
+
+If you are having issues connecting to the server, check your global and system git config for any unexpected authentication headers. You might be seeing an error like:
+
+```
+fatal: unable to access 'https://github.com/actions/checkout/': The requested URL returned error: 400
+```
+
+The following commands can be used to check for unexpected authentication headers:
+
+```
+$ git config --global --list | grep extraheader
+http.extraheader=AUTHORIZATION: unexpected_auth_header
+
+$ git config --system --list | grep extraheader
+```
+
+The following command can be used to remove the above value: `git config --global --unset http.extraheader`
+
+### 2. Check the common network issue
   
   > Please check the [network doc](./network.md)
 
-### 2. SSL certificate related issue
+### 3. SSL certificate related issue
 
   If you are seeing `SSL Certificate problem:` in the log, it means the `git` can't connect to the GitHub server due to SSL handshake failure.
   > Please check the [SSL cert doc](./sslcert.md)

src/Runner.Common/Runner.Common.csproj

@@ -16,7 +16,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
     <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
     <PackageReference Include="System.Threading.Channels" Version="4.4.0" />

src/Runner.Listener/CommandSettings.cs

@@ -39,7 +39,6 @@ namespace GitHub.Runner.Listener
                     Constants.Runner.CommandLine.Flags.RunAsService,
                     Constants.Runner.CommandLine.Flags.Unattended,
                     Constants.Runner.CommandLine.Args.Auth,
-                    Constants.Runner.CommandLine.Args.JitConfig,
                     Constants.Runner.CommandLine.Args.Labels,
                     Constants.Runner.CommandLine.Args.MonitorSocketAddress,
                     Constants.Runner.CommandLine.Args.Name,
@@ -64,6 +63,7 @@ namespace GitHub.Runner.Listener
                 new string[]
                 {
                     Constants.Runner.CommandLine.Flags.Once,
+                    Constants.Runner.CommandLine.Args.JitConfig,
                     Constants.Runner.CommandLine.Args.StartupType
                 },
             // valid warmup flags and args

src/Runner.Listener/Runner.Listener.csproj

@@ -19,7 +19,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="System.IO.FileSystem.AccessControl" Version="4.4.0" />
     <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
     <PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />

src/Runner.Listener/Runner.cs

@@ -627,7 +627,7 @@ Config Options:
  --labels string        Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}'
  --work string          Relative runner work directory (default {Constants.Path.WorkDirectory})
  --replace              Replace any existing runner with the same name (default false)
- --pat                  GitHub personal access token used for checking network connectivity when executing `.{separator}run.{ext} --check`
+ --pat                  GitHub personal access token with repo scope. Used for checking network connectivity when executing `.{separator}run.{ext} --check`
  --disableupdate        Disable self-hosted runner automatic update to the latest released version`
  --ephemeral            Configure the runner to only take one job and then let the service un-configure the runner after the job finishes (default false)");
 

src/Runner.Worker/ExecutionContext.cs

@@ -67,6 +67,8 @@ namespace GitHub.Runner.Worker
 
         bool IsEmbedded { get; }
 
+        List<string> StepEnvironmentOverrides { get; }
+
         ExecutionContext Root { get; }
 
         // Initialize
@@ -237,6 +239,8 @@ namespace GitHub.Runner.Worker
             }
         }
 
+        public List<string> StepEnvironmentOverrides { get; } = new List<string>();
+
         public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);

src/Runner.Worker/Handlers/CompositeActionHandler.cs

@@ -265,21 +265,27 @@ namespace GitHub.Runner.Worker.Handlers
                     var dict = envContextData as CaseSensitiveDictionaryContextData;
 #endif
                     foreach (var pair in dict)
+                    {
+                        // Skip global env, otherwise we merge an outdated global env
+                        if (ExecutionContext.StepEnvironmentOverrides.Contains(pair.Key))
                         {
                             envContext[pair.Key] = pair.Value;
                         }
                     }
+                }
 
                 try
                 {
                     if (step is IActionRunner actionStep)
                     {
                         // Evaluate and merge embedded-step env
+                        step.ExecutionContext.StepEnvironmentOverrides.AddRange(ExecutionContext.StepEnvironmentOverrides);
                         var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
                         var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, Common.Util.VarUtil.EnvironmentVariableKeyComparer);
                         foreach (var env in actionEnvironment)
                         {
                             envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
+                            step.ExecutionContext.StepEnvironmentOverrides.Add(env.Key);
                         }
                     }
                 }

src/Runner.Worker/StepsRunner.cs

@@ -112,6 +112,7 @@ namespace GitHub.Runner.Worker
                         foreach (var env in actionEnvironment)
                         {
                             envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
+                            step.ExecutionContext.StepEnvironmentOverrides.Add(env.Key);
                         }
                     }
                     catch (Exception ex)

src/Sdk/Common/Common/Diagnostics/VssHttpEventSource.cs

@@ -351,6 +351,18 @@ namespace GitHub.Services.Common.Diagnostics
             }
         }
 
+        [NonEvent]
+        public void AuthenticationFailedOnFirstRequest(
+            VssTraceActivity activity,
+            HttpResponseMessage response)
+        {
+            if (IsEnabled())
+            {
+                SetActivityId(activity);
+                WriteMessageEvent((Int32)response.StatusCode, response.Headers.ToString(), this.AuthenticationFailedOnFirstRequest);
+            }
+        }
+
         [NonEvent]
         public void IssuedTokenProviderCreated(
             VssTraceActivity activity,
@@ -813,7 +825,7 @@ namespace GitHub.Services.Common.Diagnostics
         [Event(31, Keywords = Keywords.Authentication, Level = EventLevel.Warning, Task = Tasks.Authentication, Opcode = EventOpcode.Info, Message = "Retrieving an AAD auth token took a long time ({0} seconds)")]
         public void AuthorizationDelayed(string timespan)
         {
-            if(IsEnabled(EventLevel.Warning, Keywords.Authentication))
+            if (IsEnabled(EventLevel.Warning, Keywords.Authentication))
             {
                 WriteEvent(31, timespan);
             }
@@ -828,6 +840,17 @@ namespace GitHub.Services.Common.Diagnostics
             }
         }
 
+        [Event(33, Keywords = Keywords.Authentication, Level = EventLevel.Verbose, Task = Tasks.HttpRequest, Message = "Authentication failed on first request with status code {0}.%n{1}")]
+        private void AuthenticationFailedOnFirstRequest(
+            Int32 statusCode,
+            String headers)
+        {
+            if (IsEnabled(EventLevel.Verbose, Keywords.Authentication))
+            {
+                WriteEvent(33, statusCode, headers);
+            }
+        }
+
         /// <summary>
         /// Sets the activity ID of the current thread.
         /// </summary>

src/Sdk/Common/Common/VssHttpMessageHandler.cs

@@ -251,7 +251,14 @@ namespace GitHub.Services.Common
 
                         // Invalidate the token and ensure that we have the correct token provider for the challenge
                         // which we just received
+                        if (retries < m_maxAuthRetries)
+                        {
                             VssHttpEventSource.Log.AuthenticationFailed(traceActivity, response);
+                        }
+                        else
+                        {
+                            VssHttpEventSource.Log.AuthenticationFailedOnFirstRequest(traceActivity, response);
+                        }
 
                         if (provider != null)
                         {

src/Sdk/PipelinesWebApi/UnknownEnumJsonConverter.cs

@@ -2,6 +2,7 @@
 using System.Linq;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Serialization;
 
 namespace GitHub.Actions.Pipelines.WebApi
 {
@@ -9,7 +10,7 @@ namespace GitHub.Actions.Pipelines.WebApi
     {
         public UnknownEnumJsonConverter()
         {
-            this.CamelCaseText = true;
+            this.NamingStrategy = new CamelCaseNamingStrategy();
         }
 
         public override bool CanConvert(Type objectType)

src/Sdk/Sdk.csproj

@@ -14,7 +14,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
-        <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
+        <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
         <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.4" />
         <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="5.2.1" />
         <PackageReference Include="System.Security.Cryptography.Cng" Version="4.4.0" />

src/Sdk/WebApi/WebApi/VssJsonMediaTypeFormatter.cs

@@ -84,7 +84,7 @@ namespace GitHub.Services.WebApi
             if (!enumsAsNumbers)
             {
                 // Serialze enums as camelCased string values
-                this.SerializerSettings.Converters.Add(new StringEnumConverter { CamelCaseText = true });
+                this.SerializerSettings.Converters.Add(new StringEnumConverter { NamingStrategy = new CamelCaseNamingStrategy() });
             }
 
             if (useMsDateFormat)

src/Test/L0/Worker/StepsRunnerL0.cs

@@ -622,6 +622,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                     _stepContext.SetOutcome("", stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
                     _stepContext.SetConclusion("", stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult());
                 });
+            stepContext.Setup(x => x.StepEnvironmentOverrides).Returns(new List<string>());
 
             stepContext.Setup(x => x.UpdateGlobalStepsContext()).Callback(() =>
             {