Enable hostcontext to track auth migration. (#3776)

src/Runner.Common/AuthMigration.cs

@@ -0,0 +1,13 @@
+using System;
+
+namespace GitHub.Runner.Common
+{
+    public class AuthMigrationEventArgs : EventArgs
+    {
+        public AuthMigrationEventArgs(string trace)
+        {
+            Trace = trace;
+        }
+        public string Trace { get; private set; }
+    }
+}

src/Runner.Common/HostContext.cs

@@ -37,6 +37,11 @@ namespace GitHub.Runner.Common
         void ShutdownRunner(ShutdownReason reason);
         void WritePerfCounter(string counter);
         void LoadDefaultUserAgents();
+
+        bool AllowAuthMigration { get; }
+        void EnableAuthMigration(string trace);
+        void DeferAuthMigration(TimeSpan deferred, string trace);
+        event EventHandler<AuthMigrationEventArgs> AuthMigrationChanged;
     }
 
     public enum StartupType
@@ -70,12 +75,21 @@ namespace GitHub.Runner.Common
         private RunnerWebProxy _webProxy = new();
         private string _hostType = string.Empty;
 
+        // disable auth migration by default
+        private readonly ManualResetEventSlim _allowAuthMigration = new ManualResetEventSlim(false);
+        private DateTime _deferredAuthMigrationTime = DateTime.MaxValue;
+        private readonly object _authMigrationLock = new object();
+        private CancellationTokenSource _authMigrationAutoReenableTaskCancellationTokenSource = new();
+        private Task _authMigrationAutoReenableTask;
+
         public event EventHandler Unloading;
+        public event EventHandler<AuthMigrationEventArgs> AuthMigrationChanged;
         public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token;
         public ShutdownReason RunnerShutdownReason { get; private set; }
         public ISecretMasker SecretMasker => _secretMasker;
         public List<ProductInfoHeaderValue> UserAgents => _userAgents;
         public RunnerWebProxy WebProxy => _webProxy;
+        public bool AllowAuthMigration => _allowAuthMigration.IsSet;
         public HostContext(string hostType, string logFile = null)
         {
             // Validate args.
@@ -207,6 +221,71 @@ namespace GitHub.Runner.Common
             LoadDefaultUserAgents();
         }
 
+        // marked as internal for testing
+        internal async Task AuthMigrationAuthReenableAsync(TimeSpan refreshInterval, CancellationToken token)
+        {
+            try
+            {
+                while (!token.IsCancellationRequested)
+                {
+                    _trace.Verbose($"Auth migration defer timer is set to expire at {_deferredAuthMigrationTime.ToString("O")}. AllowAuthMigration: {_allowAuthMigration.IsSet}.");
+                    await Task.Delay(refreshInterval, token);
+                    if (!_allowAuthMigration.IsSet && DateTime.UtcNow > _deferredAuthMigrationTime)
+                    {
+                        _trace.Info($"Auth migration defer timer expired. Allowing auth migration.");
+                        EnableAuthMigration("Auth migration defer timer expired.");
+                    }
+                }
+            }
+            catch (TaskCanceledException)
+            {
+                // Task was cancelled, exit the loop.
+            }
+            catch (Exception ex)
+            {
+                _trace.Info("Error in auth migration reenable task.");
+                _trace.Error(ex);
+            }
+        }
+
+        public void EnableAuthMigration(string trace)
+        {
+            _allowAuthMigration.Set();
+
+            lock (_authMigrationLock)
+            {
+                if (_authMigrationAutoReenableTask == null)
+                {
+                    var refreshIntervalInMS = 60 * 1000;
+#if DEBUG
+                    // For L0, we will refresh faster
+                    if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL")))
+                    {
+                        refreshIntervalInMS = int.Parse(Environment.GetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL"));
+                    }
+#endif
+                    _authMigrationAutoReenableTask = AuthMigrationAuthReenableAsync(TimeSpan.FromMilliseconds(refreshIntervalInMS), _authMigrationAutoReenableTaskCancellationTokenSource.Token);
+                }
+            }
+
+            _trace.Info($"Enable auth migration at {_deferredAuthMigrationTime.ToString("O")}.");
+            AuthMigrationChanged?.Invoke(this, new AuthMigrationEventArgs(trace));
+        }
+
+        public void DeferAuthMigration(TimeSpan deferred, string trace)
+        {
+            _allowAuthMigration.Reset();
+
+            // defer migration for a while
+            lock (_authMigrationLock)
+            {
+                _deferredAuthMigrationTime = DateTime.UtcNow.Add(deferred);
+            }
+
+            _trace.Info($"Disabled auth migration until {_deferredAuthMigrationTime.ToString("O")}.");
+            AuthMigrationChanged?.Invoke(this, new AuthMigrationEventArgs(trace));
+        }
+
         public void LoadDefaultUserAgents()
         {
             if (string.IsNullOrEmpty(WebProxy.HttpProxyAddress) && string.IsNullOrEmpty(WebProxy.HttpsProxyAddress))
@@ -549,6 +628,18 @@ namespace GitHub.Runner.Common
                     _loadContext.Unloading -= LoadContext_Unloading;
                     _loadContext = null;
                 }
+
+                if (_authMigrationAutoReenableTask != null)
+                {
+                    _authMigrationAutoReenableTaskCancellationTokenSource?.Cancel();
+                }
+
+                if (_authMigrationAutoReenableTaskCancellationTokenSource != null)
+                {
+                    _authMigrationAutoReenableTaskCancellationTokenSource?.Dispose();
+                    _authMigrationAutoReenableTaskCancellationTokenSource = null;
+                }
+
                 _httpTraceSubscription?.Dispose();
                 _diagListenerSubscription?.Dispose();
                 _traceManager?.Dispose();

src/Test/L0/HostContextL0.cs

@@ -1,10 +1,10 @@
-using GitHub.Runner.Common.Util;
+using System;
-using System;
 using System.IO;
 using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
 using Xunit;
 
 namespace GitHub.Runner.Common.Tests
@@ -172,6 +172,133 @@ namespace GitHub.Runner.Common.Tests
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void AuthMigrationDisabledByDefault()
+        {
+            try
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", "100");
+
+                // Arrange.
+                Setup();
+
+                // Assert.
+                Assert.False(_hc.AllowAuthMigration);
+
+                // Change migration state is error free.
+                _hc.EnableAuthMigration("L0Test");
+                _hc.DeferAuthMigration(TimeSpan.FromHours(1), "L0Test");
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", null);
+                // Cleanup.
+                Teardown();
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public async Task AuthMigrationReenableTaskNotRunningByDefault()
+        {
+            try
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", "50");
+
+                // Arrange.
+                Setup();
+
+                // Assert.
+                Assert.False(_hc.AllowAuthMigration);
+                await Task.Delay(TimeSpan.FromMilliseconds(200));
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", null);
+                // Cleanup.
+                Teardown();
+            }
+
+            var logFile = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), $"trace_{nameof(HostContextL0)}_{nameof(AuthMigrationReenableTaskNotRunningByDefault)}.log");
+            var logContent = await File.ReadAllTextAsync(logFile);
+            Assert.Contains("HostContext", logContent);
+            Assert.DoesNotContain("Auth migration defer timer", logContent);
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void AuthMigrationEnableDisable()
+        {
+            try
+            {
+                // Arrange.
+                Setup();
+
+                var eventFiredCount = 0;
+                _hc.AuthMigrationChanged += (sender, e) =>
+                {
+                    eventFiredCount++;
+                    Assert.Equal("L0Test", e.Trace);
+                };
+
+                // Assert.
+                _hc.EnableAuthMigration("L0Test");
+                Assert.True(_hc.AllowAuthMigration);
+
+                _hc.DeferAuthMigration(TimeSpan.FromHours(1), "L0Test");
+                Assert.False(_hc.AllowAuthMigration);
+                Assert.Equal(2, eventFiredCount);
+            }
+            finally
+            {
+                // Cleanup.
+                Teardown();
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public async Task AuthMigrationAutoReset()
+        {
+            try
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", "100");
+
+                // Arrange.
+                Setup();
+
+                var eventFiredCount = 0;
+                _hc.AuthMigrationChanged += (sender, e) =>
+                {
+                    eventFiredCount++;
+                    Assert.NotEmpty(e.Trace);
+                };
+
+                // Assert.
+                _hc.EnableAuthMigration("L0Test");
+                Assert.True(_hc.AllowAuthMigration);
+
+                _hc.DeferAuthMigration(TimeSpan.FromMilliseconds(500), "L0Test");
+                Assert.False(_hc.AllowAuthMigration);
+
+                await Task.Delay(TimeSpan.FromSeconds(1));
+                Assert.True(_hc.AllowAuthMigration);
+                Assert.Equal(3, eventFiredCount);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("_GITHUB_ACTION_AUTH_MIGRATION_REFRESH_INTERVAL", null);
+
+                // Cleanup.
+                Teardown();
+            }
+        }
+
         private void Setup([CallerMemberName] string testName = "")
         {
             _tokenSource = new CancellationTokenSource();

src/Test/L0/TestHostContext.cs

@@ -1,16 +1,15 @@
-using GitHub.Runner.Common.Util;
+using System;
-using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Net.Http.Headers;
+using System.Reflection;
 using System.Runtime.CompilerServices;
+using System.Runtime.Loader;
 using System.Threading;
 using System.Threading.Tasks;
-using System.Runtime.Loader;
-using System.Reflection;
-using System.Collections.Generic;
 using GitHub.DistributedTask.Logging;
-using System.Net.Http.Headers;
 using GitHub.Runner.Sdk;
 
 namespace GitHub.Runner.Common.Tests
@@ -31,6 +30,7 @@ namespace GitHub.Runner.Common.Tests
         private StartupType _startupType;
         public event EventHandler Unloading;
         public event EventHandler<DelayEventArgs> Delaying;
+        public event EventHandler<AuthMigrationEventArgs> AuthMigrationChanged;
         public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token;
         public ShutdownReason RunnerShutdownReason { get; private set; }
         public ISecretMasker SecretMasker => _secretMasker;
@@ -92,6 +92,8 @@ namespace GitHub.Runner.Common.Tests
 
         public RunnerWebProxy WebProxy => new();
 
+        public bool AllowAuthMigration { get; set; }
+
         public async Task Delay(TimeSpan delay, CancellationToken token)
         {
             // Event callback
@@ -387,6 +389,18 @@ namespace GitHub.Runner.Common.Tests
         {
             return;
         }
+
+        public void EnableAuthMigration(string trace)
+        {
+            AllowAuthMigration = true;
+            AuthMigrationChanged?.Invoke(this, new AuthMigrationEventArgs(trace));
+        }
+
+        public void DeferAuthMigration(TimeSpan deferred, string trace)
+        {
+            AllowAuthMigration = false;
+            AuthMigrationChanged?.Invoke(this, new AuthMigrationEventArgs(trace));
+        }
     }
 
     public class DelayEventArgs : EventArgs