Add option in OAuthCred to load authUrlV2. (#3777)

src/Runner.Listener/BrokerMessageListener.cs

@@ -65,7 +65,7 @@ namespace GitHub.Runner.Listener
 
             // Create connection.
             Trace.Info("Loading Credentials");
-            _creds = _credMgr.LoadCredentials();
+            _creds = _credMgr.LoadCredentials(allowAuthUrlV2: false);
 
             var agent = new TaskAgentReference
             {
@@ -434,7 +434,7 @@ namespace GitHub.Runner.Listener
         private async Task RefreshBrokerConnectionAsync()
         {
             Trace.Info("Reload credentials.");
-            _creds = _credMgr.LoadCredentials();
+            _creds = _credMgr.LoadCredentials(allowAuthUrlV2: false); // TODO: change to `true` in the next PR.
             await _brokerServer.ConnectAsync(new Uri(_settings.ServerUrlV2), _creds);
             Trace.Info("Connection to Broker Server recreated.");
         }

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -127,7 +127,7 @@ namespace GitHub.Runner.Listener.Configuration
                     runnerSettings.ServerUrl = inputUrl;
                     // Get the credentials
                     credProvider = GetCredentialProvider(command, runnerSettings.ServerUrl);
-                    creds = credProvider.GetVssCredentials(HostContext);
+                    creds = credProvider.GetVssCredentials(HostContext, allowAuthUrlV2: false);
                     Trace.Info("legacy vss cred retrieved");
                 }
                 else
@@ -384,7 +384,7 @@ namespace GitHub.Runner.Listener.Configuration
             if (!runnerSettings.UseV2Flow)
             {
                 var credMgr = HostContext.GetService<ICredentialManager>();
-                VssCredentials credential = credMgr.LoadCredentials();
+                VssCredentials credential = credMgr.LoadCredentials(allowAuthUrlV2: false);
                 try
                 {
                     await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), credential);
@@ -519,7 +519,7 @@ namespace GitHub.Runner.Listener.Configuration
                     if (string.IsNullOrEmpty(settings.GitHubUrl))
                     {
                         var credProvider = GetCredentialProvider(command, settings.ServerUrl);
-                        creds = credProvider.GetVssCredentials(HostContext);
+                        creds = credProvider.GetVssCredentials(HostContext, allowAuthUrlV2: false);
                         Trace.Info("legacy vss cred retrieved");
                     }
                     else

src/Runner.Listener/Configuration/CredentialManager.cs

@@ -13,7 +13,7 @@ namespace GitHub.Runner.Listener.Configuration
     public interface ICredentialManager : IRunnerService
     {
         ICredentialProvider GetCredentialProvider(string credType);
-        VssCredentials LoadCredentials();
+        VssCredentials LoadCredentials(bool allowAuthUrlV2);
     }
 
     public class CredentialManager : RunnerService, ICredentialManager
@@ -40,7 +40,7 @@ namespace GitHub.Runner.Listener.Configuration
             return creds;
         }
 
-        public VssCredentials LoadCredentials()
+        public VssCredentials LoadCredentials(bool allowAuthUrlV2)
         {
             IConfigurationStore store = HostContext.GetService<IConfigurationStore>();
 
@@ -51,21 +51,16 @@ namespace GitHub.Runner.Listener.Configuration
 
             CredentialData credData = store.GetCredentials();
             var migratedCred = store.GetMigratedCredentials();
-            if (migratedCred != null)
+            if (migratedCred != null &&
+                migratedCred.Scheme == Constants.Configuration.OAuth)
             {
                 credData = migratedCred;
-
-                // Re-write .credentials with Token URL
-                store.SaveCredential(credData);
-
-                // Delete .credentials_migrated
-                store.DeleteMigratedCredential();
             }
 
             ICredentialProvider credProv = GetCredentialProvider(credData.Scheme);
             credProv.CredentialData = credData;
 
-            VssCredentials creds = credProv.GetVssCredentials(HostContext);
+            VssCredentials creds = credProv.GetVssCredentials(HostContext, allowAuthUrlV2);
 
             return creds;
         }

src/Runner.Listener/Configuration/CredentialProvider.cs

@@ -1,7 +1,7 @@
 using System;
-using GitHub.Services.Common;
 using GitHub.Runner.Common;
 using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
 using GitHub.Services.OAuth;
 
 namespace GitHub.Runner.Listener.Configuration
@@ -10,7 +10,7 @@ namespace GitHub.Runner.Listener.Configuration
     {
         Boolean RequireInteractive { get; }
         CredentialData CredentialData { get; set; }
-        VssCredentials GetVssCredentials(IHostContext context);
+        VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2);
         void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl);
     }
 
@@ -25,7 +25,7 @@ namespace GitHub.Runner.Listener.Configuration
         public virtual Boolean RequireInteractive => false;
         public CredentialData CredentialData { get; set; }
 
-        public abstract VssCredentials GetVssCredentials(IHostContext context);
+        public abstract VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2);
         public abstract void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl);
     }
 
@@ -33,7 +33,7 @@ namespace GitHub.Runner.Listener.Configuration
     {
         public OAuthAccessTokenCredential() : base(Constants.Configuration.OAuthAccessToken) { }
 
-        public override VssCredentials GetVssCredentials(IHostContext context)
+        public override VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2)
         {
             ArgUtil.NotNull(context, nameof(context));
             Tracing trace = context.GetTrace(nameof(OAuthAccessTokenCredential));

src/Runner.Listener/Configuration/OAuthCredential.cs

@@ -22,10 +22,18 @@ namespace GitHub.Runner.Listener.Configuration
             // Nothing to verify here
         }
 
-        public override VssCredentials GetVssCredentials(IHostContext context)
+        public override VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2)
         {
             var clientId = this.CredentialData.Data.GetValueOrDefault("clientId", null);
             var authorizationUrl = this.CredentialData.Data.GetValueOrDefault("authorizationUrl", null);
+            var authorizationUrlV2 = this.CredentialData.Data.GetValueOrDefault("authorizationUrlV2", null);
+
+            if (allowAuthUrlV2 &&
+                !string.IsNullOrEmpty(authorizationUrlV2) &&
+                context.AllowAuthMigration)
+            {
+                authorizationUrl = authorizationUrlV2;
+            }
 
             // For back compat with .credential file that doesn't has 'oauthEndpointUrl' section
             var oauthEndpointUrl = this.CredentialData.Data.GetValueOrDefault("oauthEndpointUrl", authorizationUrl);

src/Runner.Listener/MessageListener.cs

@@ -80,7 +80,7 @@ namespace GitHub.Runner.Listener
 
             // Create connection.
             Trace.Info("Loading Credentials");
-            _creds = _credMgr.LoadCredentials();
+            _creds = _credMgr.LoadCredentials(allowAuthUrlV2: false);
 
             var agent = new TaskAgentReference
             {
@@ -415,6 +415,7 @@ namespace GitHub.Runner.Listener
         public async Task RefreshListenerTokenAsync()
         {
             await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
+            _creds = _credMgr.LoadCredentials(allowAuthUrlV2: false); // TODO: change to `true` in next PR
             await _brokerServer.ForceRefreshConnection(_creds);
         }
 

src/Runner.Listener/Runner.cs

@@ -570,7 +570,7 @@ namespace GitHub.Runner.Listener
 
                                     // Create connection
                                     var credMgr = HostContext.GetService<ICredentialManager>();
-                                    var creds = credMgr.LoadCredentials();
+                                    var creds = credMgr.LoadCredentials(allowAuthUrlV2: false);
 
                                     if (string.IsNullOrEmpty(messageRef.RunServiceUrl))
                                     {

src/Runner.Listener/RunnerConfigUpdater.cs

@@ -197,6 +197,16 @@ namespace GitHub.Runner.Listener
                     await ReportTelemetryAsync($"Credential clientId in refreshed config '{refreshedClientId ?? "Empty"}' does not match the current credential clientId '{clientId}'.");
                     return;
                 }
+
+                //  make sure the credential authorizationUrl in the refreshed config match the current credential authorizationUrl for OAuth auth scheme
+                var authorizationUrl = _credData.Data.GetValueOrDefault("authorizationUrl", null);
+                var refreshedAuthorizationUrl = refreshedCredConfig.Data.GetValueOrDefault("authorizationUrl", null);
+                if (authorizationUrl != refreshedAuthorizationUrl)
+                {
+                    Trace.Error($"Credential authorizationUrl in refreshed config '{refreshedAuthorizationUrl ?? "Empty"}' does not match the current credential authorizationUrl '{authorizationUrl}'.");
+                    await ReportTelemetryAsync($"Credential authorizationUrl in refreshed config '{refreshedAuthorizationUrl ?? "Empty"}' does not match the current credential authorizationUrl '{authorizationUrl}'.");
+                    return;
+                }
             }
 
             // save the refreshed runner credentials as a separate file

src/Test/L0/Listener/BrokerMessageListenerL0.cs

@@ -50,7 +50,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 

src/Test/L0/Listener/Configuration/RunnerCredentialL0.cs

@@ -1,14 +1,18 @@
-using GitHub.Runner.Listener;
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using GitHub.Runner.Listener;
 using GitHub.Runner.Listener.Configuration;
 using GitHub.Services.Common;
 using GitHub.Services.OAuth;
+using Moq;
+using Xunit;
 
 namespace GitHub.Runner.Common.Tests.Listener.Configuration
 {
     public class TestRunnerCredential : CredentialProvider
     {
         public TestRunnerCredential() : base("TEST") { }
-        public override VssCredentials GetVssCredentials(IHostContext context)
+        public override VssCredentials GetVssCredentials(IHostContext context, bool allowAuthUrlV2)
         {
             Tracing trace = context.GetTrace("OuthAccessToken");
             trace.Info("GetVssCredentials()");
@@ -23,4 +27,85 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration
         {
         }
     }
-}
+
+    public class OAuthCredentialTestsL0
+    {
+        private Mock<IRSAKeyManager> _rsaKeyManager = new Mock<IRSAKeyManager>();
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "OAuthCredential")]
+        public void NotUseAuthV2Url()
+        {
+            using (TestHostContext hc = new(this))
+            {
+                // Arrange.
+                var oauth = new OAuthCredential();
+                oauth.CredentialData = new CredentialData()
+                {
+                    Scheme = Constants.Configuration.OAuth
+                };
+                oauth.CredentialData.Data.Add("clientId", "someClientId");
+                oauth.CredentialData.Data.Add("authorizationUrl", "http://myserver/");
+                oauth.CredentialData.Data.Add("authorizationUrlV2", "http://myserverv2/");
+
+                _rsaKeyManager.Setup(x => x.GetKey()).Returns(RSA.Create(2048));
+                hc.SetSingleton<IRSAKeyManager>(_rsaKeyManager.Object);
+
+                // Act.
+                var cred = oauth.GetVssCredentials(hc, false); // not allow auth v2
+
+                var cred2 = oauth.GetVssCredentials(hc, true); // use auth v2 but hostcontext doesn't
+
+                hc.EnableAuthMigration("L0Test");
+                var cred3 = oauth.GetVssCredentials(hc, false); // not use auth v2 but hostcontext does
+
+                oauth.CredentialData.Data.Remove("authorizationUrlV2");
+                var cred4 = oauth.GetVssCredentials(hc, true); // v2 url is not there
+
+                // Assert.
+                Assert.Equal("http://myserver/", (cred.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred.Federated as VssOAuthCredential).ClientCredential.ClientId);
+
+                Assert.Equal("http://myserver/", (cred2.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred2.Federated as VssOAuthCredential).ClientCredential.ClientId);
+
+                Assert.Equal("http://myserver/", (cred3.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred3.Federated as VssOAuthCredential).ClientCredential.ClientId);
+
+                Assert.Equal("http://myserver/", (cred4.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred4.Federated as VssOAuthCredential).ClientCredential.ClientId);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "OAuthCredential")]
+        public void UseAuthV2Url()
+        {
+            using (TestHostContext hc = new(this))
+            {
+                // Arrange.
+                var oauth = new OAuthCredential();
+                oauth.CredentialData = new CredentialData()
+                {
+                    Scheme = Constants.Configuration.OAuth
+                };
+                oauth.CredentialData.Data.Add("clientId", "someClientId");
+                oauth.CredentialData.Data.Add("authorizationUrl", "http://myserver/");
+                oauth.CredentialData.Data.Add("authorizationUrlV2", "http://myserverv2/");
+
+                _rsaKeyManager.Setup(x => x.GetKey()).Returns(RSA.Create(2048));
+                hc.SetSingleton<IRSAKeyManager>(_rsaKeyManager.Object);
+
+                // Act.
+                hc.EnableAuthMigration("L0Test");
+                var cred = oauth.GetVssCredentials(hc, true);
+
+                // Assert.
+                Assert.Equal("http://myserverv2/", (cred.Federated as VssOAuthCredential).AuthorizationUrl.AbsoluteUri);
+                Assert.Equal("someClientId", (cred.Federated as VssOAuthCredential).ClientCredential.ClientId);
+            }
+        }
+    }
+}

src/Test/L0/Listener/MessageListenerL0.cs

@@ -67,7 +67,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -127,7 +127,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedBrokerSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -177,7 +177,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -237,7 +237,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedBrokerSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -301,7 +301,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -382,7 +382,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 
@@ -484,7 +484,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
 
                 var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth };
                 originalCred.Data["authorizationUrl"] = "https://s.server";
@@ -533,7 +533,7 @@ namespace GitHub.Runner.Common.Tests.Listener
                         tokenSource.Token))
                     .Returns(Task.FromResult(expectedSession));
 
-                _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials());
+                _credMgr.Setup(x => x.LoadCredentials(It.IsAny<bool>())).Returns(new VssCredentials());
                 _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken });
                 _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData));
 

src/Test/L0/Listener/RunnerConfigUpdaterTests.cs

@@ -1,13 +1,13 @@
 using System;
+using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
-using GitHub.Runner.Listener;
 using GitHub.Runner.Common;
+using GitHub.Runner.Common.Tests;
+using GitHub.Runner.Listener;
 using GitHub.Runner.Sdk;
 using Moq;
 using Xunit;
-using System.Threading;
-using GitHub.Runner.Common.Tests;
-using System.Text;
 
 namespace GitHub.Runner.Tests.Listener
 {
@@ -510,6 +510,56 @@ namespace GitHub.Runner.Tests.Listener
             }
         }
 
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async Task UpdateRunnerConfigAsync_RefreshOAuthCredentialsWithDifferentAuthUrl_ShouldReportTelemetry()
+        {
+            using (var hc = new TestHostContext(this))
+            {
+                hc.SetSingleton<IConfigurationStore>(_configurationStore.Object);
+                hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+
+                // Arrange
+                var setting = new RunnerSettings { AgentId = 1, AgentName = "agent1" };
+                _configurationStore.Setup(x => x.GetSettings()).Returns(setting);
+                var credData = new CredentialData
+                {
+                    Scheme = "OAuth"
+                };
+                credData.Data.Add("clientId", "12345");
+                credData.Data.Add("authorizationUrl", "http://example.com/");
+                _configurationStore.Setup(x => x.GetCredentials()).Returns(credData);
+
+                IOUtil.SaveObject(setting, hc.GetConfigFile(WellKnownConfigFile.Runner));
+                IOUtil.SaveObject(credData, hc.GetConfigFile(WellKnownConfigFile.Credentials));
+
+                var differentCredData = new CredentialData
+                {
+                    Scheme = "OAuth"
+                };
+                differentCredData.Data.Add("clientId", "12345");
+                differentCredData.Data.Add("authorizationUrl", "http://example2.com/");
+                var encodedConfig = Convert.ToBase64String(Encoding.UTF8.GetBytes(StringUtil.ConvertToJson(differentCredData)));
+                _runnerServer.Setup(x => x.RefreshRunnerConfigAsync(It.IsAny<int>(), It.Is<string>(s => s == "credentials"), It.IsAny<string>(), It.IsAny<CancellationToken>())).ReturnsAsync(encodedConfig);
+
+                var _runnerConfigUpdater = new RunnerConfigUpdater();
+                _runnerConfigUpdater.Initialize(hc);
+
+                var validRunnerQualifiedId = "valid/runner/qualifiedid/1";
+                var configType = "credentials";
+                var serviceType = "pipelines";
+                var configRefreshUrl = "http://example.com";
+
+                // Act
+                await _runnerConfigUpdater.UpdateRunnerConfigAsync(validRunnerQualifiedId, configType, serviceType, configRefreshUrl);
+
+                // Assert
+                _runnerServer.Verify(x => x.UpdateAgentUpdateStateAsync(It.IsAny<int>(), It.IsAny<ulong>(), It.IsAny<string>(), It.Is<string>(s => s.Contains("Credential authorizationUrl in refreshed config")), It.IsAny<CancellationToken>()), Times.Once);
+                _configurationStore.Verify(x => x.SaveMigratedCredential(It.IsAny<CredentialData>()), Times.Never);
+            }
+        }
+
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Runner")]