use token for broker and actions service

src/Runner.Common/RunServer.cs

@@ -0,0 +1,120 @@
+using GitHub.DistributedTask.WebApi;
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.Runner.Common.Util;
+using GitHub.Services.WebApi;
+using GitHub.Services.Common;
+using GitHub.Runner.Sdk;
+using GitHub.DistributedTask.Pipelines;
+
+namespace GitHub.Runner.Common
+{
+    [ServiceLocator(Default = typeof(RunServer))]
+    public interface IRunServer : IRunnerService
+    {
+        Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
+
+        Task<AgentJobRequestMessage> GetJobMessageAsync(Guid scopeId, Guid hostId, string planType, string planGroup, Guid planId, IList<InstanceRef> instanceRefsJson);
+    }
+
+    public sealed class RunServer : RunnerService, IRunServer
+    {
+        private bool _hasConnection;
+        private VssConnection _connection;
+        private TaskAgentHttpClient _taskAgentClient;
+
+        public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
+        {
+            // System.Console.WriteLine("RunServer.ConnectAsync");
+            _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            _taskAgentClient = _connection.GetClient<TaskAgentHttpClient>();
+            _hasConnection = true;
+        }
+
+        private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout)
+        {
+            // System.Console.WriteLine("EstablishVssConnection");
+            Trace.Info($"EstablishVssConnection");
+            Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout.");
+            int attemptCount = 5;
+            while (attemptCount-- > 0)
+            {
+                var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout);
+                try
+                {
+                    await connection.ConnectAsync();
+                    return connection;
+                }
+                catch (Exception ex) when (attemptCount > 0)
+                {
+                    Trace.Info($"Catch exception during connect. {attemptCount} attempt left.");
+                    Trace.Error(ex);
+
+                    await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None);
+                }
+            }
+
+            // should never reach here.
+            throw new InvalidOperationException(nameof(EstablishVssConnection));
+        }
+
+        private void CheckConnection()
+        {
+            if (!_hasConnection)
+            {
+                throw new InvalidOperationException($"SetConnection");
+            }
+        }
+
+        public Task<AgentJobRequestMessage> GetJobMessageAsync(
+            Guid scopeId,
+            Guid hostId,
+            string planType,
+            string planGroup,
+            Guid planId,
+            IList<InstanceRef> instanceRefsJson)
+        {
+            // System.Console.WriteLine("RunServer.GetMessageAsync");
+            CheckConnection();
+            return _taskAgentClient.GetJobMessageAsync(scopeId, hostId, planType, planGroup, planId, StringUtil.ConvertToJson(instanceRefsJson, Newtonsoft.Json.Formatting.None));
+        }
+    }
+
+    // todo: move to SDK?
+    [DataContract]
+    public sealed class MessageRef
+    {
+        [DataMember(Name = "url")]
+        public string Url { get; set; }
+        [DataMember(Name = "token")]
+        public string Token { get; set; }
+        [DataMember(Name = "scopeId")]
+        public Guid ScopeId { get; set; }
+        [DataMember(Name = "hostId")]
+        public Guid HostId { get; set; }
+        [DataMember(Name = "planType")]
+        public string PlanType { get; set; }
+        [DataMember(Name = "planGroup")]
+        public string PlanGroup { get; set; }
+        [DataMember(Name = "planId")]
+        public Guid PlanId { get; set; }
+        [DataMember(Name = "instanceRefs")]
+        public InstanceRef[] InstanceRefs { get; set; }
+        [DataMember(Name = "labels")]
+        public string[] Labels { get; set; }
+    }
+
+    [DataContract]
+    public sealed class InstanceRef
+    {
+        [DataMember(Name = "name")]
+        public string Name { get; set; }
+        [DataMember(Name = "instanceType")]
+        public string InstanceType { get; set; }
+        [DataMember(Name = "attempt")]
+        public int Attempt { get; set; }
+    }
+}

src/Runner.Common/RunnerServer.cs

@@ -44,7 +44,7 @@ namespace GitHub.Runner.Common
         // job request
         Task<TaskAgentJobRequest> GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken);
         Task<TaskAgentJobRequest> RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId, CancellationToken cancellationToken);
-        Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken);
+        Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, Guid targetHostId, CancellationToken cancellationToken);
 
         // agent package
         Task<List<PackageMetadata>> GetPackagesAsync(string packageType, string platform, int top, bool includeToken, CancellationToken cancellationToken);
@@ -68,11 +68,23 @@ namespace GitHub.Runner.Common
 
         public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
         {
-            var createGenericConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            // System.Console.WriteLine("RunnerServer.ConnectAsync: Create message connection");
             var createMessageConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
-            var createRequestConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+            await Task.WhenAll(createMessageConnection);
 
-            await Task.WhenAll(createGenericConnection, createMessageConnection, createRequestConnection);
+            // System.Console.WriteLine("RunnerServer.ConnectAsync: Create generic connection");
+            var createGenericConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            await Task.WhenAll(createGenericConnection);
+
+            // System.Console.WriteLine("RunnerServer.ConnectAsync: Create request connection");
+            var createRequestConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+            await Task.WhenAll(createRequestConnection);
+
+            // var createGenericConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
+            // var createMessageConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+            // var createRequestConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
+
+            // await Task.WhenAll(createGenericConnection, createMessageConnection, createRequestConnection);
 
             _genericConnection = await createGenericConnection;
             _messageConnection = await createMessageConnection;
@@ -182,6 +194,8 @@ namespace GitHub.Runner.Common
 
         private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout)
         {
+            // System.Console.WriteLine("EstablishVssConnection");
+            Trace.Info($"EstablishVssConnection");
             Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout.");
             int attemptCount = 5;
             while (attemptCount-- > 0)
@@ -238,41 +252,48 @@ namespace GitHub.Runner.Common
 
         public Task<List<TaskAgentPool>> GetAgentPoolsAsync(string agentPoolName = null, TaskAgentPoolType poolType = TaskAgentPoolType.Automation)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentPoolsAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetAgentPoolsAsync(agentPoolName, poolType: poolType);
         }
 
         public Task<TaskAgent> AddAgentAsync(Int32 agentPoolId, TaskAgent agent)
         {
+            // System.Console.WriteLine("RunnerServer.AddAgentAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.AddAgentAsync(agentPoolId, agent);
         }
 
         public Task<List<TaskAgent>> GetAgentsAsync(int agentPoolId, string agentName = null)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentsAsync 1");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetAgentsAsync(agentPoolId, agentName, false);
         }
 
         public Task<List<TaskAgent>> GetAgentsAsync(string agentName)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentsAsync 2");
             return GetAgentsAsync(0, agentName); // search in all all agentPools
         }
 
         public Task<TaskAgent> ReplaceAgentAsync(int agentPoolId, TaskAgent agent)
         {
+            // System.Console.WriteLine("RunnerServer.ReplaceAgentAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.ReplaceAgentAsync(agentPoolId, agent);
         }
 
         public Task DeleteAgentAsync(int agentPoolId, int agentId)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.DeleteAgentAsync(agentPoolId, agentId);
         }
 
         public Task DeleteAgentAsync(int agentId)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentAsync");
             return DeleteAgentAsync(0, agentId); // agentPool is ignored server side
         }
 
@@ -282,24 +303,28 @@ namespace GitHub.Runner.Common
 
         public Task<TaskAgentSession> CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.CreateAgentSessionAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.CreateAgentSessionAsync(poolId, session, cancellationToken: cancellationToken);
         }
 
         public Task DeleteAgentMessageAsync(Int32 poolId, Int64 messageId, Guid sessionId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentMessageAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.DeleteMessageAsync(poolId, messageId, sessionId, cancellationToken: cancellationToken);
         }
 
         public Task DeleteAgentSessionAsync(Int32 poolId, Guid sessionId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.DeleteAgentSessionAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.DeleteAgentSessionAsync(poolId, sessionId, cancellationToken: cancellationToken);
         }
 
         public Task<TaskAgentMessage> GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentMessageAsync");
             CheckConnection(RunnerConnectionType.MessageQueue);
             return _messageTaskAgentClient.GetMessageAsync(poolId, sessionId, lastMessageId, cancellationToken: cancellationToken);
         }
@@ -310,18 +335,21 @@ namespace GitHub.Runner.Common
 
         public Task<TaskAgentJobRequest> RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId = null, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("RunnerServer.RenewAgentRequestAsync");
             CheckConnection(RunnerConnectionType.JobRequest);
             return _requestTaskAgentClient.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId: orchestrationId, cancellationToken: cancellationToken);
         }
 
-        public Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken = default(CancellationToken))
+        public Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, Guid targetHostId, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("RunnerServer.FinishAgentRequestAsync");
             CheckConnection(RunnerConnectionType.JobRequest);
-            return _requestTaskAgentClient.FinishAgentRequestAsync(poolId, requestId, lockToken, finishTime, result, cancellationToken: cancellationToken);
+            return _requestTaskAgentClient.FinishAgentRequestAsync(poolId, requestId, lockToken, finishTime, result, targetHostId, cancellationToken: cancellationToken);
         }
 
         public Task<TaskAgentJobRequest> GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("RunnerServer.GetAgentRequestAsync");
             CheckConnection(RunnerConnectionType.JobRequest);
             return _requestTaskAgentClient.GetAgentRequestAsync(poolId, requestId, cancellationToken: cancellationToken);
         }
@@ -331,18 +359,21 @@ namespace GitHub.Runner.Common
         //-----------------------------------------------------------------
         public Task<List<PackageMetadata>> GetPackagesAsync(string packageType, string platform, int top, bool includeToken, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.GetPackagesAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetPackagesAsync(packageType, platform, top, includeToken, cancellationToken: cancellationToken);
         }
 
         public Task<PackageMetadata> GetPackageAsync(string packageType, string platform, string version, bool includeToken, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("RunnerServer.GetPackageAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.GetPackageAsync(packageType, platform, version, includeToken, cancellationToken: cancellationToken);
         }
 
         public Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, int agentId, string currentState, string trace)
         {
+            // System.Console.WriteLine("RunnerServer.UpdateAgentUpdateStateAsync");
             CheckConnection(RunnerConnectionType.Generic);
             return _genericTaskAgentClient.UpdateAgentUpdateStateAsync(agentPoolId, agentId, currentState, trace);
         }

src/Runner.Listener/JobDispatcher.cs

@@ -23,7 +23,7 @@ namespace GitHub.Runner.Listener
     {
         bool Busy { get; }
         TaskCompletionSource<bool> RunOnceJobCompleted { get; }
-        void Run(Pipelines.AgentJobRequestMessage message, bool runOnce = false);
+        void Run(Guid targetHostId, Pipelines.AgentJobRequestMessage message, bool runOnce = false);
         bool Cancel(JobCancelMessage message);
         Task WaitAsync(CancellationToken token);
         Task ShutdownAsync();
@@ -79,7 +79,7 @@ namespace GitHub.Runner.Listener
 
         public bool Busy { get; private set; }
 
-        public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce = false)
+        public void Run(Guid targetHostId, Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce = false)
         {
             Trace.Info($"Job request {jobRequestMessage.RequestId} for plan {jobRequestMessage.Plan.PlanId} job {jobRequestMessage.JobId} received.");
 
@@ -112,11 +112,11 @@ namespace GitHub.Runner.Listener
             if (runOnce)
             {
                 Trace.Info("Start dispatcher for one time used runner.");
-                newDispatch.WorkerDispatch = RunOnceAsync(jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
+                newDispatch.WorkerDispatch = RunOnceAsync(targetHostId, jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
             }
             else
             {
-                newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
+                newDispatch.WorkerDispatch = RunAsync(targetHostId, jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token);
             }
 
             _jobInfos.TryAdd(newDispatch.JobId, newDispatch);
@@ -317,11 +317,11 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task RunOnceAsync(Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
+        private async Task RunOnceAsync(Guid targetHostId, Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
         {
             try
             {
-                await RunAsync(message, orchestrationId, previousJobDispatch, jobRequestCancellationToken, workerCancelTimeoutKillToken);
+                await RunAsync(targetHostId, message, orchestrationId, previousJobDispatch, jobRequestCancellationToken, workerCancelTimeoutKillToken);
             }
             finally
             {
@@ -330,7 +330,7 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
+        private async Task RunAsync(Guid targetHostId, Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
         {
             Busy = true;
             try
@@ -383,7 +383,7 @@ namespace GitHub.Runner.Listener
                         await renewJobRequest;
 
                         // complete job request with result Cancelled
-                        await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
+                        await CompleteJobRequestAsync(targetHostId, _poolId, message, lockToken, TaskResult.Canceled);
                         return;
                     }
 
@@ -540,7 +540,7 @@ namespace GitHub.Runner.Listener
                                 await renewJobRequest;
 
                                 // complete job request
-                                await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
+                                await CompleteJobRequestAsync(targetHostId, _poolId, message, lockToken, result, detailInfo);
 
                                 // print out unhandled exception happened in worker after we complete job request.
                                 // when we run out of disk space, report back to server has higher priority.
@@ -637,7 +637,7 @@ namespace GitHub.Runner.Listener
                             await renewJobRequest;
 
                             // complete job request
-                            await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
+                            await CompleteJobRequestAsync(targetHostId, _poolId, message, lockToken, resultOnAbandonOrCancel);
                         }
                         finally
                         {
@@ -666,25 +666,25 @@ namespace GitHub.Runner.Listener
             {
                 try
                 {
-#if USE_BROKER
+// #if USE_BROKER
                     if (!firstJobRequestRenewed.Task.IsCompleted)
                     {
                         // fire first renew succeed event.
                         firstJobRequestRenewed.TrySetResult(0);
                     }
-#else
+// #else
-                    request = await runnerServer.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId, token);
+//                     request = await runnerServer.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId, token);
-                    Trace.Info($"Successfully renew job request {requestId}, job is valid till {request?.LockedUntil.Value}");
+//                     Trace.Info($"Successfully renew job request {requestId}, job is valid till {request?.LockedUntil.Value}");
 
-                    if (!firstJobRequestRenewed.Task.IsCompleted)
+//                     if (!firstJobRequestRenewed.Task.IsCompleted)
-                    {
+//                     {
-                        // fire first renew succeed event.
+//                         // fire first renew succeed event.
-                        firstJobRequestRenewed.TrySetResult(0);
+//                         firstJobRequestRenewed.TrySetResult(0);
 
-                        // Update settings if the runner name has been changed server-side
+//                         // Update settings if the runner name has been changed server-side
-                        UpdateAgentNameIfNeeded(request.ReservedAgent?.Name);
+//                         UpdateAgentNameIfNeeded(request.ReservedAgent?.Name);
-                    }
+//                     }
-#endif
+// #endif
 
                     if (encounteringError > 0)
                     {
@@ -919,7 +919,7 @@ namespace GitHub.Runner.Listener
             }
         }
 
-        private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null)
+        private async Task CompleteJobRequestAsync(Guid targetHostId, int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null)
         {
             Trace.Entering();
 
@@ -936,7 +936,7 @@ namespace GitHub.Runner.Listener
             {
                 try
                 {
-                    await runnerServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, CancellationToken.None);
+                    await runnerServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, targetHostId, CancellationToken.None);
                     return;
                 }
                 catch (TaskAgentJobNotFoundException)

src/Runner.Listener/MessageListener.cs

@@ -124,6 +124,7 @@ namespace GitHub.Runner.Listener
                 Trace.Info($"Attempt to create session.");
                 try
                 {
+                    Trace.Info("Connecting to the Runner Server...");
                     Trace.Info("Connecting to the Runner Server...");
                     await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
                     Trace.Info("VssConnection created");

src/Runner.Listener/Runner.cs

@@ -13,6 +13,10 @@ using GitHub.Runner.Sdk;
 using System.Linq;
 using GitHub.Runner.Listener.Check;
 using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
 
 namespace GitHub.Runner.Listener
 {
@@ -449,7 +453,37 @@ namespace GitHub.Runner.Listener
                                 {
                                     Trace.Info($"Received job message of length {message.Body.Length} from service, with hash '{IOUtil.GetSha256Hash(message.Body)}'");
                                     var jobMessage = StringUtil.ConvertFromJson<Pipelines.AgentJobRequestMessage>(message.Body);
-                                    jobDispatcher.Run(jobMessage, runOnce);
+                                    jobDispatcher.Run(Guid.Empty, jobMessage, runOnce);
+                                    if (runOnce)
+                                    {
+                                        Trace.Info("One time used runner received job message.");
+                                        runOnceJobReceived = true;
+                                    }
+                                }
+                            }
+                            // Broker flow
+                            else if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase))
+                            {
+                                if (autoUpdateInProgress || runOnceJobReceived)
+                                {
+                                    skipMessageDeletion = true;
+                                    Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
+                                }
+                                else
+                                {
+                                    var messageRef = StringUtil.ConvertFromJson<MessageRef>(message.Body);
+
+                                    // Create connection
+                                    var credMgr = HostContext.GetService<ICredentialManager>();
+                                    var creds = credMgr.LoadCredentials();
+
+                                    // todo: add retries
+                                    var runServer = HostContext.CreateService<IRunServer>();
+                                    await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
+                                    var jobMessage = await runServer.GetJobMessageAsync(messageRef.ScopeId, messageRef.HostId, messageRef.PlanType, messageRef.PlanGroup, messageRef.PlanId, messageRef.InstanceRefs);
+
+                                    // todo: Trace.Info($"Received job message of length {message.Body.Length} from service, with hash '{IOUtil.GetSha256Hash(message.Body)}'");
+                                    jobDispatcher.Run(messageRef.HostId, jobMessage, runOnce);
                                     if (runOnce)
                                     {
                                         Trace.Info("One time used runner received job message.");

src/Runner.Sdk/Util/VssUtil.cs

@@ -36,6 +36,7 @@ namespace GitHub.Runner.Sdk
 
         public static VssConnection CreateConnection(Uri serverUri, VssCredentials credentials, IEnumerable<DelegatingHandler> additionalDelegatingHandler = null, TimeSpan? timeout = null)
         {
+            // System.Console.WriteLine("VssUtil.CreateConnection");
             VssClientHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone();
 
             int maxRetryRequest;

src/Sdk/Common/Common/Authentication/IssuedTokenCredential.cs

@@ -88,6 +88,7 @@ namespace GitHub.Services.Common
             IHttpResponse response,
             IssuedToken failedToken)
         {
+            // System.Console.WriteLine("IssuedTokenCredential.CreateTokenProvider");
             if (response != null && !IsAuthenticationChallenge(response))
             {
                 throw new InvalidOperationException();

src/Sdk/Common/Common/Authentication/VssCredentials.cs

@@ -218,6 +218,7 @@ namespace GitHub.Services.Common
             IHttpResponse webResponse,
             IssuedToken failedToken)
         {
+            // System.Console.WriteLine("VssCredential.CreateTokenProvider");
             ArgumentUtility.CheckForNull(serverUrl, "serverUrl");
 
             IssuedTokenProvider tokenProvider = null;
@@ -272,11 +273,13 @@ namespace GitHub.Services.Common
                 {
                     if (m_federatedCredential != null)
                     {
+                        // System.Console.WriteLine($"VssCredentials.TryGetTokenProvider: Using federated credential");
                         m_currentProvider = m_federatedCredential.CreateTokenProvider(serverUrl, null, null);
                     }
 
                     if (m_currentProvider != null)
                     {
+                        // System.Console.WriteLine($"VssCredentials.TryGetTokenProvider: Issued token provider created");
                         VssHttpEventSource.Log.IssuedTokenProviderCreated(VssTraceActivity.Current, m_currentProvider);
                     }
                 }

src/Sdk/Common/Common/VssHttpMessageHandler.cs

@@ -49,6 +49,7 @@ namespace GitHub.Services.Common
             VssHttpRequestSettings settings,
             HttpMessageHandler innerHandler)
         {
+            // System.Console.WriteLine("VssHttpMessageHandler.ctor");
             this.Credentials = credentials;
             this.Settings = settings;
             this.ExpectContinue = settings.ExpectContinue;
@@ -122,6 +123,7 @@ namespace GitHub.Services.Common
             HttpRequestMessage request,
             CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("VssHttpMessageHandler.SendAsync");
             VssTraceActivity traceActivity = VssTraceActivity.Current;
 
             var traceInfo = VssHttpMessageHandlerTraceInfo.GetTraceInfo(request);
@@ -130,6 +132,7 @@ namespace GitHub.Services.Common
             if (!m_appliedClientCertificatesToTransportHandler &&
                 request.RequestUri.Scheme == "https")
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: !appliedClientCertificatesToTransportHandler");
                 HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
                 if (httpClientHandler != null &&
                     this.Settings.ClientCertificateManager != null &&
@@ -144,6 +147,7 @@ namespace GitHub.Services.Common
             if (!m_appliedServerCertificateValidationCallbackToTransportHandler &&
                 request.RequestUri.Scheme == "https")
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: !appliedServerCertificateValidationCallbackToTransportHandler");
                 HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
                 if (httpClientHandler != null &&
                     this.Settings.ServerCertificateValidationCallback != null)
@@ -165,6 +169,7 @@ namespace GitHub.Services.Common
             IssuedTokenProvider provider;
             if (this.Credentials.TryGetTokenProvider(request.RequestUri, out provider))
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: Got token provider from credentials");
                 token = provider.CurrentToken;
             }
 
@@ -232,6 +237,7 @@ namespace GitHub.Services.Common
                         // Validate the token after it has been successfully authenticated with the server.
                         if (provider != null)
                         {
+                            // System.Console.WriteLine("VssHttpMessageHandler.SendAsync: Validating token");
                             provider.ValidateToken(token, responseWrapper);
                         }
 
@@ -243,6 +249,7 @@ namespace GitHub.Services.Common
                     }
                     else
                     {
+                        // System.Console.WriteLine($"VssHttpMessageHandler.SendAsync: Auth challenge. Response status code {response.StatusCode}; headers {response.Headers}");
                         // In the case of a Windows token, only apply it to the web proxy if it
                         // returned a 407 Proxy Authentication Required. If we didn't get this
                         // status code back, then the proxy (if there is one) is clearly working fine,
@@ -432,6 +439,7 @@ namespace GitHub.Services.Common
                     activity != VssTraceActivity.Empty &&
                     !request.Headers.Contains(HttpHeaders.TfsSessionHeader))
                 {
+                    // System.Console.WriteLine($"VssHttpMessageHandler.ApplyHeaders: Activity ID {activity.Id}");
                     request.Headers.Add(HttpHeaders.TfsSessionHeader, activity.Id.ToString("D"));
                 }
 
@@ -452,13 +460,16 @@ namespace GitHub.Services.Common
             ICredentials credentialsToken = token as ICredentials;
             if (credentialsToken != null)
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Credentials token != null");
                 if (applyICredentialsToWebProxy)
                 {
+                    // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Apply credentials to web proxy");
                     HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
 
                     if (httpClientHandler != null &&
                         httpClientHandler.Proxy != null)
                     {
+                        // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Setting proxy crednetials");
                         httpClientHandler.Proxy.Credentials = credentialsToken;
                     }
                 }
@@ -467,6 +478,7 @@ namespace GitHub.Services.Common
             }
             else
             {
+                // System.Console.WriteLine("VssHttpMessageHandler.ApplyToken: Applying credentials to request");
                 token.ApplyTo(new HttpRequestMessageWrapper(request));
             }
         }
@@ -479,7 +491,8 @@ namespace GitHub.Services.Common
             HttpClientHandler httpClientHandler = handler as HttpClientHandler;
             if (httpClientHandler != null)
             {
-                httpClientHandler.AllowAutoRedirect = settings.AllowAutoRedirect;
+                // System.Console.WriteLine($"VssHttpMessageHandler.ApplySettings: Default credentials = {defaultCredentials} AllowAutoRedirect = {settings.AllowAutoRedirect}");
+                httpClientHandler.AllowAutoRedirect = true; //settings.AllowAutoRedirect;
                 httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual;
                 //Setting httpClientHandler.UseDefaultCredentials to false in .Net Core, clears httpClientHandler.Credentials if
                 //credentials is already set to defaultcredentials. Therefore httpClientHandler.Credentials must be 
@@ -550,6 +563,7 @@ namespace GitHub.Services.Common
                 Uri uri,
                 String authType)
             {
+                // System.Console.WriteLine($"CredentialWrapper.GetCredential: InnerCredentials = {InnerCredentials}");
                 return InnerCredentials != null ? InnerCredentials.GetCredential(uri, authType) : null;
             }
         }

src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs

@@ -27,6 +27,7 @@ using System.Net.Http.Headers;
 using System.Net.Http.Formatting;
 using System.Threading;
 using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
 using GitHub.Services.Common;
 using GitHub.Services.WebApi;
 
@@ -378,6 +379,7 @@ namespace GitHub.DistributedTask.WebApi
         /// <param name="requestId"></param>
         /// <param name="lockToken"></param>
         /// <param name="request"></param>
+        /// <param name="targetHostId"></param>
         /// <param name="userState"></param>
         /// <param name="cancellationToken">The cancellation token to cancel operation.</param>
         [EditorBrowsable(EditorBrowsableState.Never)]
@@ -386,6 +388,7 @@ namespace GitHub.DistributedTask.WebApi
             long requestId,
             Guid lockToken,
             TaskAgentJobRequest request,
+            Guid targetHostId,
             object userState = null,
             CancellationToken cancellationToken = default)
         {
@@ -396,6 +399,7 @@ namespace GitHub.DistributedTask.WebApi
 
             List<KeyValuePair<string, string>> queryParams = new List<KeyValuePair<string, string>>();
             queryParams.Add("lockToken", lockToken.ToString());
+            queryParams.Add("targetHostId", targetHostId.ToString());
 
             return SendAsync<TaskAgentJobRequest>(
                 httpMethod,
@@ -703,6 +707,47 @@ namespace GitHub.DistributedTask.WebApi
                 cancellationToken: cancellationToken);
         }
 
+        /// <summary>
+        /// [Preview API]
+        /// </summary>
+        /// <param name="scopeId"></param>
+        /// <param name="planType"></param>
+        /// <param name="planGroup"></param>
+        /// <param name="planId"></param>
+        /// <param name="instanceRefsJson"></param>
+        /// <param name="userState"></param>
+        /// <param name="cancellationToken">The cancellation token to cancel operation.</param>
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        public virtual Task<Pipelines.AgentJobRequestMessage> GetJobMessageAsync(
+            Guid scopeId,
+            Guid hostId,
+            string planType,
+            string planGroup,
+            Guid planId,
+            string instanceRefsJson,
+            object userState = null,
+            CancellationToken cancellationToken = default)
+        {
+            HttpMethod httpMethod = new HttpMethod("GET");
+            Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a");
+
+            List<KeyValuePair<string, string>> queryParams = new List<KeyValuePair<string, string>>();
+            queryParams.Add("scopeId", scopeId.ToString());
+            queryParams.Add("hostId", hostId.ToString());
+            queryParams.Add("planType", planType);
+            queryParams.Add("planGroup", planGroup);
+            queryParams.Add("planId", planId.ToString());
+            queryParams.Add("instanceRefsJson", instanceRefsJson);
+
+            return SendAsync<Pipelines.AgentJobRequestMessage>(
+                httpMethod,
+                locationId,
+                version: new ApiResourceVersion(6.0, 1),
+                queryParameters: queryParams,
+                userState: userState,
+                cancellationToken: cancellationToken);
+        }
+
         /// <summary>
         /// [Preview API]
         /// </summary>
@@ -717,6 +762,7 @@ namespace GitHub.DistributedTask.WebApi
             object userState = null,
             CancellationToken cancellationToken = default)
         {
+            // System.Console.WriteLine("TaskAgentHttpClientBase.CreateAgentSessionAsync");
             HttpMethod httpMethod = new HttpMethod("POST");
             Guid locationId = new Guid("134e239e-2df3-4794-a6f6-24f1f19ec8dc");
             object routeValues = new { poolId = poolId };

src/Sdk/DTWebApi/WebApi/JobRequestMessageTypes.cs

@@ -5,5 +5,6 @@ namespace GitHub.DistributedTask.WebApi
     public static class JobRequestMessageTypes
     {
         public const String PipelineAgentJobRequest = "PipelineAgentJobRequest";
+        public const String RunnerJobRequest = "RunnerJobRequest";
     }
 }

src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs

@@ -64,6 +64,7 @@ namespace GitHub.DistributedTask.WebApi
             Guid lockToken,
             DateTime finishTime,
             TaskResult result,
+            Guid targetHostId,
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
@@ -74,7 +75,7 @@ namespace GitHub.DistributedTask.WebApi
                 Result = result,
             };
 
-            return UpdateAgentRequestAsync(poolId, requestId, lockToken, request, userState, cancellationToken);
+            return UpdateAgentRequestAsync(poolId, requestId, lockToken, request, targetHostId, userState, cancellationToken);
         }
 
         public Task<List<TaskAgent>> GetAgentsAsync(
@@ -152,6 +153,7 @@ namespace GitHub.DistributedTask.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             Func<HttpResponseMessage, CancellationToken, Task<T>> processResponse = null)
         {
+            // System.Console.WriteLine("TaskAgentHttpClient.SendAsync 1");
             return SendAsync<T>(method, null, locationId, routeValues, version, content, queryParameters, userState, cancellationToken, processResponse);
         }
 
@@ -170,6 +172,7 @@ namespace GitHub.DistributedTask.WebApi
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, additionalHeaders, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
+                // System.Console.WriteLine("TaskAgentHttpClient.SendAsync 2");
                 return await SendAsync<T>(requestMessage, userState, cancellationToken, processResponse).ConfigureAwait(false);
             }
         }
@@ -180,6 +183,7 @@ namespace GitHub.DistributedTask.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             Func<HttpResponseMessage, CancellationToken, Task<T>> processResponse = null)
         {
+            // System.Console.WriteLine("TaskAgentHttpClient.SendAsync 3");
             if (processResponse == null)
             {
                 processResponse = ReadContentAsAsync<T>;

src/Sdk/WebApi/WebApi/Location/ServerDataProvider.cs

@@ -801,6 +801,7 @@ namespace GitHub.Services.WebApi.Location
 
         private async Task<ConnectionData> GetConnectionDataAsync(ConnectOptions connectOptions, int lastChangeId, CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine("ServerDataProvider.GetConnectionDataAsync");
             int timeoutRetries = 1;
 
             while (true)

src/Sdk/WebApi/WebApi/OAuth/VssOAuthAccessToken.cs

@@ -75,6 +75,7 @@ namespace GitHub.Services.OAuth
 
         internal override void ApplyTo(IHttpRequest request)
         {
+            // System.Console.WriteLine($"VssOAuthAccessToken.ApplyTo: Bearer {m_value}");
             request.Headers.SetValue(Common.Internal.HttpHeaders.Authorization, $"Bearer {m_value}");
         }
 

src/Sdk/WebApi/WebApi/OAuth/VssOAuthAccessTokenCredential.cs

@@ -60,6 +60,7 @@ namespace GitHub.Services.OAuth
             Uri serverUrl,
             IHttpResponse response)
         {
+            // System.Console.WriteLine("VssOAuthAccessTokenCredential.OnCreateTokenProvider");
             return new VssOAuthAccessTokenProvider(this, serverUrl, null);
         }
 

src/Sdk/WebApi/WebApi/OAuth/VssOAuthCredential.cs

@@ -121,6 +121,7 @@ namespace GitHub.Services.OAuth
             Uri serverUrl, 
             IHttpResponse response)
         {
+            // System.Console.WriteLine("VssOAuthCredential.OnCreateTokenProvider");
             return new VssOAuthTokenProvider(this, serverUrl);
         }
 

src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenHttpClient.cs

@@ -54,6 +54,7 @@ namespace GitHub.Services.OAuth
             VssOAuthTokenParameters tokenParameters = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // todo: qqq
             VssTraceActivity traceActivity = VssTraceActivity.Current;
             using (HttpClient client = new HttpClient(CreateMessageHandler(this.AuthorizationUrl)))
             {

src/Sdk/WebApi/WebApi/VssConnection.cs

@@ -100,6 +100,7 @@ namespace GitHub.Services.WebApi
             IDictionary<String, String> parameters,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssConnection.ConnectAsync");
             CheckForDisposed();
             // Set the connectMode on the credential's FederatedPrompt
             if (Credentials.Federated != null && Credentials.Federated.Prompt != null)

src/Sdk/WebApi/WebApi/VssHttpClientBase.cs

@@ -390,6 +390,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 1");
             return SendAsync<T>(method, null, locationId, routeValues, version, content, queryParameters, userState, cancellationToken);
         }
 
@@ -404,6 +405,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 2");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, additionalHeaders, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -422,6 +424,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 3");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, additionalHeaders, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -455,6 +458,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 4");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -473,6 +477,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 5");
             using (VssTraceActivity.GetOrCreate().EnterCorrelationScope())
             using (HttpRequestMessage requestMessage = await CreateRequestMessageAsync(method, locationId, routeValues, version, content, queryParameters, userState, cancellationToken).ConfigureAwait(false))
             {
@@ -501,6 +506,7 @@ namespace GitHub.Services.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 1");
             return CreateRequestMessageAsync(method, null, locationId, routeValues, version, content, queryParameters, userState, cancellationToken, mediaType);
         }
 
@@ -526,6 +532,7 @@ namespace GitHub.Services.WebApi
             CancellationToken cancellationToken = default(CancellationToken),
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 2");
             // Lookup the location
             ApiResourceLocation location = await GetResourceLocationAsync(locationId, userState, cancellationToken).ConfigureAwait(false);
             if (location == null)
@@ -555,6 +562,7 @@ namespace GitHub.Services.WebApi
             IEnumerable<KeyValuePair<String, String>> queryParameters = null,
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 3");
             return CreateRequestMessage(method, null, location, routeValues, version, content, queryParameters, mediaType);
         }
 
@@ -578,6 +586,7 @@ namespace GitHub.Services.WebApi
             IEnumerable<KeyValuePair<String, String>> queryParameters = null,
             String mediaType = c_jsonMediaType)
         {
+            // System.Console.WriteLine("VssHttpClientBase.CreateRequestMessageAsync 4");
             CheckForDisposed();
             // Negotiate the request version to send
             ApiResourceVersion requestVersion = NegotiateRequestVersion(location, version);
@@ -749,12 +758,14 @@ namespace GitHub.Services.WebApi
             //from deadlocking...
             using (HttpResponseMessage response = await this.SendAsync(message, userState, cancellationToken).ConfigureAwait(false))
             {
+                // System.Console.WriteLine("VssHttpClientBase.SendAsync 6");
                 return await ReadContentAsAsync<T>(response, cancellationToken).ConfigureAwait(false);
             }
         }
 
         protected async Task<T> ReadContentAsAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine($"VssHttpClientBase.ReadContentAsAsync {response.Headers}");
             CheckForDisposed();
             Boolean isJson = IsJsonResponse(response);
             bool mismatchContentType = false;
@@ -766,17 +777,20 @@ namespace GitHub.Services.WebApi
                     !typeof(Byte[]).GetTypeInfo().IsAssignableFrom(typeof(T).GetTypeInfo()) &&
                     !typeof(JObject).GetTypeInfo().IsAssignableFrom(typeof(T).GetTypeInfo()))
                 {
+                    // System.Console.WriteLine("VssHttpClientBase.ReadContentAsAsync: isJson 1");
                     // expect it to come back wrapped, if it isn't it is a bug!
                     var wrapper = await ReadJsonContentAsync<VssJsonCollectionWrapper<T>>(response, cancellationToken).ConfigureAwait(false);
                     return wrapper.Value;
                 }
                 else if (isJson)
                 {
+                    // System.Console.WriteLine("VssHttpClientBase.ReadContentAsAsync: isJson 2");
                     return await ReadJsonContentAsync<T>(response, cancellationToken).ConfigureAwait(false);
                 }
             }
             catch (JsonReaderException)
             {
+                // System.Console.WriteLine("VssHttpClientBase.ReadContentAsAsync: mismatchContentType");
                 // We thought the content was JSON but failed to parse. 
                 // In this case, do nothing and utilize the HandleUnknownContentType call below
                 mismatchContentType = true;
@@ -802,6 +816,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 7");
             // the default in httpClient for HttpCompletionOption is ResponseContentRead so that is what we do here
             return this.SendAsync(
                 message,
@@ -816,6 +831,7 @@ namespace GitHub.Services.WebApi
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
         {
+            // System.Console.WriteLine("VssHttpClientBase.SendAsync 8");
             CheckForDisposed();
             if (message.Headers.UserAgent != null)
             {
@@ -851,6 +867,7 @@ namespace GitHub.Services.WebApi
                 //ConfigureAwait(false) enables the continuation to be run outside
                 //any captured SyncronizationContext (such as ASP.NET's) which keeps things
                 //from deadlocking...
+                // System.Console.WriteLine($"VssHttpClientBase.SendAsync 8: Calling Client.SendAsync {message}");
                 HttpResponseMessage response = await Client.SendAsync(message, completionOption, cancellationToken).ConfigureAwait(false);
 
                 // Inject delay or failure for testing
@@ -868,6 +885,7 @@ namespace GitHub.Services.WebApi
         [Obsolete("Use VssHttpClientBase.HandleResponseAsync instead")]
         protected virtual void HandleResponse(HttpResponseMessage response)
         {
+            // System.Console.WriteLine("VssHttpClientBase.HandleResponse 1");
 
         }
 
@@ -875,6 +893,7 @@ namespace GitHub.Services.WebApi
             HttpResponseMessage response,
             CancellationToken cancellationToken)
         {
+            // System.Console.WriteLine($"VssHttpClientBase.HandleResponse 2 status code {response.StatusCode} headers {response.Headers}");
             response.Trace();
             VssHttpEventSource.Log.HttpRequestStop(VssTraceActivity.Current, response);
 
@@ -886,6 +905,7 @@ namespace GitHub.Services.WebApi
             }
             else if (ShouldThrowError(response))
             {
+                // System.Console.WriteLine("VssHttpClientBase.HandleResponse: Should throw error");
                 Exception exToThrow = null;
                 if (IsJsonResponse(response))
                 {
@@ -909,6 +929,7 @@ namespace GitHub.Services.WebApi
                     {
                         message = response.ReasonPhrase;
                     }
+                    // System.Console.WriteLine($"VssHttpClientBase.HandleResponse: Exception message {message}");
                     exToThrow = new VssServiceResponseException(response.StatusCode, message, exToThrow);
                 }