actions/runner
1
0
Fork 0
mirror of https://github.com/actions/runner.git synced 2025-12-11 12:57:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Recreate VssConnection on retry (#1316)

Browse Source
This commit is contained in:
Julio Barba
2021-09-09 19:09:17 -04:00
committed by GitHub
parent aab936d081
commit 4359dd605b
3 changed files with 23 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/Runner.Common/JobServer.cs
View File

@@ -2,16 +2,19 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Net.Http;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.Runner.Sdk;
using GitHub.Services.WebApi; using GitHub.Services.WebApi;
using GitHub.Services.Common;
namespace GitHub.Runner.Common namespace GitHub.Runner.Common
{ {
[ServiceLocator(Default = typeof(JobServer))] [ServiceLocator(Default = typeof(JobServer))]
public interface IJobServer : IRunnerService public interface IJobServer : IRunnerService
{ {
Task ConnectAsync(VssConnection jobConnection); Task ConnectAsync(Uri jobServerUrl, VssCredentials jobServerCredential, DelegatingHandler[] delegatingHandler = null);
// logging and console // logging and console
Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken);
@@ -32,20 +35,21 @@ namespace GitHub.Runner.Common
private VssConnection _connection; private VssConnection _connection;
private TaskHttpClient _taskClient; private TaskHttpClient _taskClient;
public async Task ConnectAsync(VssConnection jobConnection) public async Task ConnectAsync(Uri jobServerUrl, VssCredentials jobServerCredential, DelegatingHandler[] delegatingHandler = null)
{ {
_connection = jobConnection; Trace.Info($"Establishing connection for JobServer");
int attemptCount = 5; int attemptCount = 5;
while (!_connection.HasAuthenticated && attemptCount-- > 0)
while (attemptCount-- > 0)
{ {
try try
{ {
await _connection.ConnectAsync(); await RefreshConnectionAsync(jobServerUrl, jobServerCredential, delegatingHandler);
break; break;
} }
catch (Exception ex) when (attemptCount > 0) catch (Exception ex) when (attemptCount > 0)
{ {
Trace.Info($"Catch exception during connect. {attemptCount} attemp left."); Trace.Info($"Catch exception during connect. {attemptCount} attempts left.");
Trace.Error(ex); Trace.Error(ex);
} }
@@ -53,6 +57,15 @@ namespace GitHub.Runner.Common
} }
_taskClient = _connection.GetClient<TaskHttpClient>(); _taskClient = _connection.GetClient<TaskHttpClient>();
}
private async Task RefreshConnectionAsync(Uri jobServerUrl, VssCredentials jobServerCredential, DelegatingHandler[] delegatingHandler)
{
Trace.Info($"Refresh JobServer VssConnection to get on a different AFD node.");
_hasConnection = false;
_connection?.Dispose();
_connection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, delegatingHandler);
await _connection.ConnectAsync();
_hasConnection = true; _hasConnection = true;
} }

6
src/Runner.Listener/JobDispatcher.cs
View File

@@ -510,9 +510,8 @@ namespace GitHub.Runner.Listener
var jobServer = HostContext.GetService<IJobServer>(); var jobServer = HostContext.GetService<IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
await jobServer.ConnectAsync(systemConnection.Url, jobServerCredential);
await LogWorkerProcessUnhandledException(jobServer, message, detailInfo); await LogWorkerProcessUnhandledException(jobServer, message, detailInfo);
// Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space. // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
@@ -791,9 +790,8 @@ namespace GitHub.Runner.Listener
var jobServer = HostContext.GetService<IJobServer>(); var jobServer = HostContext.GetService<IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential);
await jobServer.ConnectAsync(jobConnection); await jobServer.ConnectAsync(systemConnection.Url, jobServerCredential);
var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None); var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);

4
src/Runner.Worker/JobRunner.cs
View File

@@ -48,8 +48,8 @@ namespace GitHub.Runner.Worker
Trace.Info($"Creating job server with URL: {jobServerUrl}"); Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter. // jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService<IJobServerQueue>(); _jobServerQueue = HostContext.GetService<IJobServerQueue>();
VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection); await jobServer.ConnectAsync(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
_jobServerQueue.Start(message); _jobServerQueue.Start(message);
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");