actions/runner
1
0
Fork 0
mirror of https://github.com/actions/runner.git synced 2025-12-12 15:13:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add retry logic around getting job messages from broker (#1939)

Browse Source 
* Jsut simple solution without additional funcs

* Delete old comment

* resolve

* Refactor retry function, make it more common

* Make retry function generic, get rid of extra params

* delete extra using

* Add cancellation token and limit of attempts

* Add some additional logging

* Rework condition

* replace to do..while

* return `while (true)` to simplify code structure

* Add other cancelling token, add TODO comment
This commit is contained in:
Tatyana Kostromskaya
2022-06-21 16:12:07 +02:00
committed by GitHub
parent 6bc6d475f9
commit 9499f477a2
2 changed files with 34 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/Runner.Common/RunServer.cs
View File

@@ -17,7 +17,7 @@ namespace GitHub.Runner.Common
{
Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
Task<AgentJobRequestMessage> GetJobMessageAsync(string id);
Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken token);
}
public sealed class RunServer : RunnerService, IRunServer
@@ -67,10 +67,40 @@ namespace GitHub.Runner.Common
}
}
public Task<AgentJobRequestMessage> GetJobMessageAsync(string id)
public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken cancellationToken)
{
CheckConnection();
return _taskAgentClient.GetJobMessageAsync(id);
var jobMessage = RetryRequest<AgentJobRequestMessage>(async () =>
{
return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken);
}, cancellationToken);
return jobMessage;
}
private async Task<T> RetryRequest<T>(Func<Task<T>> func,
CancellationToken cancellationToken,
int maxRetryAttemptsCount = 5
)
{
var retryCount = 0;
while (true)
{
retryCount++;
cancellationToken.ThrowIfCancellationRequested();
try
{
return await func();
}
// TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122
catch (Exception ex) when (retryCount < maxRetryAttemptsCount)
{
Trace.Error("Catch exception during get full job message");
Trace.Error(ex);
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
Trace.Warning($"Back off {backOff.TotalSeconds} seconds before next retry. {maxRetryAttemptsCount - retryCount} attempt left.");
await Task.Delay(backOff, cancellationToken);
}
}
}
}
}

3
src/Runner.Listener/Runner.cs
View File

@@ -474,10 +474,9 @@ namespace GitHub.Runner.Listener
var credMgr = HostContext.GetService<ICredentialManager>();
var creds = credMgr.LoadCredentials();
// todo: add retries https://github.com/github/actions-broker/issues/49
var runServer = HostContext.CreateService<IRunServer>();
await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
var jobMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId);
var jobMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token);
jobDispatcher.Run(jobMessage, runOnce);
if (runOnce)