actions/runner
1
0
Fork 0
mirror of https://github.com/actions/runner.git synced 2025-12-12 05:37:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

backoff if we retried polling for more than 50 times in less than 30m… (#3232)

Browse Source 
* backoff if we retried polling for more than 50 times in less than 30minutes

* run dotnet format

* move delay to after no message trace

* run dotnet format
This commit is contained in:
Aiqiao Yan
2024-04-09 14:13:07 -04:00
committed by GitHub
parent f467e9e125
commit 4a1e38095b
1 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/Runner.Listener/MessageListener.cs
View File

@@ -236,6 +236,7 @@ namespace GitHub.Runner.Listener
ArgUtil.NotNull(_settings, nameof(_settings)); ArgUtil.NotNull(_settings, nameof(_settings));
bool encounteringError = false; bool encounteringError = false;
int continuousError = 0; int continuousError = 0;
int continuousEmptyMessage = 0;
string errorMessage = string.Empty; string errorMessage = string.Empty;
Stopwatch heartbeat = new(); Stopwatch heartbeat = new();
heartbeat.Restart(); heartbeat.Restart();
@@ -359,16 +360,27 @@ namespace GitHub.Runner.Listener
if (message == null) if (message == null)
{ {
continuousEmptyMessage++;
if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
{ {
Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes."); Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
heartbeat.Restart(); heartbeat.Restart();
continuousEmptyMessage = 0;
} }
else else
{ {
Trace.Verbose($"No message retrieved from session '{_session.SessionId}'."); Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
} }
if (continuousEmptyMessage > 50)
{
// retried more than 50 times in less than 30mins and still getting empty message
// something is not right on the service side, backoff for 15-30s before retry
_getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
await HostContext.Delay(_getNextMessageRetryInterval, token);
}
continue; continue;
} }