diff --git a/src/Misc/layoutbin/RunnerService.js b/src/Misc/layoutbin/RunnerService.js index bb4b64ed2..ba0a8c659 100644 --- a/src/Misc/layoutbin/RunnerService.js +++ b/src/Misc/layoutbin/RunnerService.js @@ -24,7 +24,8 @@ if (exitServiceAfterNFailures <= 0) { exitServiceAfterNFailures = NaN; } -var consecutiveFailureCount = 0; +var unknownFailureRetryCount = 0; +var retriableFailureRetryCount = 0; var gracefulShutdown = function () { console.log("Shutting down runner listener"); @@ -62,7 +63,8 @@ var runService = function () { listener.stdout.on("data", (data) => { if (data.toString("utf8").includes("Listening for Jobs")) { - consecutiveFailureCount = 0; + unknownFailureRetryCount = 0; + retriableFailureRetryCount = 0; } process.stdout.write(data.toString("utf8")); }); @@ -92,24 +94,38 @@ var runService = function () { console.log( "Runner listener exit with retryable error, re-launch runner in 5 seconds." ); - consecutiveFailureCount = 0; + unknownFailureRetryCount = 0; + retriableFailureRetryCount++; + if (retriableFailureRetryCount >= 10) { + console.error( + "Stopping the runner after 10 consecutive re-tryable failures" + ); + stopping = true; + } } else if (code === 3 || code === 4) { console.log( "Runner listener exit because of updating, re-launch runner in 5 seconds." ); - consecutiveFailureCount = 0; + unknownFailureRetryCount = 0; + retriableFailureRetryCount++; + if (retriableFailureRetryCount >= 10) { + console.error( + "Stopping the runner after 10 consecutive re-tryable failures" + ); + stopping = true; + } } else { var messagePrefix = "Runner listener exit with undefined return code"; - consecutiveFailureCount++; + unknownFailureRetryCount++; + retriableFailureRetryCount = 0; if ( !isNaN(exitServiceAfterNFailures) && - consecutiveFailureCount >= exitServiceAfterNFailures + unknownFailureRetryCount >= exitServiceAfterNFailures ) { console.error( - `${messagePrefix}, exiting service after ${consecutiveFailureCount} consecutive failures` + `${messagePrefix}, exiting service after ${unknownFailureRetryCount} consecutive failures` ); - gracefulShutdown(); - return; + stopping = true } else { console.log(`${messagePrefix}, re-launch runner in 5 seconds.`); }