From e5d19459a714a4ddd18d3156a73ff67829f735cd Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Thu, 17 Feb 2022 10:58:12 +0100 Subject: [PATCH] RunnerService.js added logic to fail on N attempts --- src/Misc/layoutbin/RunnerService.js | 186 +++++++++++++++++----------- 1 file changed, 114 insertions(+), 72 deletions(-) diff --git a/src/Misc/layoutbin/RunnerService.js b/src/Misc/layoutbin/RunnerService.js index 6574ae0e6..9c117807e 100644 --- a/src/Misc/layoutbin/RunnerService.js +++ b/src/Misc/layoutbin/RunnerService.js @@ -3,94 +3,136 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. var childProcess = require("child_process"); -var path = require("path") +var path = require("path"); +const { exit } = require("process"); -var supported = ['linux', 'darwin'] +var supported = ["linux", "darwin"]; if (supported.indexOf(process.platform) == -1) { - console.log('Unsupported platform: ' + process.platform); - console.log('Supported platforms are: ' + supported.toString()); - process.exit(1); + console.log("Unsupported platform: " + process.platform); + console.log("Supported platforms are: " + supported.toString()); + process.exit(1); } var stopping = false; var listener = null; -var runService = function () { - var listenerExePath = path.join(__dirname, '../bin/Runner.Listener'); - var interactive = process.argv[2] === "interactive"; +var exitServiceAfterNFailures = Number( + process.env.GITHUB_ACTIONS_SERVICE_EXIT_AFTER_N_FAILURES +); - if (!stopping) { - try { - if (interactive) { - console.log('Starting Runner listener interactively'); - listener = childProcess.spawn(listenerExePath, ['run'], { env: process.env }); - } else { - console.log('Starting Runner listener with startup type: service'); - listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'], { env: process.env }); - } - - console.log(`Started listener process, pid: ${listener.pid}`); - - listener.stdout.on('data', (data) => { - process.stdout.write(data.toString('utf8')); - }); - - listener.stderr.on('data', (data) => { - process.stdout.write(data.toString('utf8')); - }); - - listener.on("error", (err) => { - console.log(`Runner listener fail to start with error ${err.message}`); - }); - - listener.on('close', (code) => { - console.log(`Runner listener exited with error code ${code}`); - - if (code === 0) { - console.log('Runner listener exit with 0 return code, stop the service, no retry needed.'); - stopping = true; - } else if (code === 1) { - console.log('Runner listener exit with terminated error, stop the service, no retry needed.'); - stopping = true; - } else if (code === 2) { - console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.'); - } else if (code === 3) { - console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.'); - } else { - console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.'); - } - - if (!stopping) { - setTimeout(runService, 5000); - } - }); - - } catch (ex) { - console.log(ex); - } - } +if (exitServiceAfterNFailures <= 0) { + exitServiceAfterNFailures = NaN; } -runService(); -console.log('Started running service'); +var consecutiveFailureCount = 0; var gracefulShutdown = function (code) { - console.log('Shutting down runner listener'); - stopping = true; - if (listener) { - console.log('Sending SIGINT to runner listener to stop'); - listener.kill('SIGINT'); + console.log("Shutting down runner listener"); + stopping = true; + if (listener) { + console.log("Sending SIGINT to runner listener to stop"); + listener.kill("SIGINT"); - console.log('Sending SIGKILL to runner listener'); - setTimeout(() => listener.kill('SIGKILL'), 30000).unref(); + console.log("Sending SIGKILL to runner listener"); + setTimeout(() => listener.kill("SIGKILL"), 30000).unref(); + } +}; + +var runService = function () { + var listenerExePath = path.join(__dirname, "../bin/Runner.Listener"); + var interactive = process.argv[2] === "interactive"; + + if (!stopping) { + try { + if (interactive) { + console.log("Starting Runner listener interactively"); + listener = childProcess.spawn(listenerExePath, ["run"], { + env: process.env, + }); + } else { + console.log("Starting Runner listener with startup type: service"); + listener = childProcess.spawn( + listenerExePath, + ["run", "--startuptype", "service"], + { env: process.env } + ); + } + + console.log(`Started listener process, pid: ${listener.pid}`); + + listener.stdout.on("data", (data) => { + if (data.toString("utf8").includes("Listening for Jobs")) { + consecutiveFailureCount = 0; + } + process.stdout.write(data.toString("utf8")); + }); + + listener.stderr.on("data", (data) => { + process.stdout.write(data.toString("utf8")); + }); + + listener.on("error", (err) => { + console.log(`Runner listener fail to start with error ${err.message}`); + }); + + listener.on("close", (code) => { + console.log(`Runner listener exited with error code ${code}`); + + if (code === 0) { + console.log( + "Runner listener exit with 0 return code, stop the service, no retry needed." + ); + stopping = true; + } else if (code === 1) { + console.log( + "Runner listener exit with terminated error, stop the service, no retry needed." + ); + stopping = true; + } else if (code === 2) { + console.log( + "Runner listener exit with retryable error, re-launch runner in 5 seconds." + ); + consecutiveFailureCount = 0; + } else if (code === 3 || code === 4) { + console.log( + "Runner listener exit because of updating, re-launch runner in 5 seconds." + ); + consecutiveFailureCount = 0; + } else { + var messagePrefix = "Runner listener exit with undefined return code"; + consecutiveFailureCount++; + if ( + !isNaN(exitServiceAfterNFailures) && + consecutiveFailureCount >= exitServiceAfterNFailures + ) { + console.error( + `${messagePrefix}, exiting service after ${consecutiveFailureCount} consecutive failures` + ); + gracefulShutdown(5); + return; + } else { + console.log(`${messagePrefix}, re-launch runner in 5 seconds.`); + } + } + + if (!stopping) { + setTimeout(runService, 5000); + } + }); + } catch (ex) { + console.log(ex); } -} + } +}; -process.on('SIGINT', () => { - gracefulShutdown(0); +runService(); +console.log("Started running service"); + +process.on("SIGINT", () => { + gracefulShutdown(0); }); -process.on('SIGTERM', () => { - gracefulShutdown(0); +process.on("SIGTERM", () => { + gracefulShutdown(0); });