diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index efa5cf707..0e05e8a15 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -457,22 +457,13 @@ namespace GitHub.Runner.Listener message = await getNextMessage; //get next message HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}"); - if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase) || - string.Equals(message.MessageType, RunnerRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) { if (autoUpdateInProgress == false) { autoUpdateInProgress = true; - AgentRefreshMessage runnerUpdateMessage = null; - if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) - { - runnerUpdateMessage = JsonUtility.FromString(message.Body); - } - else - { - var brokerRunnerUpdateMessage = JsonUtility.FromString(message.Body); - runnerUpdateMessage = new AgentRefreshMessage(brokerRunnerUpdateMessage.RunnerId, brokerRunnerUpdateMessage.TargetVersion, TimeSpan.FromSeconds(brokerRunnerUpdateMessage.TimeoutInSeconds)); - } + AgentRefreshMessage runnerUpdateMessage = JsonUtility.FromString(message.Body); + #if DEBUG // Can mock the update for testing if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_IS_MOCK_UPDATE"))) @@ -503,6 +494,22 @@ namespace GitHub.Runner.Listener Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running."); } } + else if (string.Equals(message.MessageType, RunnerRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) + { + if (autoUpdateInProgress == false) + { + autoUpdateInProgress = true; + RunnerRefreshMessage brokerRunnerUpdateMessage = JsonUtility.FromString(message.Body); + + var selfUpdater = HostContext.GetService(); + selfUpdateTask = selfUpdater.SelfUpdate(brokerRunnerUpdateMessage, jobDispatcher, false, HostContext.RunnerShutdownToken); + Trace.Info("Refresh message received, kick-off selfupdate background process."); + } + else + { + Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running."); + } + } else if (string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase)) { if (autoUpdateInProgress || runOnceJobReceived) diff --git a/src/Runner.Listener/SelfUpdaterV2.cs b/src/Runner.Listener/SelfUpdaterV2.cs new file mode 100644 index 000000000..46bced18c --- /dev/null +++ b/src/Runner.Listener/SelfUpdaterV2.cs @@ -0,0 +1,569 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; +using GitHub.Services.WebApi; + +namespace GitHub.Runner.Listener +{ + // This class is a fork of SelfUpdater.cs and is intended to only be used for the + // new self-update flow where the PackageMetadata is sent in the message directly. + // Forking the class prevents us from accidentally breaking the old flow while it's still in production + + [ServiceLocator(Default = typeof(SelfUpdaterV2))] + public interface ISelfUpdaterV2 : IRunnerService + { + bool Busy { get; } + Task SelfUpdate(RunnerRefreshMessage updateMessage, IJobDispatcher jobDispatcher, bool restartInteractiveRunner, CancellationToken token); + } + public class SelfUpdaterV2 : RunnerService, ISelfUpdaterV2 + { + private static string _platform = BuildConstants.RunnerPackage.PackageName; + private ITerminal _terminal; + private IRunnerServer _runnerServer; + private int _poolId; + private ulong _agentId; + + private const int _numberOfOldVersionsToKeep = 1; + + private readonly ConcurrentQueue _updateTrace = new(); + public bool Busy { get; private set; } + + public override void Initialize(IHostContext hostContext) + { + base.Initialize(hostContext); + + _terminal = hostContext.GetService(); + _runnerServer = HostContext.GetService(); + var configStore = HostContext.GetService(); + var settings = configStore.GetSettings(); + _poolId = settings.PoolId; + _agentId = settings.AgentId; + } + + public async Task SelfUpdate(RunnerRefreshMessage updateMessage, IJobDispatcher jobDispatcher, bool restartInteractiveRunner, CancellationToken token) + { + Busy = true; + try + { + var totalUpdateTime = Stopwatch.StartNew(); + + Trace.Info($"An update is available."); + _updateTrace.Enqueue($"RunnerPlatform: {updateMessage.OS}"); + + // Print console line that warn user not shutdown runner. + _terminal.WriteLine("Runner update in progress, do not shutdown runner."); + _terminal.WriteLine($"Downloading {updateMessage.TargetVersion} runner"); + + await DownloadLatestRunner(token, updateMessage.TargetVersion, updateMessage.DownloadUrl, updateMessage.SHA256Checksum, updateMessage.OS); + Trace.Info($"Download latest runner and unzip into runner root."); + + // wait till all running job finish + _terminal.WriteLine("Waiting for current job finish running."); + + await jobDispatcher.WaitAsync(token); + Trace.Info($"All running job has exited."); + + // We need to keep runner backup around for macOS until we fixed https://github.com/actions/runner/issues/743 + // delete runner backup + var stopWatch = Stopwatch.StartNew(); + DeletePreviousVersionRunnerBackup(token, updateMessage.TargetVersion); + Trace.Info($"Delete old version runner backup."); + stopWatch.Stop(); + // generate update script from template + _updateTrace.Enqueue($"DeleteRunnerBackupTime: {stopWatch.ElapsedMilliseconds}ms"); + _terminal.WriteLine("Generate and execute update script."); + + string updateScript = GenerateUpdateScript(restartInteractiveRunner, updateMessage.TargetVersion); + Trace.Info($"Generate update script into: {updateScript}"); + + +#if DEBUG + // For L0, we will skip execute update script. + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_EXECUTE_UPDATE_SCRIPT"))) +#endif + { + string flagFile = "update.finished"; + IOUtil.DeleteFile(flagFile); + // kick off update script + Process invokeScript = new(); +#if OS_WINDOWS + invokeScript.StartInfo.FileName = WhichUtil.Which("cmd.exe", trace: Trace); + invokeScript.StartInfo.Arguments = $"/c \"{updateScript}\""; +#elif (OS_OSX || OS_LINUX) + invokeScript.StartInfo.FileName = WhichUtil.Which("bash", trace: Trace); + invokeScript.StartInfo.Arguments = $"\"{updateScript}\""; +#endif + invokeScript.Start(); + Trace.Info($"Update script start running"); + } + + totalUpdateTime.Stop(); + + _updateTrace.Enqueue($"TotalUpdateTime: {totalUpdateTime.ElapsedMilliseconds}ms"); + _terminal.WriteLine("Runner will exit shortly for update, should be back online within 10 seconds."); + + return true; + } + catch (Exception ex) + { + _updateTrace.Enqueue(ex.ToString()); + throw; + } + finally + { + _terminal.WriteLine("Runner update process finished."); + Busy = false; + } + } + + /// + /// _work + /// \_update + /// \bin + /// \externals + /// \run.sh + /// \run.cmd + /// \package.zip //temp download .zip/.tar.gz + /// + /// + /// + private async Task DownloadLatestRunner(CancellationToken token, string targetVersion, string packageDownloadUrl, string packageHashValue, string targetPlatform) + { + string latestRunnerDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.UpdateDirectory); + IOUtil.DeleteDirectory(latestRunnerDirectory, token); + Directory.CreateDirectory(latestRunnerDirectory); + + string archiveFile = null; + + // Only try trimmed package if sever sends them and we have calculated hash value of the current runtime/externals. + _updateTrace.Enqueue($"DownloadUrl: {packageDownloadUrl}"); + + try + { +#if DEBUG + // Much of the update process (targetVersion, archive) is server-side, this is a way to control it from here for testing specific update scenarios + // Add files like 'runner2.281.2.tar.gz' or 'runner2.283.0.zip' (depending on your platform) to your runner root folder + // Note that runners still need to be older than the server's runner version in order to receive an 'AgentRefreshMessage' and trigger this update + // Wrapped in #if DEBUG as this should not be in the RELEASE build + if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_IS_MOCK_UPDATE"))) + { + var waitForDebugger = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_IS_MOCK_UPDATE_WAIT_FOR_DEBUGGER")); + if (waitForDebugger) + { + int waitInSeconds = 20; + while (!Debugger.IsAttached && waitInSeconds-- > 0) + { + await Task.Delay(1000); + } + Debugger.Break(); + } + + if (targetPlatform.StartsWith("win")) + { + archiveFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"runner{targetVersion}.zip"); + } + else + { + archiveFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"runner{targetVersion}.tar.gz"); + } + + if (File.Exists(archiveFile)) + { + _updateTrace.Enqueue($"Mocking update with file: '{archiveFile}' and targetVersion: '{targetVersion}', nothing is downloaded"); + _terminal.WriteLine($"Mocking update with file: '{archiveFile}' and targetVersion: '{targetVersion}', nothing is downloaded"); + } + else + { + archiveFile = null; + _terminal.WriteLine($"Mock runner archive not found at {archiveFile} for target version {targetVersion}, proceeding with download instead"); + _updateTrace.Enqueue($"Mock runner archive not found at {archiveFile} for target version {targetVersion}, proceeding with download instead"); + } + } +#endif + // archiveFile is not null only if we mocked it above + if (string.IsNullOrEmpty(archiveFile)) + { + archiveFile = await DownLoadRunner(latestRunnerDirectory, packageDownloadUrl, packageHashValue, targetPlatform, token); + + if (string.IsNullOrEmpty(archiveFile)) + { + throw new TaskCanceledException($"Runner package '{packageDownloadUrl}' failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts"); + } + await ValidateRunnerHash(archiveFile, packageHashValue); + } + + await ExtractRunnerPackage(archiveFile, latestRunnerDirectory, token); + } + finally + { + try + { + // delete .zip file + if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile)) + { + Trace.Verbose("Deleting latest runner package zip: {0}", archiveFile); + IOUtil.DeleteFile(archiveFile); + } + } + catch (Exception ex) + { + //it is not critical if we fail to delete the .zip file + Trace.Warning("Failed to delete runner package zip '{0}'. Exception: {1}", archiveFile, ex); + } + } + + await CopyLatestRunnerToRoot(latestRunnerDirectory, targetVersion, token); + } + + private async Task DownLoadRunner(string downloadDirectory, string packageDownloadUrl, string packageHashValue, string packagePlatform, CancellationToken token) + { + var stopWatch = Stopwatch.StartNew(); + int runnerSuffix = 1; + string archiveFile = null; + bool downloadSucceeded = false; + + // Download the runner, using multiple attempts in order to be resilient against any networking/CDN issues + for (int attempt = 1; attempt <= Constants.RunnerDownloadRetryMaxAttempts; attempt++) + { + // Generate an available package name, and do our best effort to clean up stale local zip files + while (true) + { + if (packagePlatform.StartsWith("win")) + { + archiveFile = Path.Combine(downloadDirectory, $"runner{runnerSuffix}.zip"); + } + else + { + archiveFile = Path.Combine(downloadDirectory, $"runner{runnerSuffix}.tar.gz"); + } + + try + { + // delete .zip file + if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile)) + { + Trace.Verbose("Deleting latest runner package zip '{0}'", archiveFile); + IOUtil.DeleteFile(archiveFile); + } + + break; + } + catch (Exception ex) + { + // couldn't delete the file for whatever reason, so generate another name + Trace.Warning("Failed to delete runner package zip '{0}'. Exception: {1}", archiveFile, ex); + runnerSuffix++; + } + } + + // Allow a 15-minute package download timeout, which is good enough to update the runner from a 1 Mbit/s ADSL connection. + if (!int.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds)) + { + timeoutSeconds = 15 * 60; + } + + Trace.Info($"Attempt {attempt}: save latest runner into {archiveFile}."); + + using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) + using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token)) + { + try + { + Trace.Info($"Download runner: begin download"); + long downloadSize = 0; + + //open zip stream in async mode + using (HttpClient httpClient = new(HostContext.CreateHttpClientHandler())) + { + Trace.Info($"Downloading {packageDownloadUrl}"); + + using (FileStream fs = new(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true)) + using (Stream result = await httpClient.GetStreamAsync(packageDownloadUrl)) + { + //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k). + await result.CopyToAsync(fs, 81920, downloadCts.Token); + await fs.FlushAsync(downloadCts.Token); + downloadSize = fs.Length; + } + } + + Trace.Info($"Download runner: finished download"); + downloadSucceeded = true; + stopWatch.Stop(); + _updateTrace.Enqueue($"PackageDownloadTime: {stopWatch.ElapsedMilliseconds}ms"); + _updateTrace.Enqueue($"Attempts: {attempt}"); + _updateTrace.Enqueue($"PackageSize: {downloadSize / 1024 / 1024}MB"); + break; + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + Trace.Info($"Runner download has been cancelled."); + throw; + } + catch (Exception ex) + { + if (downloadCts.Token.IsCancellationRequested) + { + Trace.Warning($"Runner download has timed out after {timeoutSeconds} seconds"); + } + + Trace.Warning($"Failed to get package '{archiveFile}' from '{packageDownloadUrl}'. Exception {ex}"); + } + } + } + + if (downloadSucceeded) + { + return archiveFile; + } + else + { + return null; + } + } + + private async Task ValidateRunnerHash(string archiveFile, string packageHashValue) + { + var stopWatch = Stopwatch.StartNew(); + // Validate Hash Matches if it is provided + using (FileStream stream = File.OpenRead(archiveFile)) + { + if (!string.IsNullOrEmpty(packageHashValue)) + { + using (SHA256 sha256 = SHA256.Create()) + { + byte[] srcHashBytes = await sha256.ComputeHashAsync(stream); + var hash = PrimitiveExtensions.ConvertToHexString(srcHashBytes); + if (hash != packageHashValue) + { + // Hash did not match, we can't recover from this, just throw + throw new Exception($"Computed runner hash {hash} did not match expected Runner Hash {packageHashValue} for {archiveFile}"); + } + + stopWatch.Stop(); + Trace.Info($"Validated Runner Hash matches {archiveFile} : {packageHashValue}"); + _updateTrace.Enqueue($"ValidateHashTime: {stopWatch.ElapsedMilliseconds}ms"); + } + } + } + } + + private async Task ExtractRunnerPackage(string archiveFile, string extractDirectory, CancellationToken token) + { + var stopWatch = Stopwatch.StartNew(); + + if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + ZipFile.ExtractToDirectory(archiveFile, extractDirectory); + } + else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase)) + { + string tar = WhichUtil.Which("tar", trace: Trace); + + if (string.IsNullOrEmpty(tar)) + { + throw new NotSupportedException($"tar -xzf"); + } + + // tar -xzf + using (var processInvoker = HostContext.CreateService()) + { + processInvoker.OutputDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + Trace.Info(args.Data); + } + }); + + processInvoker.ErrorDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + Trace.Error(args.Data); + } + }); + + int exitCode = await processInvoker.ExecuteAsync(extractDirectory, tar, $"-xzf \"{archiveFile}\"", null, token); + if (exitCode != 0) + { + throw new NotSupportedException($"Can't use 'tar -xzf' to extract archive file: {archiveFile}. return code: {exitCode}."); + } + } + } + else + { + throw new NotSupportedException($"{archiveFile}"); + } + + stopWatch.Stop(); + Trace.Info($"Finished getting latest runner package at: {extractDirectory}."); + _updateTrace.Enqueue($"PackageExtractTime: {stopWatch.ElapsedMilliseconds}ms"); + } + + private Task CopyLatestRunnerToRoot(string latestRunnerDirectory, string targetVersion, CancellationToken token) + { + var stopWatch = Stopwatch.StartNew(); + // copy latest runner into runner root folder + // copy bin from _work/_update -> bin.version under root + string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{targetVersion}"); + Directory.CreateDirectory(binVersionDir); + Trace.Info($"Copy {Path.Combine(latestRunnerDirectory, Constants.Path.BinDirectory)} to {binVersionDir}."); + IOUtil.CopyDirectory(Path.Combine(latestRunnerDirectory, Constants.Path.BinDirectory), binVersionDir, token); + + // copy externals from _work/_update -> externals.version under root + string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{targetVersion}"); + Directory.CreateDirectory(externalsVersionDir); + Trace.Info($"Copy {Path.Combine(latestRunnerDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}."); + IOUtil.CopyDirectory(Path.Combine(latestRunnerDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token); + + // copy and replace all .sh/.cmd files + Trace.Info($"Copy any remaining .sh/.cmd files into runner root."); + foreach (FileInfo file in new DirectoryInfo(latestRunnerDirectory).GetFiles() ?? new FileInfo[0]) + { + string destination = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name); + + // Removing the file instead of just trying to overwrite it works around permissions issues on linux. + // https://github.com/actions/runner/issues/981 + Trace.Info($"Copy {file.FullName} to {destination}"); + IOUtil.DeleteFile(destination); + file.CopyTo(destination, true); + } + + stopWatch.Stop(); + _updateTrace.Enqueue($"CopyRunnerToRootTime: {stopWatch.ElapsedMilliseconds}ms"); + return Task.CompletedTask; + } + + private void DeletePreviousVersionRunnerBackup(CancellationToken token, string targetVersion) + { + // delete previous backup runner (back compat, can be remove after serval sprints) + // bin.bak.2.99.0 + // externals.bak.2.99.0 + foreach (string existBackUp in Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "*.bak.*")) + { + Trace.Info($"Delete existing runner backup at {existBackUp}."); + try + { + IOUtil.DeleteDirectory(existBackUp, token); + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + Trace.Error(ex); + Trace.Info($"Catch exception during delete backup folder {existBackUp}, ignore this error try delete the backup folder on next auto-update."); + } + } + + // delete old bin.2.99.0 folder, only leave the current version and the latest download version + var allBinDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "bin.*"); + if (allBinDirs.Length > _numberOfOldVersionsToKeep) + { + // there are more than {_numberOfOldVersionsToKeep} bin.version folder. + // delete older bin.version folders. + foreach (var oldBinDir in allBinDirs) + { + if (string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin"), StringComparison.OrdinalIgnoreCase) || + string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin.{BuildConstants.RunnerPackage.Version}"), StringComparison.OrdinalIgnoreCase) || + string.Equals(oldBinDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"bin.{targetVersion}"), StringComparison.OrdinalIgnoreCase)) + { + // skip for current runner version + continue; + } + + Trace.Info($"Delete runner bin folder's backup at {oldBinDir}."); + try + { + IOUtil.DeleteDirectory(oldBinDir, token); + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + Trace.Error(ex); + Trace.Info($"Catch exception during delete backup folder {oldBinDir}, ignore this error try delete the backup folder on next auto-update."); + } + } + } + + // delete old externals.2.99.0 folder, only leave the current version and the latest download version + var allExternalsDirs = Directory.GetDirectories(HostContext.GetDirectory(WellKnownDirectory.Root), "externals.*"); + if (allExternalsDirs.Length > _numberOfOldVersionsToKeep) + { + // there are more than {_numberOfOldVersionsToKeep} externals.version folder. + // delete older externals.version folders. + foreach (var oldExternalDir in allExternalsDirs) + { + if (string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals"), StringComparison.OrdinalIgnoreCase) || + string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals.{BuildConstants.RunnerPackage.Version}"), StringComparison.OrdinalIgnoreCase) || + string.Equals(oldExternalDir, Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"externals.{targetVersion}"), StringComparison.OrdinalIgnoreCase)) + { + // skip for current runner version + continue; + } + + Trace.Info($"Delete runner externals folder's backup at {oldExternalDir}."); + try + { + IOUtil.DeleteDirectory(oldExternalDir, token); + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + Trace.Error(ex); + Trace.Info($"Catch exception during delete backup folder {oldExternalDir}, ignore this error try delete the backup folder on next auto-update."); + } + } + } + } + + private string GenerateUpdateScript(bool restartInteractiveRunner, string targetVersion) + { + int processId = Process.GetCurrentProcess().Id; + string updateLog = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Diag), $"SelfUpdate-{DateTime.UtcNow.ToString("yyyyMMdd-HHmmss")}.log"); + string runnerRoot = HostContext.GetDirectory(WellKnownDirectory.Root); + +#if OS_WINDOWS + string templateName = "update.cmd.template"; +#else + string templateName = "update.sh.template"; +#endif + + string templatePath = Path.Combine(runnerRoot, $"bin.{targetVersion}", templateName); + string template = File.ReadAllText(templatePath); + + template = template.Replace("_PROCESS_ID_", processId.ToString()); + template = template.Replace("_RUNNER_PROCESS_NAME_", $"Runner.Listener{IOUtil.ExeExtension}"); + template = template.Replace("_ROOT_FOLDER_", runnerRoot); + template = template.Replace("_EXIST_RUNNER_VERSION_", BuildConstants.RunnerPackage.Version); + template = template.Replace("_DOWNLOAD_RUNNER_VERSION_", targetVersion); + template = template.Replace("_UPDATE_LOG_", updateLog); + template = template.Replace("_RESTART_INTERACTIVE_RUNNER_", restartInteractiveRunner ? "1" : "0"); + +#if OS_WINDOWS + string scriptName = "_update.cmd"; +#else + string scriptName = "_update.sh"; +#endif + + string updateScript = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), scriptName); + if (File.Exists(updateScript)) + { + IOUtil.DeleteFile(updateScript); + } + + File.WriteAllText(updateScript, template); + return updateScript; + } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs b/src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs index 2e1e92836..43fddb56c 100644 --- a/src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs +++ b/src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs @@ -1,5 +1,6 @@ using Newtonsoft.Json; using System; +using System.Collections.Generic; using System.Runtime.Serialization; @@ -15,35 +16,32 @@ namespace GitHub.DistributedTask.WebApi { } - public RunnerRefreshMessage( - ulong runnerId, - String targetVersion, - int? timeoutInSeconds = null) - { - this.RunnerId = runnerId; - this.TimeoutInSeconds = timeoutInSeconds ?? TimeSpan.FromMinutes(60).Seconds; - this.TargetVersion = targetVersion; - } - - [DataMember] - public ulong RunnerId - { - get; - private set; - } - - [DataMember] - public int TimeoutInSeconds - { - get; - private set; - } - - [DataMember] + [DataMember(Name = "target_version")] public String TargetVersion { get; - private set; + set; + } + + [DataMember(Name = "download_url")] + public string DownloadUrl + { + get; + set; + } + + [DataMember(Name = "sha256_checksum")] + public string SHA256Checksum + { + get; + set; + } + + [DataMember(Name = "os")] + public string OS + { + get; + set; } } } diff --git a/src/Test/L0/Listener/SelfUpdaterV2L0.cs b/src/Test/L0/Listener/SelfUpdaterV2L0.cs new file mode 100644 index 000000000..51004e1fb --- /dev/null +++ b/src/Test/L0/Listener/SelfUpdaterV2L0.cs @@ -0,0 +1,241 @@ +#if !(OS_WINDOWS && ARM64) +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Listener; +using GitHub.Runner.Sdk; +using Moq; +using Xunit; + +namespace GitHub.Runner.Common.Tests.Listener +{ + public sealed class SelfUpdaterV2L0 + { + private Mock _runnerServer; + private Mock _term; + private Mock _configStore; + private Mock _jobDispatcher; + private AgentRefreshMessage _refreshMessage = new(1, "2.999.0"); + private List _trimmedPackages = new(); + +#if !OS_WINDOWS + private string _packageUrl = null; +#else + private string _packageUrl = null; +#endif + public SelfUpdaterV2L0() + { + _runnerServer = new Mock(); + _term = new Mock(); + _configStore = new Mock(); + _jobDispatcher = new Mock(); + _configStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1, AgentId = 1 }); + + Environment.SetEnvironmentVariable("_GITHUB_ACTION_EXECUTE_UPDATE_SCRIPT", "1"); + } + + private async Task FetchLatestRunner() + { + var latestVersion = ""; + var httpClientHandler = new HttpClientHandler(); + httpClientHandler.AllowAutoRedirect = false; + using (var client = new HttpClient(httpClientHandler)) + { + var response = await client.SendAsync(new HttpRequestMessage(HttpMethod.Get, "https://github.com/actions/runner/releases/latest")); + if (response.StatusCode == System.Net.HttpStatusCode.Redirect) + { + var redirectUrl = response.Headers.Location.ToString(); + Regex regex = new(@"/runner/releases/tag/v(?\d+\.\d+\.\d+)"); + var match = regex.Match(redirectUrl); + if (match.Success) + { + latestVersion = match.Groups["version"].Value; + +#if !OS_WINDOWS + _packageUrl = $"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}.tar.gz"; +#else + _packageUrl = $"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}.zip"; +#endif + } + else + { + throw new Exception("The latest runner version could not be determined so a download URL could not be generated for it. Please check the location header of the redirect response of 'https://github.com/actions/runner/releases/latest'"); + } + } + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdaterV2(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + try + { + var message = new RunnerRefreshMessage() + { + TargetVersion = "2.999.0", + OS = BuildConstants.RunnerPackage.PackageName, + DownloadUrl = _packageUrl + + }; + + var result = await updater.SelfUpdate(message, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.True(result); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"))); + } + finally + { + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None); + } + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_DownloadRetry() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdaterV2(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + var message = new RunnerRefreshMessage() + { + TargetVersion = "2.999.0", + OS = BuildConstants.RunnerPackage.PackageName, + DownloadUrl = "https://github.com/actions/runner/notexists" + }; + + var ex = await Assert.ThrowsAsync(() => updater.SelfUpdate(message, _jobDispatcher.Object, true, hc.RunnerShutdownToken)); + Assert.Contains($"failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts", ex.Message); + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_ValidateHash() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdaterV2(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + var message = new RunnerRefreshMessage() + { + TargetVersion = "2.999.0", + OS = BuildConstants.RunnerPackage.PackageName, + DownloadUrl = _packageUrl, + SHA256Checksum = "badhash" + }; + + var ex = await Assert.ThrowsAsync(() => updater.SelfUpdate(message, _jobDispatcher.Object, true, hc.RunnerShutdownToken)); + Assert.Contains("did not match expected Runner Hash", ex.Message); + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + } +} +#endif