diff --git a/src/Runner.Listener/SelfUpdater.cs b/src/Runner.Listener/SelfUpdater.cs index 045edb3e5..e2f0b2b58 100644 --- a/src/Runner.Listener/SelfUpdater.cs +++ b/src/Runner.Listener/SelfUpdater.cs @@ -1,21 +1,20 @@ -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common.Util; using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; using System.Net.Http; +using System.Reflection; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; -using System.Security.Cryptography; -using GitHub.Services.WebApi; -using GitHub.Services.Common; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; using GitHub.Runner.Sdk; -using System.Text; -using System.Collections.Generic; -using System.Reflection; +using GitHub.Services.Common; +using GitHub.Services.WebApi; namespace GitHub.Runner.Listener { @@ -30,13 +29,19 @@ namespace GitHub.Runner.Listener { private static string _packageType = "agent"; private static string _platform = BuildConstants.RunnerPackage.PackageName; + private static string _dotnetRuntime = "dotnetRuntime"; + private static string _externals = "externals"; + private readonly Dictionary _contentHashes = new Dictionary(); private PackageMetadata _targetPackage; private ITerminal _terminal; private IRunnerServer _runnerServer; private int _poolId; private int _agentId; - private readonly List _updateTrace = new List(); + private readonly ConcurrentQueue _updateTrace = new ConcurrentQueue(); + private Task _cloneAndCalculateContentHashTask; + private string _dotnetRuntimeCloneDirectory; + private string _externalsCloneDirectory; public bool Busy { get; private set; } @@ -50,6 +55,8 @@ namespace GitHub.Runner.Listener var settings = configStore.GetSettings(); _poolId = settings.PoolId; _agentId = settings.AgentId; + _dotnetRuntimeCloneDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "__dotnet_runtime__"); + _externalsCloneDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "__externals__"); } public async Task SelfUpdate(AgentRefreshMessage updateMessage, IJobDispatcher jobDispatcher, bool restartInteractiveRunner, CancellationToken token) @@ -59,6 +66,13 @@ namespace GitHub.Runner.Listener { var totalUpdateTime = Stopwatch.StartNew(); + // Copy dotnet runtime and externals of current runner to a temp folder + // So we can re-use them with trimmed runner package, if possible. + // This process is best effort, if we can't use trimmed runner package, + // we will just go with the full package. + var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); + _cloneAndCalculateContentHashTask = CloneAndCalculateAssetsHash(_dotnetRuntimeCloneDirectory, _externalsCloneDirectory, linkedTokenSource.Token); + if (!await UpdateNeeded(updateMessage.TargetVersion, token)) { Trace.Info($"Can't find available update package."); @@ -66,12 +80,30 @@ namespace GitHub.Runner.Listener } Trace.Info($"An update is available."); - _updateTrace.Add($"RunnerPlatform: {_targetPackage.Platform}"); + _updateTrace.Enqueue($"RunnerPlatform: {_targetPackage.Platform}"); // Print console line that warn user not shutdown runner. await UpdateRunnerUpdateStateAsync("Runner update in progress, do not shutdown runner."); await UpdateRunnerUpdateStateAsync($"Downloading {_targetPackage.Version} runner"); + if (_targetPackage.TrimmedPackages?.Count > 0) + { + // wait for cloning assets task to finish only if we have trimmed packages + await _cloneAndCalculateContentHashTask; + } + else + { + linkedTokenSource.Cancel(); + try + { + await _cloneAndCalculateContentHashTask; + } + catch (Exception ex) + { + Trace.Info($"Ingore errors after cancelling cloning assets task: {ex}"); + } + } + await DownloadLatestRunner(token); Trace.Info($"Download latest runner and unzip into runner root."); @@ -88,34 +120,39 @@ namespace GitHub.Runner.Listener Trace.Info($"Delete old version runner backup."); stopWatch.Stop(); // generate update script from template - _updateTrace.Add($"DeleteRunnerBackupTime: {stopWatch.ElapsedMilliseconds}ms"); + _updateTrace.Enqueue($"DeleteRunnerBackupTime: {stopWatch.ElapsedMilliseconds}ms"); await UpdateRunnerUpdateStateAsync("Generate and execute update script."); string updateScript = GenerateUpdateScript(restartInteractiveRunner); Trace.Info($"Generate update script into: {updateScript}"); - // kick off update script - Process invokeScript = new Process(); + + // For L0, we will skip execute update script. + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_EXECUTE_UPDATE_SCRIPT"))) + { + // kick off update script + Process invokeScript = new Process(); #if OS_WINDOWS - invokeScript.StartInfo.FileName = WhichUtil.Which("cmd.exe", trace: Trace); - invokeScript.StartInfo.Arguments = $"/c \"{updateScript}\""; + invokeScript.StartInfo.FileName = WhichUtil.Which("cmd.exe", trace: Trace); + invokeScript.StartInfo.Arguments = $"/c \"{updateScript}\""; #elif (OS_OSX || OS_LINUX) - invokeScript.StartInfo.FileName = WhichUtil.Which("bash", trace: Trace); - invokeScript.StartInfo.Arguments = $"\"{updateScript}\""; + invokeScript.StartInfo.FileName = WhichUtil.Which("bash", trace: Trace); + invokeScript.StartInfo.Arguments = $"\"{updateScript}\""; #endif - invokeScript.Start(); - Trace.Info($"Update script start running"); + invokeScript.Start(); + Trace.Info($"Update script start running"); + } totalUpdateTime.Stop(); - _updateTrace.Add($"TotalUpdateTime: {totalUpdateTime.ElapsedMilliseconds}ms"); + _updateTrace.Enqueue($"TotalUpdateTime: {totalUpdateTime.ElapsedMilliseconds}ms"); await UpdateRunnerUpdateStateAsync("Runner will exit shortly for update, should be back online within 10 seconds."); return true; } catch (Exception ex) { - _updateTrace.Add(ex.ToString()); + _updateTrace.Enqueue(ex.ToString()); throw; } finally @@ -178,7 +215,54 @@ namespace GitHub.Runner.Listener string archiveFile = null; var packageDownloadUrl = _targetPackage.DownloadUrl; var packageHashValue = _targetPackage.HashValue; - _updateTrace.Add($"DownloadUrl: {packageDownloadUrl}"); + var runtimeTrimmed = false; + var externalsTrimmed = false; + var fallbackToFullPackage = false; + + // Only try trimmed package if sever sends them and we have calculated hash value of the current runtime/externals. + if (_contentHashes.Count == 2 && + _contentHashes.ContainsKey(_dotnetRuntime) && + _contentHashes.ContainsKey(_externals) && + _targetPackage.TrimmedPackages?.Count > 0) + { + Trace.Info($"Current runner content hash: {StringUtil.ConvertToJson(_contentHashes)}"); + Trace.Info($"Trimmed packages info from service: {StringUtil.ConvertToJson(_targetPackage.TrimmedPackages)}"); + // Try to see whether we can use any size trimmed down package to speed up runner updates. + foreach (var trimmedPackage in _targetPackage.TrimmedPackages) + { + if (trimmedPackage.TrimmedContents.Count == 2 && + trimmedPackage.TrimmedContents.TryGetValue(_dotnetRuntime, out var trimmedRuntimeHash) && + trimmedRuntimeHash == _contentHashes[_dotnetRuntime] && + trimmedPackage.TrimmedContents.TryGetValue(_externals, out var trimmedExternalsHash) && + trimmedExternalsHash == _contentHashes[_externals]) + { + Trace.Info($"Use trimmed (runtime+externals) package '{trimmedPackage.DownloadUrl}' to update runner."); + packageDownloadUrl = trimmedPackage.DownloadUrl; + packageHashValue = trimmedPackage.HashValue; + runtimeTrimmed = true; + externalsTrimmed = true; + break; + } + else if (trimmedPackage.TrimmedContents.Count == 1 && + trimmedPackage.TrimmedContents.TryGetValue(_externals, out trimmedExternalsHash) && + trimmedExternalsHash == _contentHashes[_externals]) + { + Trace.Info($"Use trimmed (externals) package '{trimmedPackage.DownloadUrl}' to update runner."); + packageDownloadUrl = trimmedPackage.DownloadUrl; + packageHashValue = trimmedPackage.HashValue; + externalsTrimmed = true; + break; + } + else + { + Trace.Info($"Can't use trimmed package from '{trimmedPackage.DownloadUrl}' since the current runner does not carry those trimmed content (Hash mismatch)."); + } + } + } + + _updateTrace.Enqueue($"DownloadUrl: {packageDownloadUrl}"); + _updateTrace.Enqueue($"RuntimeTrimmed: {runtimeTrimmed}"); + _updateTrace.Enqueue($"ExternalsTrimmed: {externalsTrimmed}"); try { @@ -193,6 +277,12 @@ namespace GitHub.Runner.Listener await ExtractRunnerPackage(archiveFile, latestRunnerDirectory, token); } + catch (Exception ex) when (runtimeTrimmed || externalsTrimmed) + { + // if anything failed when we use trimmed package (download/validatehase/extract), try again with the full runner package. + Trace.Error($"Fail to download latest runner using trimmed package: {ex}"); + fallbackToFullPackage = true; + } finally { try @@ -211,6 +301,74 @@ namespace GitHub.Runner.Listener } } + var trimmedPackageRestoreTasks = new List>(); + if (!fallbackToFullPackage) + { + // Skip restoring externals and runtime if we are going to fullback to the full package. + if (externalsTrimmed) + { + trimmedPackageRestoreTasks.Add(RestoreTrimmedExternals(latestRunnerDirectory, token)); + } + if (runtimeTrimmed) + { + trimmedPackageRestoreTasks.Add(RestoreTrimmedDotnetRuntime(latestRunnerDirectory, token)); + } + } + + if (trimmedPackageRestoreTasks.Count > 0) + { + var restoreResults = await Task.WhenAll(trimmedPackageRestoreTasks); + if (restoreResults.Any(x => x == false)) + { + // if any of the restore failed, fallback to full package. + fallbackToFullPackage = true; + } + } + + if (fallbackToFullPackage) + { + Trace.Error("Something wrong with the trimmed runner package, failback to use the full package for runner updates."); + _updateTrace.Enqueue($"FallbackToFullPackage: {fallbackToFullPackage}"); + + IOUtil.DeleteDirectory(latestRunnerDirectory, token); + Directory.CreateDirectory(latestRunnerDirectory); + + packageDownloadUrl = _targetPackage.DownloadUrl; + packageHashValue = _targetPackage.HashValue; + _updateTrace.Enqueue($"DownloadUrl: {packageDownloadUrl}"); + + try + { + archiveFile = await DownLoadRunner(latestRunnerDirectory, packageDownloadUrl, packageHashValue, token); + + if (string.IsNullOrEmpty(archiveFile)) + { + throw new TaskCanceledException($"Runner package '{packageDownloadUrl}' failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts"); + } + + await ValidateRunnerHash(archiveFile, packageHashValue); + + await ExtractRunnerPackage(archiveFile, latestRunnerDirectory, token); + } + finally + { + try + { + // delete .zip file + if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile)) + { + Trace.Verbose("Deleting latest runner package zip: {0}", archiveFile); + IOUtil.DeleteFile(archiveFile); + } + } + catch (Exception ex) + { + //it is not critical if we fail to delete the .zip file + Trace.Warning("Failed to delete runner package zip '{0}'. Exception: {1}", archiveFile, ex); + } + } + } + await CopyLatestRunnerToRoot(latestRunnerDirectory, token); } @@ -295,9 +453,9 @@ namespace GitHub.Runner.Listener Trace.Info($"Download runner: finished download"); downloadSucceeded = true; stopWatch.Stop(); - _updateTrace.Add($"PackageDownloadTime: {stopWatch.ElapsedMilliseconds}ms"); - _updateTrace.Add($"Attempts: {attempt}"); - _updateTrace.Add($"PackageSize: {downloadSize / 1024 / 1024}MB"); + _updateTrace.Enqueue($"PackageDownloadTime: {stopWatch.ElapsedMilliseconds}ms"); + _updateTrace.Enqueue($"Attempts: {attempt}"); + _updateTrace.Enqueue($"PackageSize: {downloadSize / 1024 / 1024}MB"); break; } catch (OperationCanceledException) when (token.IsCancellationRequested) @@ -342,12 +500,12 @@ namespace GitHub.Runner.Listener if (hash != packageHashValue) { // Hash did not match, we can't recover from this, just throw - throw new Exception($"Computed runner hash {hash} did not match expected Runner Hash {packageHashValue} for {_targetPackage.Filename}"); + throw new Exception($"Computed runner hash {hash} did not match expected Runner Hash {packageHashValue} for {archiveFile}"); } stopWatch.Stop(); - Trace.Info($"Validated Runner Hash matches {_targetPackage.Filename} : {packageHashValue}"); - _updateTrace.Add($"ValidateHashTime: {stopWatch.ElapsedMilliseconds}ms"); + Trace.Info($"Validated Runner Hash matches {archiveFile} : {packageHashValue}"); + _updateTrace.Enqueue($"ValidateHashTime: {stopWatch.ElapsedMilliseconds}ms"); } } } @@ -403,7 +561,7 @@ namespace GitHub.Runner.Listener stopWatch.Stop(); Trace.Info($"Finished getting latest runner package at: {extractDirectory}."); - _updateTrace.Add($"PackageExtractTime: {stopWatch.ElapsedMilliseconds}ms"); + _updateTrace.Enqueue($"PackageExtractTime: {stopWatch.ElapsedMilliseconds}ms"); } private Task CopyLatestRunnerToRoot(string latestRunnerDirectory, CancellationToken token) @@ -436,7 +594,7 @@ namespace GitHub.Runner.Listener } stopWatch.Stop(); - _updateTrace.Add($"CopyRunnerToRootTime: {stopWatch.ElapsedMilliseconds}ms"); + _updateTrace.Enqueue($"CopyRunnerToRootTime: {stopWatch.ElapsedMilliseconds}ms"); return Task.CompletedTask; } @@ -561,9 +719,15 @@ namespace GitHub.Runner.Listener { _terminal.WriteLine(currentState); - if (_updateTrace.Count > 0) + var traces = new List(); + while (_updateTrace.TryDequeue(out var trace)) { - foreach (var trace in _updateTrace) + traces.Add(trace); + } + + if (traces.Count > 0) + { + foreach (var trace in traces) { Trace.Info(trace); } @@ -571,7 +735,7 @@ namespace GitHub.Runner.Listener try { - await _runnerServer.UpdateAgentUpdateStateAsync(_poolId, _agentId, currentState, string.Join(Environment.NewLine, _updateTrace)); + await _runnerServer.UpdateAgentUpdateStateAsync(_poolId, _agentId, currentState, string.Join(Environment.NewLine, traces)); _updateTrace.Clear(); } catch (VssResourceNotFoundException) @@ -585,5 +749,328 @@ namespace GitHub.Runner.Listener Trace.Info($"Catch exception during report update state, ignore this error and continue auto-update."); } } + + private async Task RestoreTrimmedExternals(string downloadDirectory, CancellationToken token) + { + // Copy the current runner's externals if we are using a externals trimmed package + // Execute the node.js to make sure the copied externals is working. + var stopWatch = Stopwatch.StartNew(); + try + { + Trace.Info($"Copy {_externalsCloneDirectory} to {Path.Combine(downloadDirectory, Constants.Path.ExternalsDirectory)}."); + IOUtil.CopyDirectory(_externalsCloneDirectory, Path.Combine(downloadDirectory, Constants.Path.ExternalsDirectory), token); + + // try run node.js to see if current node.js works fine after copy over to new location. + var nodeVersions = new[] { "node12", "node16" }; + foreach (var nodeVersion in nodeVersions) + { + var newNodeBinary = Path.Combine(downloadDirectory, Constants.Path.ExternalsDirectory, nodeVersion, "bin", $"node{IOUtil.ExeExtension}"); + if (File.Exists(newNodeBinary)) + { + using (var p = HostContext.CreateService()) + { + var outputs = ""; + p.ErrorDataReceived += (_, data) => + { + if (!string.IsNullOrEmpty(data.Data)) + { + Trace.Error(data.Data); + } + }; + p.OutputDataReceived += (_, data) => + { + if (!string.IsNullOrEmpty(data.Data)) + { + Trace.Info(data.Data); + outputs = data.Data; + } + }; + var exitCode = await p.ExecuteAsync(HostContext.GetDirectory(WellKnownDirectory.Root), newNodeBinary, $"-e \"console.log('{nameof(RestoreTrimmedExternals)}')\"", null, token); + if (exitCode != 0) + { + Trace.Error($"{newNodeBinary} -e \"console.log()\" failed with exit code {exitCode}"); + return false; + } + + if (!string.Equals(outputs, nameof(RestoreTrimmedExternals), StringComparison.OrdinalIgnoreCase)) + { + Trace.Error($"{newNodeBinary} -e \"console.log()\" did not output expected content."); + return false; + } + } + } + } + + return true; + } + catch (Exception ex) + { + Trace.Error($"Fail to restore externals for trimmed package: {ex}"); + return false; + } + finally + { + stopWatch.Stop(); + _updateTrace.Enqueue($"{nameof(RestoreTrimmedExternals)}Time: {stopWatch.ElapsedMilliseconds}ms"); + } + } + + private async Task RestoreTrimmedDotnetRuntime(string downloadDirectory, CancellationToken token) + { + // Copy the current runner's dotnet runtime if we are using a dotnet runtime trimmed package + // Execute the runner.listener to make sure the copied runtime is working. + var stopWatch = Stopwatch.StartNew(); + try + { + Trace.Info($"Copy {_dotnetRuntimeCloneDirectory} to {Path.Combine(downloadDirectory, Constants.Path.BinDirectory)}."); + IOUtil.CopyDirectory(_dotnetRuntimeCloneDirectory, Path.Combine(downloadDirectory, Constants.Path.BinDirectory), token); + + // try run the runner executable to see if current dotnet runtime + future runner binary works fine. + var newRunnerBinary = Path.Combine(downloadDirectory, Constants.Path.BinDirectory, "Runner.Listener"); + using (var p = HostContext.CreateService()) + { + p.ErrorDataReceived += (_, data) => + { + if (!string.IsNullOrEmpty(data.Data)) + { + Trace.Error(data.Data); + } + }; + p.OutputDataReceived += (_, data) => + { + if (!string.IsNullOrEmpty(data.Data)) + { + Trace.Info(data.Data); + } + }; + var exitCode = await p.ExecuteAsync(HostContext.GetDirectory(WellKnownDirectory.Root), newRunnerBinary, "--version", null, token); + if (exitCode != 0) + { + Trace.Error($"{newRunnerBinary} --version failed with exit code {exitCode}"); + return false; + } + else + { + return true; + } + } + } + catch (Exception ex) + { + Trace.Error($"Fail to restore dotnet runtime for trimmed package: {ex}"); + return false; + } + finally + { + stopWatch.Stop(); + _updateTrace.Enqueue($"{nameof(RestoreTrimmedDotnetRuntime)}Time: {stopWatch.ElapsedMilliseconds}ms"); + } + } + + private async Task CloneAndCalculateAssetsHash(string dotnetRuntimeCloneDirectory, string externalsCloneDirectory, CancellationToken token) + { + var runtimeCloneTask = CloneDotnetRuntime(dotnetRuntimeCloneDirectory, token); + var externalsCloneTask = CloneExternals(externalsCloneDirectory, token); + + var waitingTasks = new Dictionary() + { + {nameof(CloneDotnetRuntime), runtimeCloneTask}, + {nameof(CloneExternals),externalsCloneTask} + }; + + while (waitingTasks.Count > 0) + { + Trace.Info($"Waiting for {waitingTasks.Count} tasks to complete."); + var complatedTask = await Task.WhenAny(waitingTasks.Values); + if (waitingTasks.ContainsKey(nameof(CloneExternals)) && + complatedTask == waitingTasks[nameof(CloneExternals)]) + { + Trace.Info($"Externals clone finished."); + waitingTasks.Remove(nameof(CloneExternals)); + try + { + if (await externalsCloneTask && !token.IsCancellationRequested) + { + var externalsHash = await HashFiles(externalsCloneDirectory, token); + Trace.Info($"Externals content hash: {externalsHash}"); + _contentHashes[_externals] = externalsHash; + _updateTrace.Enqueue($"ExternalsHash: {_contentHashes[_externals]}"); + } + else + { + Trace.Error($"Skip compute hash since clone externals failed/cancelled."); + } + } + catch (Exception ex) + { + Trace.Error($"Fail to hash externals content: {ex}"); + } + } + else if (waitingTasks.ContainsKey(nameof(CloneDotnetRuntime)) && + complatedTask == waitingTasks[nameof(CloneDotnetRuntime)]) + { + Trace.Info($"Dotnet runtime clone finished."); + waitingTasks.Remove(nameof(CloneDotnetRuntime)); + try + { + if (await runtimeCloneTask && !token.IsCancellationRequested) + { + var runtimeHash = await HashFiles(dotnetRuntimeCloneDirectory, token); + Trace.Info($"Runtime content hash: {runtimeHash}"); + _contentHashes[_dotnetRuntime] = runtimeHash; + _updateTrace.Enqueue($"DotnetRuntimeHash: {_contentHashes[_dotnetRuntime]}"); + } + else + { + Trace.Error($"Skip compute hash since clone dotnet runtime failed/cancelled."); + } + } + catch (Exception ex) + { + Trace.Error($"Fail to hash runtime content: {ex}"); + } + } + + Trace.Info($"Still waiting for {waitingTasks.Count} tasks to complete."); + } + } + + private async Task CloneDotnetRuntime(string runtimeDir, CancellationToken token) + { + var stopWatch = Stopwatch.StartNew(); + try + { + Trace.Info($"Cloning dotnet runtime to {runtimeDir}"); + IOUtil.DeleteDirectory(runtimeDir, CancellationToken.None); + Directory.CreateDirectory(runtimeDir); + + var assembly = Assembly.GetExecutingAssembly(); + var assetsContent = default(string); + using (var stream = assembly.GetManifestResourceStream("GitHub.Runner.Listener.runnercoreassets")) + using (var streamReader = new StreamReader(stream)) + { + assetsContent = await streamReader.ReadToEndAsync(); + } + + if (!string.IsNullOrEmpty(assetsContent)) + { + var runnerCoreAssets = assetsContent.Split(new[] { "\n", "\r\n" }, StringSplitOptions.RemoveEmptyEntries); + if (runnerCoreAssets.Length > 0) + { + var binDir = HostContext.GetDirectory(WellKnownDirectory.Bin); + IOUtil.CopyDirectory(binDir, runtimeDir, token); + + var clonedFile = 0; + foreach (var file in Directory.EnumerateFiles(runtimeDir, "*", SearchOption.AllDirectories)) + { + token.ThrowIfCancellationRequested(); + if (runnerCoreAssets.Any(x => file.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).EndsWith(x.Trim()))) + { + Trace.Verbose($"{file} is part of the runner core, delete from cloned runtime directory."); + IOUtil.DeleteFile(file); + } + else + { + clonedFile++; + } + } + + Trace.Info($"Successfully cloned dotnet runtime to {runtimeDir}. Total files: {clonedFile}"); + return true; + } + } + } + catch (Exception ex) + { + Trace.Error($"Fail to clone dotnet runtime to {runtimeDir}"); + Trace.Error(ex); + } + finally + { + stopWatch.Stop(); + _updateTrace.Enqueue($"{nameof(CloneDotnetRuntime)}Time: {stopWatch.ElapsedMilliseconds}ms"); + } + + return false; + } + + private Task CloneExternals(string externalsDir, CancellationToken token) + { + var stopWatch = Stopwatch.StartNew(); + try + { + Trace.Info($"Cloning externals to {externalsDir}"); + IOUtil.DeleteDirectory(externalsDir, CancellationToken.None); + Directory.CreateDirectory(externalsDir); + IOUtil.CopyDirectory(HostContext.GetDirectory(WellKnownDirectory.Externals), externalsDir, token); + Trace.Info($"Successfully cloned externals to {externalsDir}."); + return Task.FromResult(true); + } + catch (Exception ex) + { + Trace.Error($"Fail to clone externals to {externalsDir}"); + Trace.Error(ex); + } + finally + { + stopWatch.Stop(); + _updateTrace.Enqueue($"{nameof(CloneExternals)}Time: {stopWatch.ElapsedMilliseconds}ms"); + } + + return Task.FromResult(false); + } + + private async Task HashFiles(string fileFolder, CancellationToken token) + { + Trace.Info($"Calculating hash for {fileFolder}"); + + var stopWatch = Stopwatch.StartNew(); + string binDir = HostContext.GetDirectory(WellKnownDirectory.Bin); + string node = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node12", "bin", $"node{IOUtil.ExeExtension}"); + string hashFilesScript = Path.Combine(binDir, "hashFiles"); + var hashResult = string.Empty; + + using (var processInvoker = HostContext.CreateService()) + { + processInvoker.ErrorDataReceived += (_, data) => + { + if (!string.IsNullOrEmpty(data.Data) && data.Data.StartsWith("__OUTPUT__") && data.Data.EndsWith("__OUTPUT__")) + { + hashResult = data.Data.Substring(10, data.Data.Length - 20); + Trace.Info($"Hash result: '{hashResult}'"); + } + else + { + Trace.Info(data.Data); + } + }; + + processInvoker.OutputDataReceived += (_, data) => + { + Trace.Verbose(data.Data); + }; + + var env = new Dictionary + { + ["patterns"] = "**" + }; + + int exitCode = await processInvoker.ExecuteAsync(workingDirectory: fileFolder, + fileName: node, + arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"", + environment: env, + requireExitCodeZero: false, + cancellationToken: token); + + if (exitCode != 0) + { + Trace.Error($"hashFiles returns '{exitCode}' failed. Fail to hash files under directory '{fileFolder}'"); + } + + stopWatch.Stop(); + _updateTrace.Enqueue($"{nameof(HashFiles)}{Path.GetFileName(fileFolder)}Time: {stopWatch.ElapsedMilliseconds}ms"); + return hashResult; + } + } } } diff --git a/src/Sdk/DTWebApi/WebApi/PackageMetadata.cs b/src/Sdk/DTWebApi/WebApi/PackageMetadata.cs index 117169272..f02f0f702 100644 --- a/src/Sdk/DTWebApi/WebApi/PackageMetadata.cs +++ b/src/Sdk/DTWebApi/WebApi/PackageMetadata.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.ComponentModel; using System.Runtime.Serialization; @@ -110,5 +111,43 @@ namespace GitHub.DistributedTask.WebApi get; set; } + + /// + /// A set of trimmed down packages: + /// - the package without 'externals' + /// - the package without 'dotnet runtime' + /// - the package without 'dotnet runtime' and 'externals' + /// + [DataMember(EmitDefaultValue = false)] + public List TrimmedPackages + { + get; + set; + } + } + + [DataContract] + public class TrimmedPackageMetadata + { + [DataMember(EmitDefaultValue = false)] + public string HashValue { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string DownloadUrl { get; set; } + + public Dictionary TrimmedContents + { + get + { + if (m_trimmedContents == null) + { + m_trimmedContents = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + return m_trimmedContents; + } + } + + [DataMember(Name = "TrimmedContents", EmitDefaultValue = false)] + private Dictionary m_trimmedContents; } } diff --git a/src/Test/L0/Listener/SelfUpdaterL0.cs b/src/Test/L0/Listener/SelfUpdaterL0.cs index 2ac6346d4..ff0dc9aeb 100644 --- a/src/Test/L0/Listener/SelfUpdaterL0.cs +++ b/src/Test/L0/Listener/SelfUpdaterL0.cs @@ -1,12 +1,17 @@ -using GitHub.DistributedTask.WebApi; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Listener; using GitHub.Runner.Sdk; using Moq; -using System; -using System.Threading; -using System.Threading.Tasks; using Xunit; -using System.IO; namespace GitHub.Runner.Common.Tests.Listener { @@ -17,11 +22,12 @@ namespace GitHub.Runner.Common.Tests.Listener private Mock _configStore; private Mock _jobDispatcher; private AgentRefreshMessage _refreshMessage = new AgentRefreshMessage(1, "2.299.0"); + private List _trimmedPackages = new List(); #if !OS_WINDOWS - private string _packageUrl = $"https://github.com/actions/runner/releases/download/v2.285.1/actions-runner-{BuildConstants.RunnerPackage.PackageName}-2.285.1.tar.gz"; + private string _packageUrl = null; #else - private string _packageUrl = $"https://github.com/actions/runner/releases/download/v2.285.1/actions-runner-{BuildConstants.RunnerPackage.PackageName}-2.285.1.zip"; + private string _packageUrl = null; #endif public SelfUpdaterL0() { @@ -31,8 +37,44 @@ namespace GitHub.Runner.Common.Tests.Listener _jobDispatcher = new Mock(); _configStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1, AgentId = 1 }); + Environment.SetEnvironmentVariable("_GITHUB_ACTION_EXECUTE_UPDATE_SCRIPT", "1"); + } + + private async Task FetchLatestRunner() + { + var latestVersion = ""; + var httpClientHandler = new HttpClientHandler(); + httpClientHandler.AllowAutoRedirect = false; + using (var client = new HttpClient(httpClientHandler)) + { + var response = await client.SendAsync(new HttpRequestMessage(HttpMethod.Get, "https://github.com/actions/runner/releases/latest")); + if (response.StatusCode == System.Net.HttpStatusCode.Redirect) + { + var redirect = await response.Content.ReadAsStringAsync(); + Regex regex = new Regex(@"/runner/releases/tag/v(?\d+\.\d+\.\d+)"); + var match = regex.Match(redirect); + if (match.Success) + { + latestVersion = match.Groups["version"].Value; + +#if !OS_WINDOWS + _packageUrl = $"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}.tar.gz"; +#else + _packageUrl = $"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}.zip"; +#endif + } + } + } + + using (var client = new HttpClient()) + { + var json = await client.GetStringAsync($"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}-trimmedpackages.json"); + _trimmedPackages = StringUtil.ConvertFromJson>(json); + } + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl })); + } [Fact] @@ -40,40 +82,60 @@ namespace GitHub.Runner.Common.Tests.Listener [Trait("Category", "Runner")] public async void TestSelfUpdateAsync() { - using (var hc = new TestHostContext(this)) + try { - //Arrange - var updater = new Runner.Listener.SelfUpdater(); - hc.SetSingleton(_term.Object); - hc.SetSingleton(_runnerServer.Object); - hc.SetSingleton(_configStore.Object); - hc.SetSingleton(new HttpClientHandlerFactory()); - - var p = new ProcessInvokerWrapper(); - p.Initialize(hc); - hc.EnqueueInstance(p); - updater.Initialize(hc); - - _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) - .Callback((int p, int a, string s, string t) => - { - hc.GetTrace().Info(t); - }) - .Returns(Task.FromResult(new TaskAgent())); - - try + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) { - var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); - Assert.True(result); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); - } - finally - { - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + try + { + var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.True(result); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + } + finally + { + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + } } } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } } [Fact] @@ -81,27 +143,51 @@ namespace GitHub.Runner.Common.Tests.Listener [Trait("Category", "Runner")] public async void TestSelfUpdateAsync_NoUpdateOnOldVersion() { - using (var hc = new TestHostContext(this)) + try { - //Arrange - var updater = new Runner.Listener.SelfUpdater(); - hc.SetSingleton(_term.Object); - hc.SetSingleton(_runnerServer.Object); - hc.SetSingleton(_configStore.Object); - updater.Initialize(hc); + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.200.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.200.0"), DownloadUrl = _packageUrl })); + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); - _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) - .Callback((int p, int a, string s, string t) => - { - hc.GetTrace().Info(t); - }) - .Returns(Task.FromResult(new TaskAgent())); + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); - var result = await updater.SelfUpdate(new AgentRefreshMessage(1, "2.200.0"), _jobDispatcher.Object, true, hc.RunnerShutdownToken); - Assert.False(result); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.200.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.200.0"), DownloadUrl = _packageUrl })); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + var result = await updater.SelfUpdate(new AgentRefreshMessage(1, "2.200.0"), _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.False(result); + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); } } @@ -110,33 +196,53 @@ namespace GitHub.Runner.Common.Tests.Listener [Trait("Category", "Runner")] public async void TestSelfUpdateAsync_DownloadRetry() { - using (var hc = new TestHostContext(this)) + try { - //Arrange - var updater = new Runner.Listener.SelfUpdater(); - hc.SetSingleton(_term.Object); - hc.SetSingleton(_runnerServer.Object); - hc.SetSingleton(_configStore.Object); - hc.SetSingleton(new HttpClientHandlerFactory()); + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = $"https://github.com/actions/runner/notexists" })); + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); - var p = new ProcessInvokerWrapper(); - p.Initialize(hc); - hc.EnqueueInstance(p); - updater.Initialize(hc); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = $"https://github.com/actions/runner/notexists" })); - _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) - .Callback((int p, int a, string s, string t) => - { - hc.GetTrace().Info(t); - }) - .Returns(Task.FromResult(new TaskAgent())); + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); - var ex = await Assert.ThrowsAsync(() => updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken)); - Assert.Contains($"failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts", ex.Message); + var ex = await Assert.ThrowsAsync(() => updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken)); + Assert.Contains($"failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts", ex.Message); + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); } } @@ -145,33 +251,535 @@ namespace GitHub.Runner.Common.Tests.Listener [Trait("Category", "Runner")] public async void TestSelfUpdateAsync_ValidateHash() { - using (var hc = new TestHostContext(this)) + try { - //Arrange - var updater = new Runner.Listener.SelfUpdater(); - hc.SetSingleton(_term.Object); - hc.SetSingleton(_runnerServer.Object); - hc.SetSingleton(_configStore.Object); - hc.SetSingleton(new HttpClientHandlerFactory()); + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, HashValue = "bad_hash" })); + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); - var p = new ProcessInvokerWrapper(); - p.Initialize(hc); - hc.EnqueueInstance(p); - updater.Initialize(hc); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, HashValue = "bad_hash" })); - _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) - .Callback((int p, int a, string s, string t) => - { - hc.GetTrace().Info(t); - }) - .Returns(Task.FromResult(new TaskAgent())); + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); - var ex = await Assert.ThrowsAsync(() => updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken)); - Assert.Contains("did not match expected Runner Hash", ex.Message); + var ex = await Assert.ThrowsAsync(() => updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken)); + Assert.Contains("did not match expected Runner Hash", ex.Message); + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_CloneHash_RuntimeAndExternals() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = new List() { new TrimmedPackageMetadata() } })); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + try + { + var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.True(result); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + + FieldInfo contentHashesProperty = updater.GetType().GetField("_contentHashes", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); + Assert.NotNull(contentHashesProperty); + Dictionary contentHashes = (Dictionary)contentHashesProperty.GetValue(updater); + hc.GetTrace().Info(StringUtil.ConvertToJson(contentHashes)); + + var dotnetRuntimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}"); + var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}"); + + Assert.Equal(File.ReadAllText(dotnetRuntimeHashFile).Trim(), contentHashes["dotnetRuntime"]); + Assert.Equal(File.ReadAllText(externalsHashFile).Trim(), contentHashes["externals"]); + } + finally + { + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + } + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_Cancel_CloneHashTask_WhenNotNeeded() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new Mock().Object); + + var p1 = new ProcessInvokerWrapper(); + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); + p3.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + updater.Initialize(hc); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + try + { + var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + + FieldInfo contentHashesProperty = updater.GetType().GetField("_contentHashes", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); + Assert.NotNull(contentHashesProperty); + Dictionary contentHashes = (Dictionary)contentHashesProperty.GetValue(updater); + hc.GetTrace().Info(StringUtil.ConvertToJson(contentHashes)); + + Assert.NotEqual(2, contentHashes.Count); + } + catch (Exception ex) + { + hc.GetTrace().Error(ex); + } + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_UseExternalsTrimmedPackage() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); // hashfiles + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); // hashfiles + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); // un-tar + p3.Initialize(hc); + var p4 = new ProcessInvokerWrapper(); // node -v + p4.Initialize(hc); + var p5 = new ProcessInvokerWrapper(); // node -v + p5.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + hc.EnqueueInstance(p4); + hc.EnqueueInstance(p5); + updater.Initialize(hc); + + var trim = _trimmedPackages.Where(x => !x.TrimmedContents.ContainsKey("dotnetRuntime")).ToList(); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + try + { + var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.True(result); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + } + finally + { + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + } + + var traceFile = Path.GetTempFileName(); + File.Copy(hc.TraceFileName, traceFile, true); + + var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}"); + var externalsHash = await File.ReadAllTextAsync(externalsHashFile); + + if (externalsHash == trim[0].TrimmedContents["externals"]) + { + Assert.Contains("Use trimmed (externals) package", File.ReadAllText(traceFile)); + } + else + { + Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile)); + } + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_UseExternalsRuntimeTrimmedPackage() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); // hashfiles + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); // hashfiles + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); // un-tar + p3.Initialize(hc); + var p4 = new ProcessInvokerWrapper(); // node -v + p4.Initialize(hc); + var p5 = new ProcessInvokerWrapper(); // node -v + p5.Initialize(hc); + var p6 = new ProcessInvokerWrapper(); // runner -v + p6.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + hc.EnqueueInstance(p4); + hc.EnqueueInstance(p5); + hc.EnqueueInstance(p6); + updater.Initialize(hc); + + var trim = _trimmedPackages.Where(x => x.TrimmedContents.ContainsKey("dotnetRuntime") && x.TrimmedContents.ContainsKey("externals")).ToList(); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + try + { + var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.True(result); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + } + finally + { + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + } + + var traceFile = Path.GetTempFileName(); + File.Copy(hc.TraceFileName, traceFile, true); + + var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}"); + var externalsHash = await File.ReadAllTextAsync(externalsHashFile); + + var runtimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}"); + var runtimeHash = await File.ReadAllTextAsync(runtimeHashFile); + + if (externalsHash == trim[0].TrimmedContents["externals"] && + runtimeHash == trim[0].TrimmedContents["dotnetRuntime"]) + { + Assert.Contains("Use trimmed (runtime+externals) package", File.ReadAllText(traceFile)); + } + else + { + Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile)); + } + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_NotUseExternalsRuntimeTrimmedPackageOnHashMismatch() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); // hashfiles + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); // hashfiles + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); // un-tar + p3.Initialize(hc); + var p4 = new ProcessInvokerWrapper(); // node -v + p4.Initialize(hc); + var p5 = new ProcessInvokerWrapper(); // node -v + p5.Initialize(hc); + var p6 = new ProcessInvokerWrapper(); // runner -v + p6.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + hc.EnqueueInstance(p4); + hc.EnqueueInstance(p5); + hc.EnqueueInstance(p6); + updater.Initialize(hc); + + var trim = _trimmedPackages.ToList(); + foreach (var package in trim) + { + foreach (var hash in package.TrimmedContents.Keys) + { + package.TrimmedContents[hash] = "mismatch"; + } + } + + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + try + { + var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.True(result); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + } + finally + { + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + } + + var traceFile = Path.GetTempFileName(); + File.Copy(hc.TraceFileName, traceFile, true); + Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile)); + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestSelfUpdateAsync_FallbackToFullPackage() + { + try + { + await FetchLatestRunner(); + Assert.NotNull(_packageUrl); + Assert.NotNull(_trimmedPackages); + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin"))); + using (var hc = new TestHostContext(this)) + { + hc.GetTrace().Info(_packageUrl); + hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages)); + + //Arrange + var updater = new Runner.Listener.SelfUpdater(); + hc.SetSingleton(_term.Object); + hc.SetSingleton(_runnerServer.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(new HttpClientHandlerFactory()); + + var p1 = new ProcessInvokerWrapper(); // hashfiles + p1.Initialize(hc); + var p2 = new ProcessInvokerWrapper(); // hashfiles + p2.Initialize(hc); + var p3 = new ProcessInvokerWrapper(); // un-tar trim + p3.Initialize(hc); + var p4 = new ProcessInvokerWrapper(); // un-tar full + p4.Initialize(hc); + hc.EnqueueInstance(p1); + hc.EnqueueInstance(p2); + hc.EnqueueInstance(p3); + hc.EnqueueInstance(p4); + updater.Initialize(hc); + + var trim = _trimmedPackages.ToList(); + foreach (var package in trim) + { + package.HashValue = "mismatch"; + } + + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + + _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) + .Callback((int p, int a, string s, string t) => + { + hc.GetTrace().Info(t); + }) + .Returns(Task.FromResult(new TaskAgent())); + + try + { + var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); + Assert.True(result); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + } + finally + { + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + } + + var traceFile = Path.GetTempFileName(); + File.Copy(hc.TraceFileName, traceFile, true); + Assert.Contains("Something wrong with the trimmed runner package, failback to use the full package for runner updates", File.ReadAllText(traceFile)); + } + } + finally + { + Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null); } } } diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index 546b3cc8e..ce1ec01bc 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -165,7 +165,15 @@ namespace GitHub.Runner.Common.Tests switch (directory) { case WellKnownDirectory.Bin: - path = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); + var overwriteBinDir = Environment.GetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR"); + if (Directory.Exists(overwriteBinDir)) + { + path = overwriteBinDir; + } + else + { + path = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); + } break; case WellKnownDirectory.Diag: