Remove runner trimmed packages. (#3074)

This commit is contained in:
Tingluo Huang
2024-01-10 12:32:17 -05:00
committed by GitHub
parent ac39c4bd0a
commit f1f18f67e1
15 changed files with 6 additions and 2451 deletions

View File

@@ -23,7 +23,6 @@ namespace GitHub.Runner.Common.Tests.Listener
private Mock<IConfigurationStore> _configStore;
private Mock<IJobDispatcher> _jobDispatcher;
private AgentRefreshMessage _refreshMessage = new(1, "2.999.0");
private List<TrimmedPackageMetadata> _trimmedPackages = new();
#if !OS_WINDOWS
private string _packageUrl = null;
@@ -71,12 +70,6 @@ namespace GitHub.Runner.Common.Tests.Listener
}
}
using (var client = new HttpClient())
{
var json = await client.GetStringAsync($"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}-trimmedpackages.json");
_trimmedPackages = StringUtil.ConvertFromJson<List<TrimmedPackageMetadata>>(json);
}
_runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl }));
@@ -91,12 +84,10 @@ namespace GitHub.Runner.Common.Tests.Listener
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
@@ -152,12 +143,10 @@ namespace GitHub.Runner.Common.Tests.Listener
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
@@ -205,12 +194,10 @@ namespace GitHub.Runner.Common.Tests.Listener
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
@@ -260,12 +247,10 @@ namespace GitHub.Runner.Common.Tests.Listener
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
@@ -305,495 +290,6 @@ namespace GitHub.Runner.Common.Tests.Listener
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void TestSelfUpdateAsync_CloneHash_RuntimeAndExternals()
{
try
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
hc.SetSingleton<ITerminal>(_term.Object);
hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
hc.SetSingleton<IConfigurationStore>(_configStore.Object);
hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
var p1 = new ProcessInvokerWrapper();
p1.Initialize(hc);
var p2 = new ProcessInvokerWrapper();
p2.Initialize(hc);
var p3 = new ProcessInvokerWrapper();
p3.Initialize(hc);
hc.EnqueueInstance<IProcessInvoker>(p1);
hc.EnqueueInstance<IProcessInvoker>(p2);
hc.EnqueueInstance<IProcessInvoker>(p3);
updater.Initialize(hc);
_runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = new List<TrimmedPackageMetadata>() { new TrimmedPackageMetadata() } }));
_runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
.Callback((int p, ulong a, string s, string t) =>
{
hc.GetTrace().Info(t);
})
.Returns(Task.FromResult(new TaskAgent()));
try
{
var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
Assert.True(result);
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0")));
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0")));
FieldInfo contentHashesProperty = updater.GetType().GetField("_contentHashes", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(contentHashesProperty);
Dictionary<string, string> contentHashes = (Dictionary<string, string>)contentHashesProperty.GetValue(updater);
hc.GetTrace().Info(StringUtil.ConvertToJson(contentHashes));
var dotnetRuntimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}");
var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
Assert.Equal(File.ReadAllText(dotnetRuntimeHashFile).Trim(), contentHashes["dotnetRuntime"]);
Assert.Equal(File.ReadAllText(externalsHashFile).Trim(), contentHashes["externals"]);
}
finally
{
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None);
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None);
}
}
}
finally
{
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void TestSelfUpdateAsync_Cancel_CloneHashTask_WhenNotNeeded()
{
try
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
hc.SetSingleton<ITerminal>(_term.Object);
hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
hc.SetSingleton<IConfigurationStore>(_configStore.Object);
hc.SetSingleton<IHttpClientHandlerFactory>(new Mock<IHttpClientHandlerFactory>().Object);
var p1 = new ProcessInvokerWrapper();
p1.Initialize(hc);
var p2 = new ProcessInvokerWrapper();
p2.Initialize(hc);
var p3 = new ProcessInvokerWrapper();
p3.Initialize(hc);
hc.EnqueueInstance<IProcessInvoker>(p1);
hc.EnqueueInstance<IProcessInvoker>(p2);
hc.EnqueueInstance<IProcessInvoker>(p3);
updater.Initialize(hc);
_runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
.Callback((int p, ulong a, string s, string t) =>
{
hc.GetTrace().Info(t);
})
.Returns(Task.FromResult(new TaskAgent()));
try
{
var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
FieldInfo contentHashesProperty = updater.GetType().GetField("_contentHashes", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(contentHashesProperty);
Dictionary<string, string> contentHashes = (Dictionary<string, string>)contentHashesProperty.GetValue(updater);
hc.GetTrace().Info(StringUtil.ConvertToJson(contentHashes));
Assert.NotEqual(2, contentHashes.Count);
}
catch (Exception ex)
{
hc.GetTrace().Error(ex);
}
}
}
finally
{
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void TestSelfUpdateAsync_UseExternalsTrimmedPackage()
{
try
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
hc.SetSingleton<ITerminal>(_term.Object);
hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
hc.SetSingleton<IConfigurationStore>(_configStore.Object);
hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
var p1 = new ProcessInvokerWrapper(); // hashfiles
p1.Initialize(hc);
var p2 = new ProcessInvokerWrapper(); // hashfiles
p2.Initialize(hc);
var p3 = new ProcessInvokerWrapper(); // un-tar
p3.Initialize(hc);
var p4 = new ProcessInvokerWrapper(); // node -v
p4.Initialize(hc);
var p5 = new ProcessInvokerWrapper(); // node -v
p5.Initialize(hc);
hc.EnqueueInstance<IProcessInvoker>(p1);
hc.EnqueueInstance<IProcessInvoker>(p2);
hc.EnqueueInstance<IProcessInvoker>(p3);
hc.EnqueueInstance<IProcessInvoker>(p4);
hc.EnqueueInstance<IProcessInvoker>(p5);
updater.Initialize(hc);
var trim = _trimmedPackages.Where(x => !x.TrimmedContents.ContainsKey("dotnetRuntime")).ToList();
_runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
_runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
.Callback((int p, ulong a, string s, string t) =>
{
hc.GetTrace().Info(t);
})
.Returns(Task.FromResult(new TaskAgent()));
try
{
var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
Assert.True(result);
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0")));
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0")));
}
finally
{
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None);
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None);
}
var traceFile = Path.GetTempFileName();
File.Copy(hc.TraceFileName, traceFile, true);
var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
var externalsHash = await File.ReadAllTextAsync(externalsHashFile);
if (externalsHash == trim[0].TrimmedContents["externals"])
{
Assert.Contains("Use trimmed (externals) package", File.ReadAllText(traceFile));
}
else
{
Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile));
}
}
}
finally
{
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void TestSelfUpdateAsync_UseExternalsRuntimeTrimmedPackage()
{
try
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
hc.SetSingleton<ITerminal>(_term.Object);
hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
hc.SetSingleton<IConfigurationStore>(_configStore.Object);
hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
var p1 = new ProcessInvokerWrapper(); // hashfiles
p1.Initialize(hc);
var p2 = new ProcessInvokerWrapper(); // hashfiles
p2.Initialize(hc);
var p3 = new ProcessInvokerWrapper(); // un-tar
p3.Initialize(hc);
var p4 = new ProcessInvokerWrapper(); // node -v
p4.Initialize(hc);
var p5 = new ProcessInvokerWrapper(); // node -v
p5.Initialize(hc);
var p6 = new ProcessInvokerWrapper(); // runner -v
p6.Initialize(hc);
hc.EnqueueInstance<IProcessInvoker>(p1);
hc.EnqueueInstance<IProcessInvoker>(p2);
hc.EnqueueInstance<IProcessInvoker>(p3);
hc.EnqueueInstance<IProcessInvoker>(p4);
hc.EnqueueInstance<IProcessInvoker>(p5);
hc.EnqueueInstance<IProcessInvoker>(p6);
updater.Initialize(hc);
var trim = _trimmedPackages.Where(x => x.TrimmedContents.ContainsKey("dotnetRuntime") && x.TrimmedContents.ContainsKey("externals")).ToList();
_runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
_runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
.Callback((int p, ulong a, string s, string t) =>
{
hc.GetTrace().Info(t);
})
.Returns(Task.FromResult(new TaskAgent()));
try
{
var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
Assert.True(result);
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0")));
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0")));
}
finally
{
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None);
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None);
}
var traceFile = Path.GetTempFileName();
File.Copy(hc.TraceFileName, traceFile, true);
var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
var externalsHash = await File.ReadAllTextAsync(externalsHashFile);
var runtimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}");
var runtimeHash = await File.ReadAllTextAsync(runtimeHashFile);
if (externalsHash == trim[0].TrimmedContents["externals"] &&
runtimeHash == trim[0].TrimmedContents["dotnetRuntime"])
{
Assert.Contains("Use trimmed (runtime+externals) package", File.ReadAllText(traceFile));
}
else
{
Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile));
}
}
}
finally
{
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void TestSelfUpdateAsync_NotUseExternalsRuntimeTrimmedPackageOnHashMismatch()
{
try
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
hc.SetSingleton<ITerminal>(_term.Object);
hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
hc.SetSingleton<IConfigurationStore>(_configStore.Object);
hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
var p1 = new ProcessInvokerWrapper(); // hashfiles
p1.Initialize(hc);
var p2 = new ProcessInvokerWrapper(); // hashfiles
p2.Initialize(hc);
var p3 = new ProcessInvokerWrapper(); // un-tar
p3.Initialize(hc);
var p4 = new ProcessInvokerWrapper(); // node -v
p4.Initialize(hc);
var p5 = new ProcessInvokerWrapper(); // node -v
p5.Initialize(hc);
var p6 = new ProcessInvokerWrapper(); // runner -v
p6.Initialize(hc);
hc.EnqueueInstance<IProcessInvoker>(p1);
hc.EnqueueInstance<IProcessInvoker>(p2);
hc.EnqueueInstance<IProcessInvoker>(p3);
hc.EnqueueInstance<IProcessInvoker>(p4);
hc.EnqueueInstance<IProcessInvoker>(p5);
hc.EnqueueInstance<IProcessInvoker>(p6);
updater.Initialize(hc);
var trim = _trimmedPackages.ToList();
foreach (var package in trim)
{
foreach (var hash in package.TrimmedContents.Keys)
{
package.TrimmedContents[hash] = "mismatch";
}
}
_runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
_runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
.Callback((int p, ulong a, string s, string t) =>
{
hc.GetTrace().Info(t);
})
.Returns(Task.FromResult(new TaskAgent()));
try
{
var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
Assert.True(result);
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0")));
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0")));
}
finally
{
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None);
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None);
}
var traceFile = Path.GetTempFileName();
File.Copy(hc.TraceFileName, traceFile, true);
Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile));
}
}
finally
{
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void TestSelfUpdateAsync_FallbackToFullPackage()
{
try
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdater();
hc.SetSingleton<ITerminal>(_term.Object);
hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
hc.SetSingleton<IConfigurationStore>(_configStore.Object);
hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
var p1 = new ProcessInvokerWrapper(); // hashfiles
p1.Initialize(hc);
var p2 = new ProcessInvokerWrapper(); // hashfiles
p2.Initialize(hc);
var p3 = new ProcessInvokerWrapper(); // un-tar trim
p3.Initialize(hc);
var p4 = new ProcessInvokerWrapper(); // un-tar full
p4.Initialize(hc);
hc.EnqueueInstance<IProcessInvoker>(p1);
hc.EnqueueInstance<IProcessInvoker>(p2);
hc.EnqueueInstance<IProcessInvoker>(p3);
hc.EnqueueInstance<IProcessInvoker>(p4);
updater.Initialize(hc);
var trim = _trimmedPackages.ToList();
foreach (var package in trim)
{
package.HashValue = "mismatch";
}
_runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
_runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
.Callback((int p, ulong a, string s, string t) =>
{
hc.GetTrace().Info(t);
})
.Returns(Task.FromResult(new TaskAgent()));
try
{
var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
Assert.True(result);
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0")));
Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0")));
}
finally
{
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None);
IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None);
}
var traceFile = Path.GetTempFileName();
File.Copy(hc.TraceFileName, traceFile, true);
if (File.ReadAllText(traceFile).Contains("Use trimmed (runtime+externals) package"))
{
Assert.Contains("Something wrong with the trimmed runner package, failback to use the full package for runner updates", File.ReadAllText(traceFile));
}
else
{
hc.GetTrace().Warning("Skipping the 'TestSelfUpdateAsync_FallbackToFullPackage' test, as the `externals` or `runtime` hashes have been updated");
}
}
}
finally
{
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
}
}
}
}
#endif

View File

@@ -23,7 +23,6 @@ namespace GitHub.Runner.Common.Tests.Listener
private Mock<IConfigurationStore> _configStore;
private Mock<IJobDispatcher> _jobDispatcher;
private AgentRefreshMessage _refreshMessage = new(1, "2.999.0");
private List<TrimmedPackageMetadata> _trimmedPackages = new();
#if !OS_WINDOWS
private string _packageUrl = null;
@@ -81,12 +80,10 @@ namespace GitHub.Runner.Common.Tests.Listener
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdaterV2();
@@ -143,12 +140,10 @@ namespace GitHub.Runner.Common.Tests.Listener
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdaterV2();
@@ -194,12 +189,10 @@ namespace GitHub.Runner.Common.Tests.Listener
{
await FetchLatestRunner();
Assert.NotNull(_packageUrl);
Assert.NotNull(_trimmedPackages);
Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
using (var hc = new TestHostContext(this))
{
hc.GetTrace().Info(_packageUrl);
hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
//Arrange
var updater = new Runner.Listener.SelfUpdaterV2();

View File

@@ -1,301 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using Xunit;
namespace GitHub.Runner.Common.Tests
{
public sealed class PackagesTrimL0
{
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task RunnerLayoutParts_NewFilesCrossAll()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
var runnerCoreAssetsFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnercoreassets");
var runnerDotnetRuntimeFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnerdotnetruntimeassets");
string layoutBin = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/bin");
var newFiles = new List<string>();
if (Directory.Exists(layoutBin))
{
var coreAssets = await File.ReadAllLinesAsync(runnerCoreAssetsFile);
var runtimeAssets = await File.ReadAllLinesAsync(runnerDotnetRuntimeFile);
foreach (var file in Directory.GetFiles(layoutBin, "*", SearchOption.AllDirectories))
{
if (!coreAssets.Any(x => file.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).EndsWith(x)) &&
!runtimeAssets.Any(x => file.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).EndsWith(x)))
{
newFiles.Add(file);
}
}
if (newFiles.Count > 0)
{
Assert.True(false, $"Found new files '{string.Join('\n', newFiles)}'. These will break runner update using trimmed packages.");
}
}
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task RunnerLayoutParts_OverlapFiles()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
var runnerCoreAssetsFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnercoreassets");
var runnerDotnetRuntimeFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnerdotnetruntimeassets");
var coreAssets = await File.ReadAllLinesAsync(runnerCoreAssetsFile);
var runtimeAssets = await File.ReadAllLinesAsync(runnerDotnetRuntimeFile);
foreach (var line in coreAssets)
{
if (runtimeAssets.Contains(line, StringComparer.OrdinalIgnoreCase))
{
Assert.True(false, $"'Misc/runnercoreassets' and 'Misc/runnerdotnetruntimeassets' should not overlap.");
}
}
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task RunnerLayoutParts_NewRunnerCoreAssets()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
var runnerCoreAssetsFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnercoreassets");
var coreAssets = await File.ReadAllLinesAsync(runnerCoreAssetsFile);
string layoutBin = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/bin");
var newFiles = new List<string>();
if (Directory.Exists(layoutBin))
{
var binDirs = Directory.GetDirectories(TestUtil.GetSrcPath(), "net6.0", SearchOption.AllDirectories);
foreach (var binDir in binDirs)
{
if (binDir.Contains("Test") || binDir.Contains("obj"))
{
continue;
}
Directory.GetFiles(binDir, "*", SearchOption.TopDirectoryOnly).ToList().ForEach(x =>
{
if (!x.Contains("runtimeconfig.dev.json"))
{
if (!coreAssets.Any(y => x.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).EndsWith(y)))
{
newFiles.Add(x);
}
}
});
}
if (newFiles.Count > 0)
{
Assert.True(false, $"Found new files '{string.Join('\n', newFiles)}'. These will break runner update using trimmed packages. You might need to update `Misc/runnercoreassets`.");
}
}
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task RunnerLayoutParts_NewDotnetRuntimeAssets()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
var runnerDotnetRuntimeFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnerdotnetruntimeassets");
var runtimeAssets = await File.ReadAllLinesAsync(runnerDotnetRuntimeFile);
string layoutTrimsRuntimeAssets = Path.Combine(TestUtil.GetSrcPath(), @"../_layout_trims/runnerdotnetruntimeassets");
var newFiles = new List<string>();
if (File.Exists(layoutTrimsRuntimeAssets))
{
var runtimeAssetsCurrent = await File.ReadAllLinesAsync(layoutTrimsRuntimeAssets);
foreach (var runtimeFile in runtimeAssetsCurrent)
{
if (runtimeAssets.Any(x => runtimeFile.EndsWith(x, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
else
{
newFiles.Add(runtimeFile);
}
}
if (newFiles.Count > 0)
{
Assert.True(false, $"Found new dotnet runtime files '{string.Join('\n', newFiles)}'. These will break runner update using trimmed packages. You might need to update `Misc/runnerdotnetruntimeassets`.");
}
}
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task RunnerLayoutParts_CheckDotnetRuntimeHash()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
var dotnetRuntimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}");
trace.Info($"Current hash: {File.ReadAllText(dotnetRuntimeHashFile)}");
string layoutTrimsRuntimeAssets = Path.Combine(TestUtil.GetSrcPath(), @"../_layout_trims/runtime");
string binDir = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/bin");
#if OS_WINDOWS
string node = Path.Combine(TestUtil.GetSrcPath(), @"..\_layout\externals\node16\bin\node");
#else
string node = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/externals/node16/bin/node");
#endif
string hashFilesScript = Path.Combine(binDir, "hashFiles");
var hashResult = string.Empty;
var p1 = new ProcessInvokerWrapper();
p1.Initialize(hc);
p1.ErrorDataReceived += (_, data) =>
{
if (!string.IsNullOrEmpty(data.Data) && data.Data.StartsWith("__OUTPUT__") && data.Data.EndsWith("__OUTPUT__"))
{
hashResult = data.Data.Substring(10, data.Data.Length - 20);
trace.Info($"Hash result: '{hashResult}'");
}
else
{
trace.Info(data.Data);
}
};
p1.OutputDataReceived += (_, data) =>
{
trace.Info(data.Data);
};
var env = new Dictionary<string, string>
{
["patterns"] = "**"
};
int exitCode = await p1.ExecuteAsync(workingDirectory: layoutTrimsRuntimeAssets,
fileName: node,
arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"",
environment: env,
requireExitCodeZero: true,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: CancellationToken.None);
Assert.True(string.Equals(hashResult, File.ReadAllText(dotnetRuntimeHashFile).Trim()), $"Hash mismatch for dotnet runtime. You might need to update `Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}` or check if `hashFiles.ts` ever changed recently.");
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task RunnerLayoutParts_CheckExternalsHash()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
trace.Info($"Current hash: {File.ReadAllText(externalsHashFile)}");
string layoutTrimsExternalsAssets = Path.Combine(TestUtil.GetSrcPath(), @"../_layout_trims/externals");
string binDir = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/bin");
#if OS_WINDOWS
string node = Path.Combine(TestUtil.GetSrcPath(), @"..\_layout\externals\node16\bin\node");
#else
string node = Path.Combine(TestUtil.GetSrcPath(), @"../_layout/externals/node16/bin/node");
#endif
string hashFilesScript = Path.Combine(binDir, "hashFiles");
var hashResult = string.Empty;
var p1 = new ProcessInvokerWrapper();
p1.Initialize(hc);
p1.ErrorDataReceived += (_, data) =>
{
if (!string.IsNullOrEmpty(data.Data) && data.Data.StartsWith("__OUTPUT__") && data.Data.EndsWith("__OUTPUT__"))
{
hashResult = data.Data.Substring(10, data.Data.Length - 20);
trace.Info($"Hash result: '{hashResult}'");
}
else
{
trace.Info(data.Data);
}
};
p1.OutputDataReceived += (_, data) =>
{
trace.Info(data.Data);
};
var env = new Dictionary<string, string>
{
["patterns"] = "**"
};
int exitCode = await p1.ExecuteAsync(workingDirectory: layoutTrimsExternalsAssets,
fileName: node,
arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"",
environment: env,
requireExitCodeZero: true,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: CancellationToken.None);
Assert.True(string.Equals(hashResult, File.ReadAllText(externalsHashFile).Trim()), $"Hash mismatch for externals. You might need to update `Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}` or check if `hashFiles.ts` ever changed recently.");
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public Task RunnerLayoutParts_ContentHashFilesNoNewline()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
var dotnetRuntimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}");
var dotnetRuntimeHash = File.ReadAllText(dotnetRuntimeHashFile);
trace.Info($"Current hash: {dotnetRuntimeHash}");
var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
var externalsHash = File.ReadAllText(externalsHashFile);
trace.Info($"Current hash: {externalsHash}");
Assert.False(externalsHash.Any(x => char.IsWhiteSpace(x)), $"Found whitespace in externals hash file.");
Assert.False(dotnetRuntimeHash.Any(x => char.IsWhiteSpace(x)), $"Found whitespace in dotnet runtime hash file.");
return Task.CompletedTask;
}
}
}
}