Use hardcoded docker override

98aa9c1152

.github/workflows/release.yml

@@ -101,11 +101,11 @@ jobs:
       working-directory: src
 
     # Run tests
-    # - name: L0
+    - name: L0
-    #   run: |
+      run: |
-    #     ${{ matrix.devScript }} test
+        ${{ matrix.devScript }} test
-    #   working-directory: src
+      working-directory: src
-    #   if: matrix.runtime != 'linux-arm64' && matrix.runtime != 'linux-arm'
+      if: matrix.runtime != 'linux-arm64' && matrix.runtime != 'linux-arm'
 
     # Create runner package tar.gz/zip
     - name: Package Release
@@ -282,7 +282,6 @@ jobs:
         release_name: "v${{ steps.releaseNote.outputs.version }}"
         body: |
           ${{ steps.releaseNote.outputs.note }}
-        prerelease: true
 
     # Upload release assets (full runner packages)
     - name: Upload Release Asset (win-x64)

releaseNote.md

@@ -1,8 +1,11 @@
 ## Features
 
 ## Bugs
-- Fixed an issue where container environment variables names or values could escape the docker command (#2108)
+- Fixed a crash on runner startup (#1770)
-- Sanitize Windows ENVs (#2280)
+
+## Misc
+
+- Clarified the type of step running when running job started or completed hooks (#1769)
 
 
 ## Windows x64

releaseVersion

@@ -1 +1 @@
-2.289.5
+<Update to ./src/runnerversion when creating release>

src/Misc/layoutroot/run-helper.cmd.template

@@ -0,0 +1,39 @@
+@echo off
+
+"%~dp0\bin\Runner.Listener.exe" run %*
+
+rem using `if %ERRORLEVEL% EQU N` insterad of `if ERRORLEVEL N`
+rem `if ERRORLEVEL N` means: error level is N or MORE
+  
+if %ERRORLEVEL% EQU 0 (
+  echo "Runner listener exit with 0 return code, stop the service, no retry needed."
+  exit /b 0
+)
+
+if %ERRORLEVEL% EQU 1 (
+  echo "Runner listener exit with terminated error, stop the service, no retry needed."
+  exit /b 0
+)
+
+if %ERRORLEVEL% EQU 2 (
+  echo "Runner listener exit with retryable error, re-launch runner in 5 seconds."
+  ping 127.0.0.1 -n 6 -w 1000 >NUL
+  exit /b 1
+)
+
+if %ERRORLEVEL% EQU 3 (
+  rem Sleep 5 seconds to wait for the runner update process finish
+  echo "Runner listener exit because of updating, re-launch runner in 5 seconds"
+  ping 127.0.0.1 -n 6 -w 1000 >NUL
+  exit /b 1
+)
+
+if %ERRORLEVEL% EQU 4 (
+  rem Sleep 5 seconds to wait for the ephemeral runner update process finish
+  echo "Runner listener exit because of updating, re-launch ephemeral runner in 5 seconds"
+  ping 127.0.0.1 -n 6 -w 1000 >NUL
+  exit /b 1
+)
+
+echo "Exiting after unknown error code: %ERRORLEVEL%"
+exit /b 0

src/Misc/layoutroot/run-helper.sh.template

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Validate not sudo
+user_id=`id -u`
+if [ $user_id -eq 0 -a -z "$RUNNER_ALLOW_RUNASROOT" ]; then
+    echo "Must not run interactively with sudo"
+    exit 1
+fi
+
+# Run
+shopt -s nocasematch
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+    DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+    SOURCE="$(readlink "$SOURCE")"
+    [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+"$DIR"/bin/Runner.Listener run $*
+
+returnCode=$?
+if [[ $returnCode == 0 ]]; then
+    echo "Runner listener exit with 0 return code, stop the service, no retry needed."
+    exit 0
+elif [[ $returnCode == 1 ]]; then
+    echo "Runner listener exit with terminated error, stop the service, no retry needed."
+    exit 0
+elif [[ $returnCode == 2 ]]; then
+    echo "Runner listener exit with retryable error, re-launch runner in 5 seconds."
+    "$DIR"/safe_sleep.sh 5
+    exit 2
+elif [[ $returnCode == 3 ]]; then
+    # Sleep 5 seconds to wait for the runner update process finish
+    echo "Runner listener exit because of updating, re-launch runner in 5 seconds"
+    "$DIR"/safe_sleep.sh 5
+    exit 2
+elif [[ $returnCode == 4 ]]; then
+    # Sleep 5 seconds to wait for the ephemeral runner update process finish
+    echo "Runner listener exit because of updating, re-launch ephemeral runner in 5 seconds"
+    "$DIR"/safe_sleep.sh 5
+    exit 2
+else
+    echo "Exiting with unknown error code: ${returnCode}"
+    exit 0
+fi

src/Misc/layoutroot/run.cmd

@@ -13,21 +13,19 @@ if defined VERBOSE_ARG (
 rem Unblock files in the root of the layout folder. E.g. .cmd files.
 powershell.exe -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "$VerbosePreference = %VERBOSE_ARG% ; Get-ChildItem -LiteralPath '%~dp0' | ForEach-Object { Write-Verbose ('Unblock: {0}' -f $_.FullName) ; $_ } | Unblock-File | Out-Null"
 
-if /i "%~1" equ "localRun" (
+
-    rem ********************************************************************************
-    rem Local run.
-    rem ********************************************************************************
-    "%~dp0bin\Runner.Listener.exe" %*
-) else (
 rem ********************************************************************************
 rem Run.
 rem ********************************************************************************
-  "%~dp0bin\Runner.Listener.exe" run %*
 
-  rem Return code 4 means the run once runner received an update message.
+:launch_helper
-  rem Sleep 5 seconds to wait for the update process finish and run the runner again.
+copy "%~dp0run-helper.cmd.template" "%~dp0run-helper.cmd" /Y
-  if ERRORLEVEL 4 (
+call "%~dp0run-helper.cmd" %*
-    timeout /t 5 /nobreak > NUL
+  
-    "%~dp0bin\Runner.Listener.exe" run %*
+if %ERRORLEVEL% EQU 1 (
-  )
+  echo "Restarting runner..."
+  goto :launch_helper
+) else (  
+  echo "Exiting runner..."
+  exit /b 0
 )

src/Misc/layoutroot/run.sh

@@ -1,12 +1,5 @@
 #!/bin/bash
 
-# Validate not sudo
-user_id=`id -u`
-if [ $user_id -eq 0 -a -z "$RUNNER_ALLOW_RUNASROOT" ]; then
-    echo "Must not run interactively with sudo"
-    exit 1
-fi
-
 # Change directory to the script root directory
 # https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
 SOURCE="${BASH_SOURCE[0]}"
@@ -16,49 +9,16 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
     [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
+cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh
-# Do not "cd $DIR". For localRun, the current directory is expected to be the repo location on disk.
+# run the helper process which keep the listener alive
-
+while :;
-# Run
+do
-shopt -s nocasematch
+    "$DIR"/run-helper.sh $*
-if [[ "$1" == "localRun" ]]; then
-    "$DIR"/bin/Runner.Listener $*
-else
-    "$DIR"/bin/Runner.Listener run $*
-
-# Return code 3 means the run once runner received an update message.
-# Sleep 5 seconds to wait for the update process finish
     returnCode=$?
-    if [[ $returnCode == 3 ]]; then
+    if [[ $returnCode -eq 2 ]]; then
-        if [ ! -x "$(command -v sleep)" ]; then
+        echo "Restarting runner..."
-            if [ ! -x "$(command -v ping)" ]; then
+    else
-                COUNT="0"
+        echo "Exiting runner..."
-                while [[ $COUNT != 5000 ]]; do
+        exit 0
-                    echo "SLEEP" > /dev/null
+    fi
-                    COUNT=$[$COUNT+1]
 done
-            else
-                ping -c 5 127.0.0.1 > /dev/null
-            fi
-        else
-            sleep 5
-        fi
-    elif [[ $returnCode == 4 ]]; then
-        if [ ! -x "$(command -v sleep)" ]; then
-            if [ ! -x "$(command -v ping)" ]; then
-                COUNT="0"
-                while [[ $COUNT != 5000 ]]; do
-                    echo "SLEEP" > /dev/null
-                    COUNT=$[$COUNT+1]
-                done
-            else
-                ping -c 5 127.0.0.1 > /dev/null
-            fi
-        else
-            sleep 5
-        fi
-        "$DIR"/bin/Runner.Listener run $*
-    else
-        exit $returnCode
-    fi
-fi

src/Runner.Common/JobServer.cs

@@ -143,10 +143,8 @@ namespace GitHub.Runner.Common
 
         public ValueTask DisposeAsync()
         {
-            CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);
+            _websocketClient?.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "Shutdown", CancellationToken.None);
-
             GC.SuppressFinalize(this);
-
             return ValueTask.CompletedTask;
         }
 
@@ -250,8 +248,7 @@ namespace GitHub.Runner.Common
                         if (failedAttemptsToPostBatchedLinesByWebsocket * 100 / totalBatchedLinesAttemptedByWebsocket > _minWebsocketFailurePercentageAllowed)
                         {
                             Trace.Info($"Exhausted websocket allowed retries, we will not attempt websocket connection for this job to post lines again.");
-                            CloseWebSocket(WebSocketCloseStatus.InternalServerError, cancellationToken);
+                            _websocketClient?.CloseOutputAsync(WebSocketCloseStatus.InternalServerError, "Shutdown due to failures", cancellationToken);
-
                             // By setting it to null, we will ensure that we never try websocket path again for this job
                             _websocketClient = null;
                         }
@@ -279,19 +276,6 @@ namespace GitHub.Runner.Common
             }
         }
 
-        private void CloseWebSocket(WebSocketCloseStatus closeStatus, CancellationToken cancellationToken)
-        {
-            try
-            {
-                _websocketClient?.CloseOutputAsync(closeStatus, "Closing websocket", cancellationToken);
-            }
-            catch (Exception websocketEx)
-            {
-                // In some cases this might be okay since the websocket might be open yet, so just close and don't trace exceptions
-                Trace.Info($"Failed to close websocket gracefully {websocketEx.GetType().Name}");
-            }
-        }
-
         public Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, string type, string name, Stream uploadStream, CancellationToken cancellationToken)
         {
             CheckConnection();

src/Runner.Listener/Configuration/ConfigurationManager.cs

@@ -54,7 +54,7 @@ namespace GitHub.Runner.Listener.Configuration
             Trace.Info(nameof(LoadSettings));
             if (!IsConfigured())
             {
-                throw new InvalidOperationException("Not configured. Run config.(sh/cmd) to configure the runner.");
+                throw new NonRetryableException("Not configured. Run config.(sh/cmd) to configure the runner.");
             }
 
             RunnerSettings settings = _store.GetSettings();

src/Runner.Listener/Runner.cs

@@ -430,7 +430,7 @@ namespace GitHub.Runner.Listener
                                     }
 #endif
                                     var selfUpdater = HostContext.GetService<ISelfUpdater>();
-                                    selfUpdateTask = selfUpdater.SelfUpdate(runnerUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.RunnerShutdownToken);
+                                    selfUpdateTask = selfUpdater.SelfUpdate(runnerUpdateMessage, jobDispatcher, false, HostContext.RunnerShutdownToken);
                                     Trace.Info("Refresh message received, kick-off selfupdate background process.");
                                 }
                                 else

src/Runner.Sdk/ProcessInvoker.cs

@@ -264,17 +264,7 @@ namespace GitHub.Runner.Sdk
             {
                 foreach (KeyValuePair<string, string> kvp in environment)
                 {
-#if OS_WINDOWS
-                    string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : kvp.Key.Split('\0')[0];
-                    string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : kvp.Value.Split('\0')[0];
-                    if(!String.IsNullOrWhiteSpace(tempKey))
-                    {
-                         _proc.StartInfo.Environment[tempKey] = tempValue;
-                    }
-#else
                     _proc.StartInfo.Environment[kvp.Key] = kvp.Value;
-
-#endif
                 }
             }
 

src/Runner.Worker/Container/DockerCommandManager.cs

@@ -11,7 +11,7 @@ using GitHub.Runner.Sdk;
 
 namespace GitHub.Runner.Worker.Container
 {
-    [ServiceLocator(Default = typeof(DockerCommandManager))]
+    [ServiceLocator(Default = typeof(DockerHookCommandManager))]
     public interface IDockerCommandManager : IRunnerService
     {
         string DockerPath { get; }
@@ -131,11 +131,11 @@ namespace GitHub.Runner.Worker.Container
             {
                 if (String.IsNullOrEmpty(env.Value))
                 {
-                    dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key));
+                    dockerOptions.Add($"-e \"{env.Key}\"");
                 }
                 else
                 {
-                    dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key, env.Value));
+                    dockerOptions.Add($"-e \"{env.Key}={env.Value.Replace("\"", "\\\"")}\"");
                 }
             }
 
@@ -188,7 +188,7 @@ namespace GitHub.Runner.Worker.Container
             return outputStrings.FirstOrDefault();
         }
 
-        public async Task<int> DockerRun(IExecutionContext context, ContainerInfo container, EventHandler<ProcessDataReceivedEventArgs> stdoutDataReceived, EventHandler<ProcessDataReceivedEventArgs> stderrDataReceived)
+        public virtual async Task<int> DockerRun(IExecutionContext context, ContainerInfo container, EventHandler<ProcessDataReceivedEventArgs> stdoutDataReceived, EventHandler<ProcessDataReceivedEventArgs> stderrDataReceived)
         {
             IList<string> dockerOptions = new List<string>();
             // OPTIONS
@@ -202,7 +202,7 @@ namespace GitHub.Runner.Worker.Container
             {
                 // e.g. -e MY_SECRET maps the value into the exec'ed process without exposing
                 // the value directly in the command
-                dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key));
+                dockerOptions.Add($"-e {env.Key}");
             }
 
             // Watermark for GitHub Action environment
@@ -258,7 +258,7 @@ namespace GitHub.Runner.Worker.Container
             return await ExecuteDockerCommandAsync(context, "run", optionsString, container.ContainerEnvironmentVariables, stdoutDataReceived, stderrDataReceived, context.CancellationToken);
         }
 
-        public async Task<int> DockerStart(IExecutionContext context, string containerId)
+        public virtual async Task<int> DockerStart(IExecutionContext context, string containerId)
         {
             return await ExecuteDockerCommandAsync(context, "start", containerId, context.CancellationToken);
         }

src/Runner.Worker/Container/DockerHookCommandManager.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.Pipelines;
+using GitHub.Runner.Common.Util;
+using GitHub.Runner.Worker.Handlers;
+
+namespace GitHub.Runner.Worker.Container
+{
+    public class DockerHookCommandManager : DockerCommandManager
+    {
+        public override async Task<int> DockerStart(IExecutionContext context, string containerId)
+        {
+            // check for env var
+            // execute script
+            
+            // Create the handler data.
+            var path = "/home/ferenc/Documents/runner/_layout/docker_run.sh";
+            var scriptDirectory = Path.GetDirectoryName(path);
+            var stepHost = HostContext.CreateService<IDefaultStepHost>();
+            var prependPath = string.Join(Path.PathSeparator.ToString(), context.Global.PrependPath.Reverse<string>());
+            Dictionary<string, string> inputs = new()
+            {
+                ["script"] = $"CONT_ID={containerId} " + "/usr/bin/bash" + " " + path,
+                // /bin/bash
+                ["shell"] = ScriptHandlerHelpers.GetDefaultShellForScript(path, Trace, prependPath)
+            };
+
+            // Create the handler
+            var handlerFactory = HostContext.GetService<IHandlerFactory>();
+            var handler = handlerFactory.Create(
+                            context,
+                            action: new ScriptReference(),
+                            stepHost,
+                            new ScriptActionExecutionData(),
+                            inputs,
+                            environment: new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer),
+                            context.Global.Variables,
+                            actionDirectory: scriptDirectory,
+                            localActionContainerSetupSteps: null);
+            handler.PrepareExecution(ActionRunStage.Main); // TODO: find out stage
+
+            await handler.RunAsync(ActionRunStage.Main);
+
+            return ((int?) handler.ExecutionContext.CommandResult) ?? 0;
+        }
+    }
+}

src/Runner.Worker/Container/DockerUtil.cs

@@ -6,9 +6,6 @@ namespace GitHub.Runner.Worker.Container
 {
     public class DockerUtil
     {
-        private static readonly Regex QuoteEscape = new Regex(@"(\\*)" + "\"", RegexOptions.Compiled);
-        private static readonly Regex EndOfStringEscape = new Regex(@"(\\+)$", RegexOptions.Compiled);
-
         public static List<PortMapping> ParseDockerPort(IList<string> portMappingLines)
         {
             const string targetPort = "targetPort";
@@ -64,44 +61,5 @@ namespace GitHub.Runner.Worker.Container
             }
             return "";
         }
-
-        public static string CreateEscapedOption(string flag, string key)
-        {
-            if (String.IsNullOrEmpty(key))
-            {
-                return "";
-            }
-            return $"{flag} {EscapeString(key)}";
-        }
-
-        public static string CreateEscapedOption(string flag, string key, string value)
-        {
-            if (String.IsNullOrEmpty(key))
-            {
-                return "";
-            }
-            var escapedString = EscapeString($"{key}={value}");
-            return $"{flag} {escapedString}";
-        }
-
-        private static string EscapeString(string value)
-        {
-            if (String.IsNullOrEmpty(value))
-            {
-                return "";
-            }
-            // Dotnet escaping rules are weird here, we can only escape \ if it precedes a "
-            // If a double quotation mark follows two or an even number of backslashes, each proceeding backslash pair is replaced with one backslash and the double quotation mark is removed.
-            // If a double quotation mark follows an odd number of backslashes, including just one, each preceding pair is replaced with one backslash and the remaining backslash is removed; however, in this case the double quotation mark is not removed.
-            // https://docs.microsoft.com/en-us/dotnet/api/system.environment.getcommandlineargs?redirectedfrom=MSDN&view=net-6.0#remarks
-
-            // First, find any \ followed by a " and double the number of \ + 1.
-             value = QuoteEscape.Replace(value, @"$1$1\" + "\"");
-            // Next, what if it ends in `\`, it would escape the end quote. So, we need to detect that at the end of the string and perform the same escape
-            // Luckily, we can just use the $ character with detects the end of string in regex
-            value = EndOfStringEscape.Replace(value, @"$1$1");
-            // Finally, wrap it in quotes
-            return $"\"{value}\"";
-        }
     }
 }

src/Runner.Worker/Handlers/StepHost.cs

@@ -188,7 +188,7 @@ namespace GitHub.Runner.Worker.Handlers
             {
                 // e.g. -e MY_SECRET maps the value into the exec'ed process without exposing
                 // the value directly in the command
-                dockerCommandArgs.Add(DockerUtil.CreateEscapedOption("-e", env.Key));
+                dockerCommandArgs.Add($"-e {env.Key}");
             }
             if (!string.IsNullOrEmpty(PrependPath))
             {

src/Test/L0/Container/DockerUtilL0.cs

@@ -144,54 +144,5 @@ namespace GitHub.Runner.Common.Tests.Worker.Container
             var actual = DockerUtil.ParseRegistryHostnameFromImageName(input);
             Assert.Equal(expected, actual);
         }
-
-        [Theory]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Worker")]
-        [InlineData("", "")]
-        [InlineData("foo", "foo")]
-        [InlineData("foo \\ bar", "foo \\ bar")]
-        [InlineData("foo \\", "foo \\\\")]
-        [InlineData("foo \\\\", "foo \\\\\\\\")]
-        [InlineData("foo \\\" bar", "foo \\\\\\\" bar")]
-        [InlineData("foo \\\\\" bar", "foo \\\\\\\\\\\" bar")]
-        public void CreateEscapedOption_keyOnly(string input, string escaped)
-        {
-            var flag = "--example";
-            var actual = DockerUtil.CreateEscapedOption(flag, input);
-            string expected;
-            if (String.IsNullOrEmpty(input))
-            {
-                expected = "";
-            }
-            else
-            {
-                expected = $"{flag} \"{escaped}\"";
-            }
-            Assert.Equal(expected, actual);
-        }
-
-        [Theory]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Worker")]
-        [InlineData("foo", "bar", "foo=bar")]
-        [InlineData("foo\\", "bar", "foo\\=bar")]
-        [InlineData("foo\\", "bar\\", "foo\\=bar\\\\")]
-        [InlineData("foo \\","bar \\", "foo \\=bar \\\\")]
-        public void CreateEscapedOption_keyValue(string keyInput, string valueInput, string escapedString)
-        {
-            var flag = "--example";
-            var actual = DockerUtil.CreateEscapedOption(flag, keyInput, valueInput);
-            string expected;
-            if (String.IsNullOrEmpty(keyInput))
-            {
-                expected = "";
-            }
-            else
-            {
-                expected = $"{flag} \"{escapedString}\"";
-            }
-            Assert.Equal(expected, actual);
-        }
     }
 }

src/Test/L0/Listener/SelfUpdaterL0.cs

@@ -0,0 +1,793 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Reflection;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Listener;
+using GitHub.Runner.Sdk;
+using Moq;
+using Xunit;
+
+namespace GitHub.Runner.Common.Tests.Listener
+{
+    public sealed class SelfUpdaterL0
+    {
+        private Mock<IRunnerServer> _runnerServer;
+        private Mock<ITerminal> _term;
+        private Mock<IConfigurationStore> _configStore;
+        private Mock<IJobDispatcher> _jobDispatcher;
+        private AgentRefreshMessage _refreshMessage = new AgentRefreshMessage(1, "2.299.0");
+        private List<TrimmedPackageMetadata> _trimmedPackages = new List<TrimmedPackageMetadata>();
+
+#if !OS_WINDOWS
+        private string _packageUrl = null;
+#else
+        private string _packageUrl = null;
+#endif
+        public SelfUpdaterL0()
+        {
+            _runnerServer = new Mock<IRunnerServer>();
+            _term = new Mock<ITerminal>();
+            _configStore = new Mock<IConfigurationStore>();
+            _jobDispatcher = new Mock<IJobDispatcher>();
+            _configStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1, AgentId = 1 });
+
+            Environment.SetEnvironmentVariable("_GITHUB_ACTION_EXECUTE_UPDATE_SCRIPT", "1");
+        }
+
+        private async Task FetchLatestRunner()
+        {
+            var latestVersion = "";
+            var httpClientHandler = new HttpClientHandler();
+            httpClientHandler.AllowAutoRedirect = false;
+            using (var client = new HttpClient(httpClientHandler))
+            {
+                var response = await client.SendAsync(new HttpRequestMessage(HttpMethod.Get, "https://github.com/actions/runner/releases/latest"));
+                if (response.StatusCode == System.Net.HttpStatusCode.Redirect)
+                {
+                    var redirect = await response.Content.ReadAsStringAsync();
+                    Regex regex = new Regex(@"/runner/releases/tag/v(?<version>\d+\.\d+\.\d+)");
+                    var match = regex.Match(redirect);
+                    if (match.Success)
+                    {
+                        latestVersion = match.Groups["version"].Value;
+
+#if !OS_WINDOWS
+                        _packageUrl = $"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}.tar.gz";
+#else
+                        _packageUrl = $"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}.zip";
+#endif
+                    }
+                }
+            }
+
+            using (var client = new HttpClient())
+            {
+                var json = await client.GetStringAsync($"https://github.com/actions/runner/releases/download/v{latestVersion}/actions-runner-{BuildConstants.RunnerPackage.PackageName}-{latestVersion}-trimmedpackages.json");
+                _trimmedPackages = StringUtil.ConvertFromJson<List<TrimmedPackageMetadata>>(json);
+            }
+
+            _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                         .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl }));
+
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    var p1 = new ProcessInvokerWrapper();
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();
+                    p3.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    updater.Initialize(hc);
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    try
+                    {
+                        var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+                        Assert.True(result);
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0")));
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0")));
+                    }
+                    finally
+                    {
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None);
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None);
+                    }
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_NoUpdateOnOldVersion()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+
+                    var p1 = new ProcessInvokerWrapper();
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();
+                    p3.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    updater.Initialize(hc);
+
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.200.0", true, It.IsAny<CancellationToken>()))
+                             .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.200.0"), DownloadUrl = _packageUrl }));
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    var result = await updater.SelfUpdate(new AgentRefreshMessage(1, "2.200.0"), _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+                    Assert.False(result);
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_DownloadRetry()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                             .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = $"https://github.com/actions/runner/notexists" }));
+
+                    var p1 = new ProcessInvokerWrapper();
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();
+                    p3.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    updater.Initialize(hc);
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+
+                    var ex = await Assert.ThrowsAsync<TaskCanceledException>(() => updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken));
+                    Assert.Contains($"failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts", ex.Message);
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_ValidateHash()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                             .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, HashValue = "bad_hash" }));
+
+                    var p1 = new ProcessInvokerWrapper();
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();
+                    p3.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    updater.Initialize(hc);
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+
+                    var ex = await Assert.ThrowsAsync<Exception>(() => updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken));
+                    Assert.Contains("did not match expected Runner Hash", ex.Message);
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_CloneHash_RuntimeAndExternals()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    var p1 = new ProcessInvokerWrapper();
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();
+                    p3.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    updater.Initialize(hc);
+
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                         .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = new List<TrimmedPackageMetadata>() { new TrimmedPackageMetadata() } }));
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    try
+                    {
+                        var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+                        Assert.True(result);
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0")));
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0")));
+
+                        FieldInfo contentHashesProperty = updater.GetType().GetField("_contentHashes", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
+                        Assert.NotNull(contentHashesProperty);
+                        Dictionary<string, string> contentHashes = (Dictionary<string, string>)contentHashesProperty.GetValue(updater);
+                        hc.GetTrace().Info(StringUtil.ConvertToJson(contentHashes));
+
+                        var dotnetRuntimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}");
+                        var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
+
+                        Assert.Equal(File.ReadAllText(dotnetRuntimeHashFile).Trim(), contentHashes["dotnetRuntime"]);
+                        Assert.Equal(File.ReadAllText(externalsHashFile).Trim(), contentHashes["externals"]);
+                    }
+                    finally
+                    {
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None);
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None);
+                    }
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_Cancel_CloneHashTask_WhenNotNeeded()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new Mock<IHttpClientHandlerFactory>().Object);
+
+                    var p1 = new ProcessInvokerWrapper();
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();
+                    p3.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    updater.Initialize(hc);
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    try
+                    {
+                        var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+
+                        FieldInfo contentHashesProperty = updater.GetType().GetField("_contentHashes", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
+                        Assert.NotNull(contentHashesProperty);
+                        Dictionary<string, string> contentHashes = (Dictionary<string, string>)contentHashesProperty.GetValue(updater);
+                        hc.GetTrace().Info(StringUtil.ConvertToJson(contentHashes));
+
+                        Assert.NotEqual(2, contentHashes.Count);
+                    }
+                    catch (Exception ex)
+                    {
+                        hc.GetTrace().Error(ex);
+                    }
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_UseExternalsTrimmedPackage()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    var p1 = new ProcessInvokerWrapper();  // hashfiles
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();  // hashfiles
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();  // un-tar
+                    p3.Initialize(hc);
+                    var p4 = new ProcessInvokerWrapper();  // node -v
+                    p4.Initialize(hc);
+                    var p5 = new ProcessInvokerWrapper();  // node -v
+                    p5.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    hc.EnqueueInstance<IProcessInvoker>(p4);
+                    hc.EnqueueInstance<IProcessInvoker>(p5);
+                    updater.Initialize(hc);
+
+                    var trim = _trimmedPackages.Where(x => !x.TrimmedContents.ContainsKey("dotnetRuntime")).ToList();
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                         .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    try
+                    {
+                        var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+                        Assert.True(result);
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0")));
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0")));
+                    }
+                    finally
+                    {
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None);
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None);
+                    }
+
+                    var traceFile = Path.GetTempFileName();
+                    File.Copy(hc.TraceFileName, traceFile, true);
+
+                    var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
+                    var externalsHash = await File.ReadAllTextAsync(externalsHashFile);
+
+                    if (externalsHash == trim[0].TrimmedContents["externals"])
+                    {
+                        Assert.Contains("Use trimmed (externals) package", File.ReadAllText(traceFile));
+                    }
+                    else
+                    {
+                        Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile));
+                    }
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_UseExternalsRuntimeTrimmedPackage()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    var p1 = new ProcessInvokerWrapper();  // hashfiles
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();  // hashfiles
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();  // un-tar
+                    p3.Initialize(hc);
+                    var p4 = new ProcessInvokerWrapper();  // node -v
+                    p4.Initialize(hc);
+                    var p5 = new ProcessInvokerWrapper();  // node -v
+                    p5.Initialize(hc);
+                    var p6 = new ProcessInvokerWrapper();  // runner -v
+                    p6.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    hc.EnqueueInstance<IProcessInvoker>(p4);
+                    hc.EnqueueInstance<IProcessInvoker>(p5);
+                    hc.EnqueueInstance<IProcessInvoker>(p6);
+                    updater.Initialize(hc);
+
+                    var trim = _trimmedPackages.Where(x => x.TrimmedContents.ContainsKey("dotnetRuntime") && x.TrimmedContents.ContainsKey("externals")).ToList();
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                         .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    try
+                    {
+                        var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+                        Assert.True(result);
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0")));
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0")));
+                    }
+                    finally
+                    {
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None);
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None);
+                    }
+
+                    var traceFile = Path.GetTempFileName();
+                    File.Copy(hc.TraceFileName, traceFile, true);
+
+                    var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}");
+                    var externalsHash = await File.ReadAllTextAsync(externalsHashFile);
+
+                    var runtimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}");
+                    var runtimeHash = await File.ReadAllTextAsync(runtimeHashFile);
+
+                    if (externalsHash == trim[0].TrimmedContents["externals"] &&
+                        runtimeHash == trim[0].TrimmedContents["dotnetRuntime"])
+                    {
+                        Assert.Contains("Use trimmed (runtime+externals) package", File.ReadAllText(traceFile));
+                    }
+                    else
+                    {
+                        Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile));
+                    }
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_NotUseExternalsRuntimeTrimmedPackageOnHashMismatch()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    var p1 = new ProcessInvokerWrapper();  // hashfiles
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();  // hashfiles
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();  // un-tar
+                    p3.Initialize(hc);
+                    var p4 = new ProcessInvokerWrapper();  // node -v
+                    p4.Initialize(hc);
+                    var p5 = new ProcessInvokerWrapper();  // node -v
+                    p5.Initialize(hc);
+                    var p6 = new ProcessInvokerWrapper();  // runner -v
+                    p6.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    hc.EnqueueInstance<IProcessInvoker>(p4);
+                    hc.EnqueueInstance<IProcessInvoker>(p5);
+                    hc.EnqueueInstance<IProcessInvoker>(p6);
+                    updater.Initialize(hc);
+
+                    var trim = _trimmedPackages.ToList();
+                    foreach (var package in trim)
+                    {
+                        foreach (var hash in package.TrimmedContents.Keys)
+                        {
+                            package.TrimmedContents[hash] = "mismatch";
+                        }
+                    }
+
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                         .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
+
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    try
+                    {
+                        var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+                        Assert.True(result);
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0")));
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0")));
+                    }
+                    finally
+                    {
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None);
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None);
+                    }
+
+                    var traceFile = Path.GetTempFileName();
+                    File.Copy(hc.TraceFileName, traceFile, true);
+                    Assert.Contains("the current runner does not carry those trimmed content (Hash mismatch)", File.ReadAllText(traceFile));
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Runner")]
+        public async void TestSelfUpdateAsync_FallbackToFullPackage()
+        {
+            try
+            {
+                await FetchLatestRunner();
+                Assert.NotNull(_packageUrl);
+                Assert.NotNull(_trimmedPackages);
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", Path.GetFullPath(Path.Combine(TestUtil.GetSrcPath(), "..", "_layout", "bin")));
+                using (var hc = new TestHostContext(this))
+                {
+                    hc.GetTrace().Info(_packageUrl);
+                    hc.GetTrace().Info(StringUtil.ConvertToJson(_trimmedPackages));
+
+                    //Arrange
+                    var updater = new Runner.Listener.SelfUpdater();
+                    hc.SetSingleton<ITerminal>(_term.Object);
+                    hc.SetSingleton<IRunnerServer>(_runnerServer.Object);
+                    hc.SetSingleton<IConfigurationStore>(_configStore.Object);
+                    hc.SetSingleton<IHttpClientHandlerFactory>(new HttpClientHandlerFactory());
+
+                    var p1 = new ProcessInvokerWrapper();  // hashfiles
+                    p1.Initialize(hc);
+                    var p2 = new ProcessInvokerWrapper();  // hashfiles
+                    p2.Initialize(hc);
+                    var p3 = new ProcessInvokerWrapper();  // un-tar trim
+                    p3.Initialize(hc);
+                    var p4 = new ProcessInvokerWrapper();  // un-tar full
+                    p4.Initialize(hc);
+                    hc.EnqueueInstance<IProcessInvoker>(p1);
+                    hc.EnqueueInstance<IProcessInvoker>(p2);
+                    hc.EnqueueInstance<IProcessInvoker>(p3);
+                    hc.EnqueueInstance<IProcessInvoker>(p4);
+                    updater.Initialize(hc);
+
+                    var trim = _trimmedPackages.ToList();
+                    foreach (var package in trim)
+                    {
+                        package.HashValue = "mismatch";
+                    }
+
+                    _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny<CancellationToken>()))
+                         .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim }));
+
+                    _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny<string>(), It.IsAny<string>()))
+                                 .Callback((int p, int a, string s, string t) =>
+                                 {
+                                     hc.GetTrace().Info(t);
+                                 })
+                                 .Returns(Task.FromResult(new TaskAgent()));
+
+                    try
+                    {
+                        var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken);
+                        Assert.True(result);
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0")));
+                        Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0")));
+                    }
+                    finally
+                    {
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None);
+                        IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None);
+                    }
+
+                    var traceFile = Path.GetTempFileName();
+                    File.Copy(hc.TraceFileName, traceFile, true);
+                    if (File.ReadAllText(traceFile).Contains("Use trimmed (runtime+externals) package"))
+                    {
+                        Assert.Contains("Something wrong with the trimmed runner package, failback to use the full package for runner updates", File.ReadAllText(traceFile));
+                    }
+                    else
+                    {
+                        hc.GetTrace().Warning("Skipping the 'TestSelfUpdateAsync_FallbackToFullPackage' test, as the `externals` or `runtime` hashes have been updated");
+                    }
+                }
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("RUNNER_L0_OVERRIDEBINDIR", null);
+            }
+        }
+    }
+}

src/Test/L0/ProcessInvokerL0.cs

@@ -129,76 +129,7 @@ namespace GitHub.Runner.Common.Tests
                 }
             }
         }
-#if OS_WINDOWS
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public async Task SetTestEnvWithNullInKey()
-        {
-            using (TestHostContext hc = new(this))
-            {
-                Tracing trace = hc.GetTrace();
 
-                Int32 exitCode = -1;
-                var processInvoker = new ProcessInvokerWrapper();
-                processInvoker.Initialize(hc);
-                var stdout = new List<string>();
-                var stderr = new List<string>();
-                processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
-                {
-                    trace.Info(e.Data);
-                    stdout.Add(e.Data);
-                };
-                processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
-                {
-                    trace.Info(e.Data);
-                    stderr.Add(e.Data);
-                };
-
-                exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"",  new Dictionary<string, string>() { { "TEST\0second", "first" } }, CancellationToken.None);
-
-
-                trace.Info("Exit Code: {0}", exitCode);
-                Assert.Equal(0, exitCode);
-                Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x)));
-
-            }
-        }
-
-        [Fact]
-        [Trait("Level", "L0")]
-        [Trait("Category", "Common")]
-        public async Task SetTestEnvWithNullInValue()
-        {
-            using (TestHostContext hc = new(this))
-            {
-                Tracing trace = hc.GetTrace();
-
-                Int32 exitCode = -1;
-                var processInvoker = new ProcessInvokerWrapper();
-                processInvoker.Initialize(hc);
-                var stdout = new List<string>();
-                var stderr = new List<string>();
-                processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
-                {
-                    trace.Info(e.Data);
-                    stdout.Add(e.Data);
-                };
-                processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
-                {
-                    trace.Info(e.Data);
-                    stderr.Add(e.Data);
-                };
-
-                exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"",  new Dictionary<string, string>() { { "TEST", "first\0second" } }, CancellationToken.None);
-
-                trace.Info("Exit Code: {0}", exitCode);
-                Assert.Equal(0, exitCode);
-                Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x)));
-
-            }
-        }
-#endif
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Common")]

src/runnerversion

@@ -1 +1 @@
-2.289.5
+2.289.1