Add edge case for heredoc

src/Runner.Worker/ExecutionContext.cs

@@ -1468,7 +1468,7 @@ namespace GitHub.Runner.Worker
 
         private bool logTemplateErrorsAsDebugMessages()
         {
-            if (_executionContext.Global.Variables.TryGetValue(Constants.Runner.Features.LogTemplateErrorsAsDebugMessages, out var logErrorsAsDebug))
+            if (_executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Runner.Features.LogTemplateErrorsAsDebugMessages, out var logErrorsAsDebug))
             {
                 return StringUtil.ConvertToBoolean(logErrorsAsDebug, defaultValue: false);
             }

src/Runner.Worker/FileCommandManager.cs

@@ -322,28 +322,16 @@ namespace GitHub.Runner.Worker
                     var equalsIndex = line.IndexOf("=", StringComparison.Ordinal);
                     var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal);
 
-                    // Heredoc style NAME=<<EOF (where EOF is typically randomly-generated Base64 and may include an '=' character)
+                    // Heredoc style NAME<<EOF (where EOF is typically randomly-generated Base64 and may include an '=' character)
                     // see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
-                    // "key=<<EOF is also considered heredoc where the key contains an =.
-                    // Also we detect heredoc if whitespace chars exist between = and <<. (ex: "key      <<EOF")
-                    var isHeredoc = heredocIndex >= 0 &&
-                    (
-                        equalsIndex < 0 ||
-                        heredocIndex < equalsIndex ||
-                        (
-                            heredocIndex > equalsIndex &&
-                            OnlyContainsWhiteSpaceBetweenPositions(line, equalsIndex, heredocIndex)
-                        ) 
-                    );
-                    
-                    if (isHeredoc)
+                    if (heredocIndex >= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex))
                     {
                         var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None);
                         if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1]))
                         {
                             throw new Exception($"Invalid format '{line}'. Name must not be empty and delimiter must not be empty");
                         }
-                        key = split[0].Trim().TrimEnd('=');
+                        key = split[0];
                         var delimiter = split[1];
                         var startIndex = index; // Start index of the value (inclusive)
                         var endIndex = index;   // End index of the value (exclusive)
@@ -364,8 +352,8 @@ namespace GitHub.Runner.Worker
 
                         output = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty;
                     }
-                    // Normal style NAME=VALUE, can have << in the value.
-                    else if (equalsIndex >= 0)
+                    // Normal style NAME=VALUE
+                    else if (equalsIndex >= 0 && heredocIndex < 0)
                     {
                         var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None);
                         if (string.IsNullOrEmpty(line))
@@ -395,32 +383,6 @@ namespace GitHub.Runner.Worker
             return GetEnumerator();
         }
 
-        // accepts a string, two indexes, and returns true if only whitespace chars exist between those two indexes (non-inclusive)
-        private static bool OnlyContainsWhiteSpaceBetweenPositions(string str, int pos1, int pos2)
-        {
-            // Check if the provided positions are valid
-            if (pos1 < 0 || pos2 < 0 || pos1 >= str.Length || pos2 >= str.Length)
-            {
-                throw new ArgumentOutOfRangeException("OnlyContainsWhiteSpaceBetweenPositions: Positions should be within the string length.");
-            }
-
-            // Ensure pos1 is always the smaller position
-            if (pos2 < pos1)
-            {
-                throw new ArgumentException("OnlyContainsWhiteSpaceBetweenPositions: pos1 must be less than or equal to pos2.");
-            }
-
-            for (int i = pos1 + 1; i < pos2; i++)
-            {
-                if (!char.IsWhiteSpace(str[i]))
-                {
-                    return false;
-                }
-            }
-
-            return true;
-        }
-
         private static string ReadLine(
                 string text,
                 ref int index)

src/Runner.Worker/JobExtension.cs

@@ -4,7 +4,6 @@ using System.Diagnostics;
 using System.Globalization;
 using System.IO;
 using System.Linq;
-using System.Net.Http;
 using System.Runtime.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
@@ -16,7 +15,6 @@ using GitHub.DistributedTask.WebApi;
 using GitHub.Runner.Common;
 using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
-using GitHub.Services.Common;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
@@ -36,13 +34,12 @@ namespace GitHub.Runner.Worker
     public interface IJobExtension : IRunnerService
     {
         Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message);
-        Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc);
+        void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc);
     }
 
     public sealed class JobExtension : RunnerService, IJobExtension
     {
         private readonly HashSet<string> _existingProcesses = new(StringComparer.OrdinalIgnoreCase);
-        private readonly List<Task<string>> _connectivityCheckTasks = new();
         private bool _processCleanup;
         private string _processLookupId = $"github_{Guid.NewGuid()}";
         private CancellationTokenSource _diskSpaceCheckToken = new();
@@ -431,22 +428,6 @@ namespace GitHub.Runner.Worker
                         _diskSpaceCheckTask = CheckDiskSpaceAsync(context, _diskSpaceCheckToken.Token);
                     }
 
-                    // Check server connectivity in background
-                    ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
-                    if (systemConnection.Data.TryGetValue("ConnectivityChecks", out var connectivityChecksPayload) &&
-                        !string.IsNullOrEmpty(connectivityChecksPayload))
-                    {
-                        Trace.Info($"Start checking server connectivity.");
-                        var checkUrls = StringUtil.ConvertFromJson<List<string>>(connectivityChecksPayload);
-                        if (checkUrls?.Count > 0)
-                        {
-                            foreach (var checkUrl in checkUrls)
-                            {
-                                _connectivityCheckTasks.Add(CheckConnectivity(checkUrl));
-                            }
-                        }
-                    }
-
                     return steps;
                 }
                 catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested)
@@ -491,7 +472,7 @@ namespace GitHub.Runner.Worker
             return reference;
         }
 
-        public async Task FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
+        public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
         {
             Trace.Entering();
             ArgUtil.NotNull(jobContext, nameof(jobContext));
@@ -668,28 +649,6 @@ namespace GitHub.Runner.Worker
                     {
                         _diskSpaceCheckToken.Cancel();
                     }
-
-                    // Collect server connectivity check result
-                    if (_connectivityCheckTasks.Count > 0)
-                    {
-                        try
-                        {
-                            Trace.Info($"Wait for all connectivity checks to finish.");
-                            await Task.WhenAll(_connectivityCheckTasks);
-                            foreach (var check in _connectivityCheckTasks)
-                            {
-                                var result = await check;
-                                Trace.Info($"Connectivity check result: {result}");
-                                context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = result });
-                            }
-                        }
-                        catch (Exception ex)
-                        {
-                            Trace.Error($"Fail to check server connectivity.");
-                            Trace.Error(ex);
-                            context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.ConnectivityCheck, Message = $"Fail to check server connectivity. {ex.Message}" });
-                        }
-                    }
                 }
                 catch (Exception ex)
                 {
@@ -705,37 +664,6 @@ namespace GitHub.Runner.Worker
             }
         }
 
-        private async Task<string> CheckConnectivity(string endpointUrl)
-        {
-            Trace.Info($"Check server connectivity for {endpointUrl}.");
-            string result = string.Empty;
-            using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(5)))
-            {
-                try
-                {
-                    using (var httpClientHandler = HostContext.CreateHttpClientHandler())
-                    using (var httpClient = new HttpClient(httpClientHandler))
-                    {
-                        httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
-                        var response = await httpClient.GetAsync(endpointUrl, timeoutTokenSource.Token);
-                        result = $"{endpointUrl}: {response.StatusCode}";
-                    }
-                }
-                catch (Exception ex) when (ex is OperationCanceledException && timeoutTokenSource.IsCancellationRequested)
-                {
-                    Trace.Error($"Request timeout during connectivity check: {ex}");
-                    result = $"{endpointUrl}: timeout";
-                }
-                catch (Exception ex)
-                {
-                    Trace.Error($"Catch exception during connectivity check: {ex}");
-                    result = $"{endpointUrl}: {ex.Message}";
-                }
-            }
-
-            return result;
-        }
-
         private async Task CheckDiskSpaceAsync(IExecutionContext context, CancellationToken token)
         {
             while (!token.IsCancellationRequested)

src/Runner.Worker/JobRunner.cs

@@ -229,7 +229,7 @@ namespace GitHub.Runner.Worker
                 finally
                 {
                     Trace.Info("Finalize job.");
-                    await jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
+                    jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
                 }
 
                 Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");

src/Sdk/DTWebApi/WebApi/JobTelemetryType.cs

@@ -9,8 +9,5 @@ namespace GitHub.DistributedTask.WebApi
 
         [EnumMember]
         ActionCommand = 1,
-
-        [EnumMember]
-        ConnectivityCheck = 2,
     }
 }

src/Test/L0/Worker/FileCommandTestBase.cs

@@ -239,25 +239,16 @@ namespace GitHub.Runner.Common.Tests.Worker
                     "EOF EOF",
                     "EOF",
                     "MY_KEY_5=abc << def",
-                    "MY_KEY_6=    <<EOF",
-                    "white space test",
-                    "EOF",
-                    "MY_KEY_7 <<=EOF=",
-                    "abc",
-                    "=EOF=",
-                    string.Empty
                 };
                 TestUtil.WriteContent(stateFile, content);
                 _fileCmdExtension.ProcessCommand(_executionContext.Object, stateFile, null);
                 Assert.Equal(0, _issues.Count);
-                Assert.Equal(7, _store.Count);
+                Assert.Equal(4, _store.Count);
                 Assert.Equal($"hello{BREAK}{BREAK}three{BREAK}", _store["MY_KEY_1"]);
                 Assert.Equal($"hello=two", _store["MY_KEY_2"]);
                 Assert.Equal($" EOF", _store["MY_KEY_3"]);
                 Assert.Equal($"EOF EOF", _store["MY_KEY_4"]);
                 Assert.Equal($"abc << def", _store["MY_KEY_5"]);
-                Assert.Equal($"white space test", _store["MY_KEY_6"]);
-                Assert.Equal($"abc", _store["MY_KEY_7"]);
             }
         }
 

src/Test/L0/Worker/JobExtensionL0.cs

@@ -1,13 +1,13 @@
-using System;
+using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Worker;
+using Moq;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
-using System.Threading;
 using System.Threading.Tasks;
-using GitHub.DistributedTask.WebApi;
-using GitHub.Runner.Worker;
-using Moq;
 using Xunit;
+using System.Threading;
 using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Common.Tests.Worker
@@ -105,18 +105,6 @@ namespace GitHub.Runner.Common.Tests.Worker
             github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner");
             github["secret_source"] = new Pipelines.ContextData.StringContextData("Actions");
             _message.ContextData.Add("github", github);
-            _message.Resources.Endpoints.Add(new ServiceEndpoint()
-            {
-                Name = WellKnownServiceEndpointNames.SystemVssConnection,
-                Url = new Uri("https://pipelines.actions.githubusercontent.com"),
-                Authorization = new EndpointAuthorization()
-                {
-                    Scheme = "Test",
-                    Parameters = {
-                        {"AccessToken", "token"}
-                    }
-                },
-            });
 
             hc.SetSingleton(_actionManager.Object);
             hc.SetSingleton(_config.Object);
@@ -243,7 +231,7 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
-        public async Task UploadDiganosticLogIfEnvironmentVariableSet()
+        public void UploadDiganosticLogIfEnvironmentVariableSet()
         {
             using (TestHostContext hc = CreateTestContext())
             {
@@ -256,7 +244,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 _jobEc.Initialize(hc);
                 _jobEc.InitializeJob(_message, _tokenSource.Token);
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 _diagnosticLogManager.Verify(x =>
                     x.UploadDiagnosticLogs(
@@ -271,7 +259,7 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
-        public async Task DontUploadDiagnosticLogIfEnvironmentVariableFalse()
+        public void DontUploadDiagnosticLogIfEnvironmentVariableFalse()
         {
             using (TestHostContext hc = CreateTestContext())
             {
@@ -284,7 +272,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 _jobEc.Initialize(hc);
                 _jobEc.InitializeJob(_message, _tokenSource.Token);
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 _diagnosticLogManager.Verify(x =>
                     x.UploadDiagnosticLogs(
@@ -299,14 +287,14 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
-        public async Task DontUploadDiagnosticLogIfEnvironmentVariableMissing()
+        public void DontUploadDiagnosticLogIfEnvironmentVariableMissing()
         {
             using (TestHostContext hc = CreateTestContext())
             {
                 var jobExtension = new JobExtension();
                 jobExtension.Initialize(hc);
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 _diagnosticLogManager.Verify(x =>
                     x.UploadDiagnosticLogs(
@@ -321,7 +309,7 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
-        public async Task EnsureFinalizeJobRunsIfMessageHasNoEnvironmentUrl()
+        public void EnsureFinalizeJobRunsIfMessageHasNoEnvironmentUrl()
         {
             using (TestHostContext hc = CreateTestContext())
             {
@@ -334,7 +322,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 _jobEc.Initialize(hc);
                 _jobEc.InitializeJob(_message, _tokenSource.Token);
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 Assert.Equal(TaskResult.Succeeded, _jobEc.Result);
             }
@@ -343,7 +331,7 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
-        public async Task EnsureFinalizeJobHandlesNullEnvironmentUrl()
+        public void EnsureFinalizeJobHandlesNullEnvironmentUrl()
         {
             using (TestHostContext hc = CreateTestContext())
             {
@@ -359,7 +347,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 _jobEc.Initialize(hc);
                 _jobEc.InitializeJob(_message, _tokenSource.Token);
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 Assert.Equal(TaskResult.Succeeded, _jobEc.Result);
             }
@@ -368,7 +356,7 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
-        public async Task EnsureFinalizeJobHandlesNullEnvironment()
+        public void EnsureFinalizeJobHandlesNullEnvironment()
         {
             using (TestHostContext hc = CreateTestContext())
             {
@@ -381,7 +369,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 _jobEc.Initialize(hc);
                 _jobEc.InitializeJob(_message, _tokenSource.Token);
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 Assert.Equal(TaskResult.Succeeded, _jobEc.Result);
             }
@@ -409,7 +397,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 var hookStart = result.First() as JobExtensionRunner;
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 Assert.Equal(Constants.Hooks.JobStartedStepName, hookStart.DisplayName);
                 Assert.Equal(Constants.Hooks.JobCompletedStepName, (_jobEc.PostJobSteps.Last() as JobExtensionRunner).DisplayName);
@@ -422,7 +410,7 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Fact]
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
-        public async Task EnsureNoPreAndPostHookSteps()
+        public void EnsureNoPreAndPostHookSteps()
         {
             using (TestHostContext hc = CreateTestContext())
             {
@@ -437,7 +425,7 @@ namespace GitHub.Runner.Common.Tests.Worker
 
                 var x = _jobEc.JobSteps;
 
-                await jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
+                jobExtension.FinalizeJob(_jobEc, _message, DateTime.UtcNow);
 
                 Assert.Equal(TaskResult.Succeeded, _jobEc.Result);
                 Assert.Equal(0, _jobEc.PostJobSteps.Count);