Improve reliability for waiting job to start before running steps.

* fix: add `git` and `curl` to default packages installed

Hi 👋🏼 

These packages are used in a ton of actions on the marketplace. It would be nice if they were installed and ready for use instead of having to install them with `apt-get` on every single Github workflow.

* Update Dockerfile

* Update images/Dockerfile

Co-authored-by: Guillermo Caracuel <633810+gcaracuel@users.noreply.github.com>

* Update images/Dockerfile

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

---------

Co-authored-by: Guillermo Caracuel <633810+gcaracuel@users.noreply.github.com>
Co-authored-by: Tingluo Huang <tingluohuang@github.com>

images/Dockerfile

@@ -41,12 +41,13 @@ ENV ImageOS=ubuntu22
 
 # 'gpg-agent' and 'software-properties-common' are needed for the 'add-apt-repository' command that follows
 RUN apt update -y \
-    && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common \
+    && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common curl jq unzip \
     && rm -rf /var/lib/apt/lists/*
 
 # Configure git-core/ppa based on guidance here:  https://git-scm.com/download/linux
 RUN add-apt-repository ppa:git-core/ppa \
-    && apt update -y
+    && apt update -y \
+    && apt install -y --no-install-recommends git
 
 RUN adduser --disabled-password --gecos "" --uid 1001 runner \
     && groupadd docker --gid 123 \

releaseNote.md

@@ -1,8 +1,10 @@
 ## What's Changed
 
-- .NET 8 compat test adjustments: 1) do not trim SDK, 2) support pattern to match output, 3) modify output truncation length https://github.com/actions/runner/pull/3427
+- .NET 8 OS compatibility test https://github.com/actions/runner/pull/3422
+- Ignore ssl cert on websocket client https://github.com/actions/runner/pull/3423
+- Revert "Bump runner to dotnet 8" https://github.com/actions/runner/pull/3412
 
-**Full Changelog**: https://github.com/actions/runner/compare/v2.319.0...v2.319.1
+**Full Changelog**: https://github.com/actions/runner/compare/v2.318.0...v2.319.0
 
 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet.
 To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.

releaseVersion

@@ -1 +1 @@
-2.319.1
+<Update to ./src/runnerversion when creating release>

src/Runner.Common/HostContext.cs

@@ -244,6 +244,10 @@ namespace GitHub.Runner.Common
                 _trace.Info($"Adding extra user agent '{extraUserAgentHeader}' to all HTTP requests.");
                 _userAgents.Add(extraUserAgentHeader);
             }
+
+            var currentProcess = Process.GetCurrentProcess();
+            _userAgents.Add(new ProductInfoHeaderValue("Pid", currentProcess.Id.ToString()));
+            _userAgents.Add(new ProductInfoHeaderValue("CreationTime", Uri.EscapeDataString(DateTime.UtcNow.ToString("O"))));
         }
 
         public string GetDirectory(WellKnownDirectory directory)

src/Runner.Common/JobServerQueue.cs

@@ -77,6 +77,7 @@ namespace GitHub.Runner.Common
         private int _resultsServiceExceptionsCount = 0;
         private Stopwatch _resultsUploadTimer = new();
         private Stopwatch _actionsUploadTimer = new();
+        private Stopwatch _jobRecordUpdatedTimer = new();
 
         public TaskCompletionSource<int> JobRecordUpdated => _jobRecordUpdated;
 
@@ -96,6 +97,8 @@ namespace GitHub.Runner.Common
 
         private bool _resultsClientInitiated = false;
         private bool _enableTelemetry = false;
+        private bool _enableJobRecordUpdatedTelemetry = false;
+        private bool _enableAutoRetry = false;
         private delegate Task ResultsFileUploadHandler(ResultsUploadFileInfo file);
 
         public override void Initialize(IHostContext hostContext)
@@ -180,6 +183,23 @@ namespace GitHub.Runner.Common
 
             _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _resultsUploadDequeueTask };
             _queueInProcess = true;
+
+
+            if (jobRequest.Variables.TryGetValue("DistributedTask.EnableJobRecordUpdatedTelemetry", out VariableValue enableTelemetry))
+            {
+                _enableJobRecordUpdatedTelemetry = StringUtil.ConvertToBoolean(enableTelemetry?.Value);
+                if (_enableJobRecordUpdatedTelemetry)
+                {
+                    Trace.Info("Enable telemetry for first job record update.");
+                    _jobRecordUpdatedTimer.Start();
+                }
+            }
+
+            if (jobRequest.Variables.TryGetValue("DistributedTask.EnableRecordUpdateAutoRetry", out VariableValue enableAutoRetry))
+            {
+                Trace.Info("Enable auto retry for timeline record update.");
+                _enableAutoRetry = StringUtil.ConvertToBoolean(enableAutoRetry?.Value);
+            }
         }
 
         // WebConsoleLine queue and FileUpload queue are always best effort
@@ -232,6 +252,11 @@ namespace GitHub.Runner.Common
                 Trace.Info(uploadTimeComparison);
                 _jobTelemetries.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = uploadTimeComparison });
             }
+
+            if (_enableJobRecordUpdatedTelemetry)
+            {
+                _jobTelemetries.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"First job record updated time after: {_jobRecordUpdatedTimer.ElapsedMilliseconds} ms" });
+            }
         }
 
         public void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber)
@@ -729,6 +754,11 @@ namespace GitHub.Runner.Common
                                 // We have changed the state of the job
                                 Trace.Info("Job timeline record has been updated for the first time.");
                                 _jobRecordUpdated.TrySetResult(0);
+
+                                if (_enableJobRecordUpdatedTelemetry)
+                                {
+                                    _jobRecordUpdatedTimer.Stop();
+                                }
                             }
                         }
                         catch (Exception ex)
@@ -740,6 +770,15 @@ namespace GitHub.Runner.Common
                             {
                                 mainTimelineRecordsUpdateErrors.Add(ex);
                             }
+
+                            if (!runOnce && _enableAutoRetry)
+                            {
+                                foreach (var retryRecordId in update.PendingRecords.DistinctBy(x => x.Id).Select(r => r.Id))
+                                {
+                                    Trace.Verbose("Enqueue timeline record {0} update for retry.", retryRecordId);
+                                    _timelineUpdateQueue[update.TimelineId].Enqueue(new TimelineRecord() { Id = retryRecordId });
+                                }
+                            }
                         }
                     }
                 }

src/Runner.Listener/BrokerMessageListener.cs

@@ -69,7 +69,8 @@ namespace GitHub.Runner.Listener
                 Version = BuildConstants.RunnerPackage.Version,
                 OSDescription = RuntimeInformation.OSDescription,
             };
-            string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
+            var currentProcess = Process.GetCurrentProcess();
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})";
             var taskAgentSession = new TaskAgentSession(sessionName, agent);
 
             string errorMessage = string.Empty;

src/Runner.Listener/MessageListener.cs

@@ -88,7 +88,8 @@ namespace GitHub.Runner.Listener
                 Version = BuildConstants.RunnerPackage.Version,
                 OSDescription = RuntimeInformation.OSDescription,
             };
-            string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
+            var currentProcess = Process.GetCurrentProcess();
+            string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})";
             var taskAgentSession = new TaskAgentSession(sessionName, agent);
 
             string errorMessage = string.Empty;

src/Runner.Worker/ActionManager.cs

@@ -1102,6 +1102,7 @@ namespace GitHub.Runner.Worker
             int timeoutSeconds = 20 * 60;
             while (retryCount < 3)
             {
+                string requestId = string.Empty;
                 using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
                 using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
                 {
@@ -1117,7 +1118,7 @@ namespace GitHub.Runner.Worker
                             httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
                             using (var response = await httpClient.GetAsync(downloadUrl))
                             {
-                                var requestId = UrlUtil.GetGitHubRequestId(response.Headers);
+                                requestId = UrlUtil.GetGitHubRequestId(response.Headers);
                                 if (!string.IsNullOrEmpty(requestId))
                                 {
                                     Trace.Info($"Request URL: {downloadUrl} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}");
@@ -1155,7 +1156,7 @@ namespace GitHub.Runner.Worker
                     catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2)
                     {
                         Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds.");
-                        throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message}");
+                        throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message} {requestId}");
                     }
                     catch (ActionNotFoundException)
                     {
@@ -1170,11 +1171,11 @@ namespace GitHub.Runner.Worker
                         if (actionDownloadTimeout.Token.IsCancellationRequested)
                         {
                             // action download didn't finish within timeout
-                            executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds.");
+                            executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds. {requestId}");
                         }
                         else
                         {
-                            executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message}");
+                            executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message} {requestId}");
                         }
                     }
                 }

src/Runner.Worker/ExecutionContext.cs

@@ -83,7 +83,7 @@ namespace GitHub.Runner.Worker
         // Initialize
         void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token);
         void CancelToken();
-        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null);
+        IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, List<Issue> embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null);
         IExecutionContext CreateEmbeddedChild(string scopeName, string contextName, Guid embeddedId, ActionRunStage stage, Dictionary<string, string> intraActionState = null, string siblingScopeName = null);
 
         // logging
@@ -135,7 +135,6 @@ namespace GitHub.Runner.Worker
 
         private readonly TimelineRecord _record = new();
         private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new();
-        private readonly List<Issue> _embeddedIssueCollector;
         private readonly object _loggerLock = new();
         private readonly object _matchersLock = new();
         private readonly ExecutionContext _parentExecutionContext;
@@ -154,6 +153,7 @@ namespace GitHub.Runner.Worker
         private CancellationTokenSource _cancellationTokenSource;
         private TaskCompletionSource<int> _forceCompleted = new();
         private bool _throttlingReported = false;
+        private List<Issue> _embeddedIssueCollector;
 
         // only job level ExecutionContext will track throttling delay.
         private long _totalThrottlingDelayInMilliseconds = 0;
@@ -356,6 +356,7 @@ namespace GitHub.Runner.Worker
             int? recordOrder = null,
             IPagingLogger logger = null,
             bool isEmbedded = false,
+            List<Issue> embeddedIssueCollector = null,
             CancellationTokenSource cancellationTokenSource = null,
             Guid embeddedId = default(Guid),
             string siblingScopeName = null,
@@ -365,6 +366,10 @@ namespace GitHub.Runner.Worker
 
             var child = new ExecutionContext(this, isEmbedded);
             child.Initialize(HostContext);
+            if ((Global.Variables.GetBoolean("RunService.FixEmbeddedIssues") ?? false) && embeddedIssueCollector != null)
+            {
+                child._embeddedIssueCollector = embeddedIssueCollector;
+            }
             child.Global = Global;
             child.ScopeName = scopeName;
             child.ContextName = contextName;
@@ -433,7 +438,7 @@ namespace GitHub.Runner.Worker
             Dictionary<string, string> intraActionState = null,
             string siblingScopeName = null)
         {
-            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order);
+            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, embeddedIssueCollector: _embeddedIssueCollector, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order);
         }
 
         public void Start(string currentOperation = null)
@@ -520,7 +525,6 @@ namespace GitHub.Runner.Worker
                 Global.StepsResult.Add(stepResult);
             }
 
-
             if (Root != this)
             {
                 // only dispose TokenSource for step level ExecutionContext
@@ -837,7 +841,6 @@ namespace GitHub.Runner.Worker
             // Actions environment
             ActionsEnvironment = message.ActionsEnvironment;
 
-
             // Service container info
             Global.ServiceContainers = new List<ContainerInfo>();
 
@@ -1418,7 +1421,7 @@ namespace GitHub.Runner.Worker
             {
                 if (key == PipelineTemplateConstants.HostWorkspace)
                 {
-                    // The HostWorkspace context var is excluded so that there is a var that always points to the host path. 
+                    // The HostWorkspace context var is excluded so that there is a var that always points to the host path.
                     // This var can be used to translate back from container paths, e.g. in HashFilesFunction, which always runs on the host machine
                     continue;
                 }

src/Runner.Worker/JobRunner.cs

@@ -210,8 +210,14 @@ namespace GitHub.Runner.Worker
                     // Server won't issue ID_TOKEN for non-inprogress job.
                     // If the job is trying to use OIDC feature, we want the job to be marked as in-progress before running any customer's steps as much as we can.
                     // Timeline record update background process runs every 500ms, so delay 1000ms is enough for most of the cases
-                    Trace.Info($"Waiting for job to be marked as started.");
+                    var maxWaitTimeInSeconds = jobContext.Global.Variables.GetInt("DistributedTask.FirstJobRecordUpdateWaitTimeInSeconds");
-                    await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(1000));
+                    if (maxWaitTimeInSeconds == null)
+                    {
+                        maxWaitTimeInSeconds = 1;
+                    }
+
+                    Trace.Info($"Waiting {maxWaitTimeInSeconds.Value} seconds for job to be marked as started.");
+                    await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(maxWaitTimeInSeconds.Value * 1000));
                 }
 
                 // Run all job steps

src/Test/L0/Worker/ExecutionContextL0.cs

@@ -773,6 +773,82 @@ namespace GitHub.Runner.Common.Tests.Worker
         [Trait("Level", "L0")]
         [Trait("Category", "Worker")]
         public void PublishStepResult_EmbeddedStep()
+        {
+            using (TestHostContext hc = CreateTestContext())
+            {
+                // Job request
+                TaskOrchestrationPlanReference plan = new();
+                TimelineReference timeline = new();
+                Guid jobId = Guid.NewGuid();
+                string jobName = "some job name";
+                var variables = new Dictionary<string, VariableValue>()
+                {
+                    ["RunService.FixEmbeddedIssues"] = new VariableValue("true"),
+                };
+                var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, variables, new List<MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List<Pipelines.ActionStep>(), null, null, null, null, null);
+                jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
+                {
+                    Alias = Pipelines.PipelineConstants.SelfAlias,
+                    Id = "github",
+                    Version = "sha1"
+                });
+                jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
+
+                // Mocks
+                var pagingLogger = new Mock<IPagingLogger>();
+                var pagingLogger2 = new Mock<IPagingLogger>();
+                var jobServerQueue = new Mock<IJobServerQueue>();
+                jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny<Guid>(), It.IsAny<TimelineRecord>()));
+                hc.EnqueueInstance(pagingLogger.Object);
+                hc.EnqueueInstance(pagingLogger2.Object);
+                hc.SetSingleton(jobServerQueue.Object);
+
+                // Job context
+                var jobContext = new Runner.Worker.ExecutionContext();
+                jobContext.Initialize(hc);
+                jobContext.InitializeJob(jobRequest, CancellationToken.None);
+                jobContext.Start();
+
+                // Step 1 context
+                var step1 = jobContext.CreateChild(Guid.NewGuid(), "my_step", "my_step", null, null, ActionRunStage.Main);
+                step1.Start();
+
+                // Embedded step 1a context
+                var embeddedStep1a = step1.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main);
+                embeddedStep1a.Start();
+                embeddedStep1a.StepTelemetry.Type = "node16";
+                embeddedStep1a.StepTelemetry.Action = "actions/checkout";
+                embeddedStep1a.StepTelemetry.Ref = "v2";
+                embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default);
+                embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default);
+                embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }, ExecutionContextLogOptions.Default);
+                embeddedStep1a.Complete();
+
+                // Embedded step 1b context
+                var embeddedStep1b = step1.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main);
+                embeddedStep1b.Start();
+                embeddedStep1b.StepTelemetry.Type = "node16";
+                embeddedStep1b.StepTelemetry.Action = "actions/checkout";
+                embeddedStep1b.StepTelemetry.Ref = "v2";
+                embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Error, Message = "error 2" }, ExecutionContextLogOptions.Default);
+                embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning 2" }, ExecutionContextLogOptions.Default);
+                embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice 2" }, ExecutionContextLogOptions.Default);
+                embeddedStep1b.Complete();
+
+                step1.Complete();
+
+                // Assert
+                Assert.Equal(3, jobContext.Global.StepsResult.Count);
+                Assert.Equal(0, jobContext.Global.StepsResult[0].Annotations.Count);
+                Assert.Equal(0, jobContext.Global.StepsResult[1].Annotations.Count);
+                Assert.Equal(6, jobContext.Global.StepsResult[2].Annotations.Count);
+            }
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        public void PublishStepResult_EmbeddedStep_Legacy()
         {
             using (TestHostContext hc = CreateTestContext())
             {
@@ -807,7 +883,7 @@ namespace GitHub.Runner.Common.Tests.Worker
                 ec.InitializeJob(jobRequest, CancellationToken.None);
                 ec.Start();
 
-                var embeddedStep = ec.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, ActionRunStage.Main, isEmbedded: true);
+                var embeddedStep = ec.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main);
                 embeddedStep.Start();
 
                 embeddedStep.StepTelemetry.Type = "node16";