Create 2.304.0 runner release

* Add orchestrationId to useragent for better correlation.

* .

releaseNote.md

@@ -1,17 +1,18 @@
 ## Features
-- Support matrix context in output keys (#2477)
-- Add update certificates to `./run.sh` if `RUNNER_UPDATE_CA_CERTS` env is set (#2471)
-- Bypass all proxies for all hosts if `no_proxy='*'` is set (#2395)
-- Change runner image to make user/folder align with `ubuntu-latest` hosted runner. (#2469)
+- Runner changes for communication with Results service (#2510, #2531, #2535, #2516) 
+- Add `*.ghe.localhost` domains to hosted server check (#2536) 
+- Add `OrchestrationId` to user-agent for better telemetry correlation. (#2568) 
 
 ## Bugs
-- Exit on runner version deprecation error (#2299)
-- Runner service exit after consecutive re-try exits (#2426)
+- Fix JIT configurations on Windows (#2497) 
+- Guard against NullReference while creating HostContext (#2343) 
+- Handles broken symlink in `Which` (#2150, #2196) 
+- Adding curl retry for external tool downloads (#2552, #2557) 
+- Limit the time we wait for waiting websocket to connect. (#2554) 
 
 ## Misc
-- Replace deprecated command with environment file (#2429)
-- Make requests to `Run` service to renew job request (#2461)
-- Add job/step log upload to Result service (#2447, #2439)
+- Bump container hooks version to 0.3.1 in runner image (#2496) 
+- Runner changes to communicate with vNext services (#2487, #2500, #2505, #2541, #2547) 
 
 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. 
 To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. 

releaseVersion

@@ -1 +1 @@
-<Update to ./src/runnerversion when creating release>
+2.304.0

src/Misc/externals.sh

@@ -55,12 +55,23 @@ function acquireExternalTool() {
             # Download from source to the partial file.
             echo "Downloading $download_source"
             mkdir -p "$(dirname "$download_target")" || checkRC 'mkdir'
+
+            CURL_VERSION=$(curl --version | awk 'NR==1{print $2}')
+            echo "Curl version: $CURL_VERSION"
+
             # curl -f Fail silently (no output at all) on HTTP errors (H)
             #      -k Allow connections to SSL sites without certs (H)
             #      -S Show error. With -s, make curl show errors when they occur
             #      -L Follow redirects (H)
             #      -o FILE    Write to FILE instead of stdout
-            curl -fkSL -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
+            #      --retry 3   Retries transient errors 3 times (timeouts, 5xx)
+            if [[ "$(printf '%s\n' "7.71.0" "$CURL_VERSION" | sort -V | head -n1)" != "7.71.0" ]]; then
+                # Curl version is less than or equal to 7.71.0, skipping retry-all-errors flag
+                 curl -fkSL --retry 3 -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
+            else
+                # Curl version is greater than 7.71.0, running curl with --retry-all-errors flag
+                 curl -fkSL --retry 3 --retry-all-errors -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
+            fi
 
             # Move the partial file to the download target.
             mv "$partial_target" "$download_target" || checkRC 'mv'

src/Runner.Common/Constants.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 
 namespace GitHub.Runner.Common
 {
@@ -261,6 +261,7 @@ namespace GitHub.Runner.Common
                 public static readonly string AccessToken = "system.accessToken";
                 public static readonly string Culture = "system.culture";
                 public static readonly string PhaseDisplayName = "system.phaseDisplayName";
+                public static readonly string OrchestrationId = "system.orchestrationId";
             }
         }
 

src/Runner.Common/JobServer.cs

@@ -199,13 +199,15 @@ namespace GitHub.Runner.Common
             {
                 Trace.Info($"Attempting to start websocket client with delay {delay}.");
                 await Task.Delay(delay);
-                await this._websocketClient.ConnectAsync(new Uri(feedStreamUrl), default(CancellationToken));
+                using var connectTimeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+                await this._websocketClient.ConnectAsync(new Uri(feedStreamUrl), connectTimeoutTokenSource.Token);
                 Trace.Info($"Successfully started websocket client.");
             }
             catch (Exception ex)
             {
                 Trace.Info("Exception caught during websocket client connect, fallback of HTTP would be used now instead of websocket.");
                 Trace.Error(ex);
+                this._websocketClient = null;
             }
         }
 

src/Runner.Common/JobServerQueue.cs

@@ -17,7 +17,7 @@ namespace GitHub.Runner.Common
         TaskCompletionSource<int> JobRecordUpdated { get; }
         event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
         Task ShutdownAsync();
-        void Start(Pipelines.AgentJobRequestMessage jobRequest);
+        void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultServiceOnly = false);
         void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
         void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
         void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines);
@@ -70,6 +70,7 @@ namespace GitHub.Runner.Common
         private readonly TaskCompletionSource<int> _jobCompletionSource = new();
         private readonly TaskCompletionSource<int> _jobRecordUpdated = new();
         private bool _queueInProcess = false;
+        private bool _resultsServiceOnly = false;
 
         public TaskCompletionSource<int> JobRecordUpdated => _jobRecordUpdated;
 
@@ -95,13 +96,17 @@ namespace GitHub.Runner.Common
             _resultsServer = hostContext.GetService<IResultsServer>();
         }
 
-        public void Start(Pipelines.AgentJobRequestMessage jobRequest)
+        public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultServiceOnly = false)
         {
             Trace.Entering();
+            _resultsServiceOnly = resultServiceOnly;
 
             var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
 
+            if (!resultServiceOnly)
+            {
                 _jobServer.InitializeWebsocketClient(serviceEndPoint);
+            }
 
             // This code is usually wrapped by an instance of IExecutionContext which isn't available here.
             jobRequest.Variables.TryGetValue("system.github.results_endpoint", out VariableValue resultsEndpointVariable);
@@ -112,8 +117,16 @@ namespace GitHub.Runner.Common
                 !string.IsNullOrEmpty(accessToken) &&
                 !string.IsNullOrEmpty(resultsReceiverEndpoint))
             {
+                string liveConsoleFeedUrl = null;
                 Trace.Info("Initializing results client");
-                _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken);
+                if (resultServiceOnly
+                    && serviceEndPoint.Data.TryGetValue("FeedStreamUrl", out var feedStreamUrl)
+                    && !string.IsNullOrEmpty(feedStreamUrl))
+                {
+                    liveConsoleFeedUrl = feedStreamUrl;
+                }
+
+                _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken);
                 _resultsClientInitiated = true;
             }
 
@@ -194,6 +207,9 @@ namespace GitHub.Runner.Common
             Trace.Info($"Disposing job server ...");
             await _jobServer.DisposeAsync();
 
+            Trace.Info($"Disposing results server ...");
+            await _resultsServer.DisposeAsync();
+
             Trace.Info("All queue process tasks have been stopped, and all queues are drained.");
         }
 
@@ -371,9 +387,16 @@ namespace GitHub.Runner.Common
                             {
                                 // Give at most 60s for each request.
                                 using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(60)))
+                                {
+                                    if (_resultsServiceOnly)
+                                    {
+                                        await _resultsServer.AppendLiveConsoleFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, timeoutTokenSource.Token);
+                                    }
+                                    else
                                     {
                                         await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, timeoutTokenSource.Token);
                                     }
+                                }
 
                                 if (_firstConsoleOutputs)
                                 {
@@ -599,7 +622,7 @@ namespace GitHub.Runner.Common
 
                         foreach (var detailTimeline in update.PendingRecords.Where(r => r.Details != null))
                         {
-                            if (!_allTimelines.Contains(detailTimeline.Details.Id))
+                            if (!_resultsServiceOnly && !_allTimelines.Contains(detailTimeline.Details.Id))
                             {
                                 try
                                 {
@@ -620,8 +643,12 @@ namespace GitHub.Runner.Common
                         }
 
                         try
+                        {
+                            if (!_resultsServiceOnly)
                             {
                                 await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken));
+                            }
+
                             try
                             {
                                 if (_resultsClientInitiated)
@@ -796,6 +823,8 @@ namespace GitHub.Runner.Common
         {
             bool uploadSucceed = false;
             try
+            {
+                if (!_resultsServiceOnly)
                 {
                     if (String.Equals(file.Type, CoreAttachmentType.Log, StringComparison.OrdinalIgnoreCase))
                     {
@@ -820,6 +849,7 @@ namespace GitHub.Runner.Common
                             var result = await _jobServer.CreateAttachmentAsync(_scopeIdentifier, _hubName, _planId, file.TimelineId, file.TimelineRecordId, file.Type, file.Name, fs, default(CancellationToken));
                         }
                     }
+                }
 
                 uploadSucceed = true;
             }

src/Runner.Common/ResultsServer.cs

@@ -1,17 +1,26 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http.Headers;
+using System.Net.WebSockets;
+using System.Security;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.WebApi;
+using GitHub.Runner.Sdk;
+using GitHub.Services.Common;
 using GitHub.Services.Results.Client;
+using GitHub.Services.WebApi.Utilities.Internal;
 
 namespace GitHub.Runner.Common
 {
     [ServiceLocator(Default = typeof(ResultServer))]
-    public interface IResultsServer : IRunnerService
+    public interface IResultsServer : IRunnerService, IAsyncDisposable
     {
-        void InitializeResultsClient(Uri uri, string token);
+        void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token);
+
+        Task<bool> AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken);
 
         // logging and console
         Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file,
@@ -31,10 +40,26 @@ namespace GitHub.Runner.Common
     {
         private ResultsHttpClient _resultsClient;
 
-        public void InitializeResultsClient(Uri uri, string token)
+        private ClientWebSocket _websocketClient;
+        private DateTime? _lastConnectionFailure;
+
+        private static readonly TimeSpan MinDelayForWebsocketReconnect = TimeSpan.FromMilliseconds(100);
+        private static readonly TimeSpan MaxDelayForWebsocketReconnect = TimeSpan.FromMilliseconds(500);
+
+        private Task _websocketConnectTask;
+        private String _liveConsoleFeedUrl;
+        private string _token;
+
+        public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token)
         {
             var httpMessageHandler = HostContext.CreateHttpClientHandler();
             this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true);
+            _token = token;
+            if (!string.IsNullOrEmpty(liveConsoleFeedUrl))
+            {
+                _liveConsoleFeedUrl = liveConsoleFeedUrl;
+                InitializeWebsocketClient(liveConsoleFeedUrl, token, TimeSpan.Zero, retryConnection: true);
+            }
         }
 
         public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file,
@@ -94,5 +119,144 @@ namespace GitHub.Runner.Common
 
             throw new InvalidOperationException("Results client is not initialized.");
         }
+
+        public ValueTask DisposeAsync()
+        {
+            CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);
+
+            GC.SuppressFinalize(this);
+
+            return ValueTask.CompletedTask;
+        }
+
+        private void InitializeWebsocketClient(string liveConsoleFeedUrl, string accessToken, TimeSpan delay, bool retryConnection = false)
+        {
+            if (!string.IsNullOrEmpty(accessToken))
+            {
+                Trace.Info($"No access token from server");
+                return;
+            }
+
+            if (!string.IsNullOrEmpty(liveConsoleFeedUrl))
+            {
+                Trace.Info($"No live console feed url from server");
+                return;
+            }
+
+            Trace.Info($"Creating websocket client ..." + liveConsoleFeedUrl);
+            this._websocketClient = new ClientWebSocket();
+            this._websocketClient.Options.SetRequestHeader("Authorization", $"Bearer {accessToken}");
+            var userAgentValues = new List<ProductInfoHeaderValue>();
+            userAgentValues.AddRange(UserAgentUtility.GetDefaultRestUserAgent());
+            userAgentValues.AddRange(HostContext.UserAgents);
+            this._websocketClient.Options.SetRequestHeader("User-Agent", string.Join(" ", userAgentValues.Select(x => x.ToString())));
+
+            // during initialization, retry upto 3 times to setup connection
+            this._websocketConnectTask = ConnectWebSocketClient(liveConsoleFeedUrl, delay, retryConnection);
+        }
+
+        private async Task ConnectWebSocketClient(string feedStreamUrl, TimeSpan delay, bool retryConnection = false)
+        {
+            bool connected = false;
+            int retries = 0;
+
+            do
+            {
+                try
+                {
+                    Trace.Info($"Attempting to start websocket client with delay {delay}.");
+                    await Task.Delay(delay);
+                    using var connectTimeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+                    await this._websocketClient.ConnectAsync(new Uri(feedStreamUrl), connectTimeoutTokenSource.Token);
+                    Trace.Info($"Successfully started websocket client.");
+                    connected = true;
+                }
+                catch (Exception ex)
+                {
+                    Trace.Info("Exception caught during websocket client connect, retry connection.");
+                    Trace.Error(ex);
+                    retries++;
+                    this._websocketClient = null;
+                    _lastConnectionFailure = DateTime.Now;
+                }
+            } while (retryConnection && !connected && retries < 3);
+        }
+
+        public async Task<bool> AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken)
+        {
+            if (_websocketConnectTask != null)
+            {
+                await _websocketConnectTask;
+            }
+
+            bool delivered = false;
+            int retries = 0;
+
+            // "_websocketClient != null" implies either: We have a successful connection OR we have to attempt sending again and then reconnect
+            // ...in other words, if websocket client is null, we will skip sending to websocket
+            if (_websocketClient != null)
+            {
+                var linesWrapper = startLine.HasValue
+                    ? new TimelineRecordFeedLinesWrapper(stepId, lines, startLine.Value)
+                    : new TimelineRecordFeedLinesWrapper(stepId, lines);
+                var jsonData = StringUtil.ConvertToJson(linesWrapper);
+                var jsonDataBytes = Encoding.UTF8.GetBytes(jsonData);
+                // break the message into chunks of 1024 bytes
+                for (var i = 0; i < jsonDataBytes.Length; i += 1 * 1024)
+                {
+                    var lastChunk = i + (1 * 1024) >= jsonDataBytes.Length;
+                    var chunk = new ArraySegment<byte>(jsonDataBytes, i, Math.Min(1 * 1024, jsonDataBytes.Length - i));
+
+                    delivered = false;
+                    while (!delivered && retries < 3)
+                    {
+                        try
+                        {
+                            if (_websocketClient != null)
+                            {
+                                await _websocketClient.SendAsync(chunk, WebSocketMessageType.Text, endOfMessage: lastChunk, cancellationToken);
+                                delivered = true;
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            var delay = BackoffTimerHelper.GetRandomBackoff(MinDelayForWebsocketReconnect, MaxDelayForWebsocketReconnect);
+                            Trace.Info($"Websocket is not open, let's attempt to connect back again with random backoff {delay} ms.");
+                            Trace.Error(ex);
+                            retries++;
+                            InitializeWebsocketClient(_liveConsoleFeedUrl, _token, delay);
+                        }
+                    }
+                }
+            }
+
+            if (!delivered)
+            {
+                // Giving up for now, so next invocation of this method won't attempt to reconnect
+                _websocketClient = null;
+
+                // however if 10 minutes have already passed, let's try reestablish connection again
+                if (_lastConnectionFailure.HasValue && DateTime.Now > _lastConnectionFailure.Value.AddMinutes(10))
+                {
+                    // Some minutes passed since we retried last time, try connection again
+                    InitializeWebsocketClient(_liveConsoleFeedUrl, _token, TimeSpan.Zero);
+                }
+            }
+
+            return delivered;
+        }
+
+        private void CloseWebSocket(WebSocketCloseStatus closeStatus, CancellationToken cancellationToken)
+        {
+            try
+            {
+                _websocketClient?.CloseOutputAsync(closeStatus, "Closing websocket", cancellationToken);
+            }
+            catch (Exception websocketEx)
+            {
+                // In some cases this might be okay since the websocket might be open yet, so just close and don't trace exceptions
+                Trace.Info($"Failed to close websocket gracefully {websocketEx.GetType().Name}");
+            }
+        }
     }
 }

src/Runner.Common/RunServer.cs

@@ -51,14 +51,8 @@ namespace GitHub.Runner.Common
         public Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken cancellationToken)
         {
             CheckConnection();
-            var jobMessage = RetryRequest<AgentJobRequestMessage>(
+            return RetryRequest<AgentJobRequestMessage>(
                 async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken);
-            if (jobMessage == null)
-            {
-                throw new TaskOrchestrationJobNotFoundException(id);
-            }
-
-            return jobMessage;
         }
 
         public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary<String, VariableValue> outputs, IList<StepResult> stepResults, CancellationToken cancellationToken)
@@ -71,14 +65,8 @@ namespace GitHub.Runner.Common
         public Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken)
         {
             CheckConnection();
-            var renewJobResponse = RetryRequest<RenewJobResponse>(
+            return RetryRequest<RenewJobResponse>(
                 async () => await _runServiceHttpClient.RenewJobAsync(requestUri, planId, jobId, cancellationToken), cancellationToken);
-            if (renewJobResponse == null)
-            {
-                throw new TaskOrchestrationJobNotFoundException(jobId.ToString());
-            }
-
-            return renewJobResponse;
         }
     }
 }

src/Runner.Sdk/Util/UrlUtil.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 
 namespace GitHub.Runner.Sdk
 {
@@ -15,6 +15,7 @@ namespace GitHub.Runner.Sdk
                 string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) ||
                 string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) ||
                 string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase) ||
+                gitHubUrl.Host.EndsWith(".ghe.localhost", StringComparison.OrdinalIgnoreCase) ||
                 gitHubUrl.Host.EndsWith(".ghe.com", StringComparison.OrdinalIgnoreCase);
         }
 

src/Runner.Sdk/Util/WhichUtil.cs

@@ -60,7 +60,7 @@ namespace GitHub.Runner.Sdk
                             trace?.Verbose(ex.ToString());
                         }
 
-                        if (matches != null && matches.Length > 0)
+                        if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
                         {
                             trace?.Info($"Location: '{matches.First()}'");
                             return matches.First();
@@ -86,7 +86,7 @@ namespace GitHub.Runner.Sdk
                             for (int i = 0; i < pathExtSegments.Length; i++)
                             {
                                 string fullPath = Path.Combine(pathSegment, $"{command}{pathExtSegments[i]}");
-                                if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)))
+                                if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)) && IsPathValid(fullPath, trace))
                                 {
                                     trace?.Info($"Location: '{fullPath}'");
                                     return fullPath;
@@ -105,7 +105,7 @@ namespace GitHub.Runner.Sdk
                         trace?.Verbose(ex.ToString());
                     }
 
-                    if (matches != null && matches.Length > 0)
+                    if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace))
                     {
                         trace?.Info($"Location: '{matches.First()}'");
                         return matches.First();
@@ -128,5 +128,15 @@ namespace GitHub.Runner.Sdk
 
             return null;
         }
+
+        // checks if the file is a symlink and if the symlink`s target exists.
+        private static bool IsPathValid(string path, ITraceWriter trace = null)
+        {
+            var fileInfo = new FileInfo(path);
+            var linkTargetFullPath = fileInfo.Directory?.FullName + Path.DirectorySeparatorChar + fileInfo.LinkTarget;
+            if(fileInfo.LinkTarget == null || File.Exists(linkTargetFullPath) || File.Exists(fileInfo.LinkTarget)) return true;
+            trace?.Info($"the target '{fileInfo.LinkTarget}' of the symbolic link '{path}', does not exist");
+            return false;
+        }
     }
 }

src/Runner.Worker/JobRunner.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -42,6 +43,13 @@ namespace GitHub.Runner.Worker
             DateTime jobStartTimeUtc = DateTime.UtcNow;
             IRunnerService server = null;
 
+            // add orchestration id to useragent for better correlation.
+            if (message.Variables.TryGetValue(Constants.Variables.System.OrchestrationId, out VariableValue orchestrationId) &&
+                !string.IsNullOrEmpty(orchestrationId.Value))
+            {
+                HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value));
+            }
+
             ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
             if (MessageUtil.IsRunServiceJob(message.MessageType))
             {
@@ -49,6 +57,9 @@ namespace GitHub.Runner.Worker
                 VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
                 await runServer.ConnectAsync(systemConnection.Url, jobServerCredential);
                 server = runServer;
+
+                _jobServerQueue = HostContext.GetService<IJobServerQueue>();
+                _jobServerQueue.Start(message, resultServiceOnly: true);
             }
             else
             {

src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs

@@ -1,13 +1,16 @@
-using System.Collections.Generic;
 using System.Runtime.Serialization;
-using GitHub.DistributedTask.WebApi;
 
 namespace GitHub.Actions.RunService.WebApi
 {
     [DataContract]
     public class AcquireJobRequest
     {
+        [DataMember(Name = "jobMessageId", EmitDefaultValue = false)]
+        public string JobMessageId { get; set; }
+
+        // This field will be removed in an upcoming Runner release.
+        // It's left here temporarily to facilitate the transition to the new field name, JobMessageId.
         [DataMember(Name = "streamId", EmitDefaultValue = false)]
-        public string StreamID { get; set; }
+        public string StreamId { get; set; }
     }
 }

src/Sdk/RSWebApi/RunServiceHttpClient.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
@@ -55,7 +56,7 @@ namespace GitHub.Actions.RunService.WebApi
         {
         }
 
-        public Task<AgentJobRequestMessage> GetJobMessageAsync(
+        public async Task<AgentJobRequestMessage> GetJobMessageAsync(
             Uri requestUri,
             string messageId,
             CancellationToken cancellationToken = default)
@@ -63,20 +64,34 @@ namespace GitHub.Actions.RunService.WebApi
             HttpMethod httpMethod = new HttpMethod("POST");
             var payload = new AcquireJobRequest
             {
-                StreamID = messageId
+                JobMessageId = messageId,
+                StreamId = messageId,
             };
 
             requestUri = new Uri(requestUri, "acquirejob");
 
             var requestContent = new ObjectContent<AcquireJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
-            return SendAsync<AgentJobRequestMessage>(
+            var result = await SendAsync<AgentJobRequestMessage>(
                 httpMethod,
                 requestUri: requestUri,
                 content: requestContent,
                 cancellationToken: cancellationToken);
+
+            if (result.IsSuccess)
+            {
+                return result.Value;
             }
 
-        public Task CompleteJobAsync(
+            switch (result.StatusCode)
+            {
+                case HttpStatusCode.NotFound:
+                    throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}");
+                default:
+                    throw new Exception($"Failed to get job message: {result.Error}");
+            }
+        }
+
+        public async Task CompleteJobAsync(
             Uri requestUri,
             Guid planId,
             Guid jobId,
@@ -98,14 +113,26 @@ namespace GitHub.Actions.RunService.WebApi
             requestUri = new Uri(requestUri, "completejob");
 
             var requestContent = new ObjectContent<CompleteJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
-            return SendAsync(
+            var response = await SendAsync(
                     httpMethod,
                     requestUri,
                     content: requestContent,
                     cancellationToken: cancellationToken);
+            if (response.IsSuccessStatusCode)
+            {
+                return;
             }
 
-        public Task<RenewJobResponse> RenewJobAsync(
+            switch (response.StatusCode)
+            {
+                case HttpStatusCode.NotFound:
+                    throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}");
+                default:
+                    throw new Exception($"Failed to complete job: {response.ReasonPhrase}");
+            }
+        }
+
+        public async Task<RenewJobResponse> RenewJobAsync(
             Uri requestUri,
             Guid planId,
             Guid jobId,
@@ -121,11 +148,24 @@ namespace GitHub.Actions.RunService.WebApi
             requestUri = new Uri(requestUri, "renewjob");
 
             var requestContent = new ObjectContent<RenewJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
-            return SendAsync<RenewJobResponse>(
+            var result = await SendAsync<RenewJobResponse>(
                 httpMethod,
                 requestUri,
                 content: requestContent,
                 cancellationToken: cancellationToken);
+
+            if (result.IsSuccess)
+            {
+                return result.Value;
+            }
+
+            switch (result.StatusCode)
+            {
+                case HttpStatusCode.NotFound:
+                    throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}");
+                default:
+                    throw new Exception($"Failed to renew job: {result.Error}");
+            }
         }
     }
 }

src/Sdk/WebApi/WebApi/BrokerHttpClient.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
@@ -55,7 +56,7 @@ namespace GitHub.Actions.RunService.WebApi
         {
         }
 
-        public Task<TaskAgentMessage> GetRunnerMessageAsync(
+        public async Task<TaskAgentMessage> GetRunnerMessageAsync(
             string runnerVersion,
             TaskAgentStatus? status,
             CancellationToken cancellationToken = default
@@ -74,11 +75,18 @@ namespace GitHub.Actions.RunService.WebApi
                 queryParams.Add("runnerVersion", runnerVersion);
             }
 
-            return SendAsync<TaskAgentMessage>(
+            var result = await SendAsync<TaskAgentMessage>(
                 new HttpMethod("GET"),
                 requestUri: requestUri,
                 queryParameters: queryParams,
                 cancellationToken: cancellationToken);
+
+            if (result.IsSuccess)
+            {
+                return result.Value;
+            }
+
+            throw new Exception($"Failed to get job message: {result.Error}");
         }
     }
 }

src/Sdk/WebApi/WebApi/Contracts.cs

@@ -157,6 +157,8 @@ namespace GitHub.Services.Results.Contracts
         public string StartedAt;
         [DataMember]
         public string CompletedAt;
+        [DataMember]
+        public Conclusion Conclusion;
     }
 
     public enum Status
@@ -167,6 +169,15 @@ namespace GitHub.Services.Results.Contracts
         StatusCompleted = 6
     }
 
+    public enum Conclusion
+    {
+        ConclusionUnknown = 0,
+        ConclusionSuccess = 2,
+        ConclusionFailure = 3,
+        ConclusionCancelled = 4,
+        ConclusionSkipped = 7,
+    }
+
     public static class BlobStorageTypes
     {
         public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE";

src/Sdk/WebApi/WebApi/RawHttpClientBase.cs

@@ -21,7 +21,7 @@ using Newtonsoft.Json.Linq;
 
 namespace Sdk.WebApi.WebApi
 {
-    public class RawHttpClientBase: IDisposable
+    public class RawHttpClientBase : IDisposable
     {
         protected RawHttpClientBase(
             Uri baseUrl,
@@ -101,7 +101,7 @@ namespace Sdk.WebApi.WebApi
             }
         }
 
-        protected Task<T> SendAsync<T>(
+        protected Task<RawHttpClientResult<T>> SendAsync<T>(
             HttpMethod method,
             Uri requestUri,
             HttpContent content = null,
@@ -112,7 +112,7 @@ namespace Sdk.WebApi.WebApi
             return SendAsync<T>(method, null, requestUri, content, queryParameters, userState, cancellationToken);
         }
 
-        protected async Task<T> SendAsync<T>(
+        protected async Task<RawHttpClientResult<T>> SendAsync<T>(
             HttpMethod method,
             IEnumerable<KeyValuePair<String, String>> additionalHeaders,
             Uri requestUri,
@@ -128,7 +128,7 @@ namespace Sdk.WebApi.WebApi
             }
         }
 
-        protected async Task<T> SendAsync<T>(
+        protected async Task<RawHttpClientResult<T>> SendAsync<T>(
             HttpRequestMessage message,
             Object userState = null,
             CancellationToken cancellationToken = default(CancellationToken))
@@ -138,7 +138,16 @@ namespace Sdk.WebApi.WebApi
             //from deadlocking...
             using (HttpResponseMessage response = await this.SendAsync(message, userState, cancellationToken).ConfigureAwait(false))
             {
-                return await ReadContentAsAsync<T>(response, cancellationToken).ConfigureAwait(false);
+                if (response.IsSuccessStatusCode)
+                {
+                    T data = await ReadContentAsAsync<T>(response, cancellationToken).ConfigureAwait(false);
+                    return RawHttpClientResult<T>.Ok(data);
+                }
+                else
+                {
+                    string errorMessage = $"Error: {response.ReasonPhrase}";
+                    return RawHttpClientResult<T>.Fail(errorMessage, response.StatusCode);
+                }
             }
         }
 

src/Sdk/WebApi/WebApi/RawHttpClientResult.cs

@@ -0,0 +1,33 @@
+using System.Net;
+
+namespace Sdk.WebApi.WebApi
+{
+    public class RawHttpClientResult
+    {
+        public bool IsSuccess { get; protected set; }
+        public string Error { get; protected set; }
+        public HttpStatusCode StatusCode { get; protected set; }
+        public bool IsFailure => !IsSuccess;
+
+        protected RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode)
+        {
+            IsSuccess = isSuccess;
+            Error = error;
+            StatusCode = statusCode;
+        }
+    }
+
+    public class RawHttpClientResult<T> : RawHttpClientResult
+    {
+        public T Value { get; private set; }
+
+        protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode)
+            : base(isSuccess, error, statusCode)
+        {
+            Value = value;
+        }
+
+        public static RawHttpClientResult<T> Fail(string message, HttpStatusCode statusCode) => new RawHttpClientResult<T>(default(T), false, message, statusCode);
+        public static RawHttpClientResult<T> Ok(T value) => new RawHttpClientResult<T>(value, true, string.Empty, HttpStatusCode.OK);
+    }
+}

src/Sdk/WebApi/WebApi/ResultsHttpClient.cs

@@ -329,7 +329,8 @@ namespace GitHub.Services.Results.Client
                 Name = r.Name,
                 Status = ConvertStateToStatus(r.State.GetValueOrDefault()),
                 StartedAt = r.StartTime?.ToString(Constants.TimestampFormat),
-                CompletedAt = r.FinishTime?.ToString(Constants.TimestampFormat)
+                CompletedAt = r.FinishTime?.ToString(Constants.TimestampFormat),
+                Conclusion = ConvertResultToConclusion(r.Result)
             };
         }
 
@@ -348,6 +349,29 @@ namespace GitHub.Services.Results.Client
             }
         }
 
+        private Conclusion ConvertResultToConclusion(TaskResult? r)
+        {
+            if (!r.HasValue)
+            {
+                return Conclusion.ConclusionUnknown;
+            }
+
+            switch (r)
+            {
+                case TaskResult.Succeeded:
+                case TaskResult.SucceededWithIssues:
+                    return Conclusion.ConclusionSuccess;
+                case TaskResult.Canceled:
+                    return Conclusion.ConclusionCancelled;
+                case TaskResult.Skipped:
+                    return Conclusion.ConclusionSkipped;
+                case TaskResult.Failed:
+                    return Conclusion.ConclusionFailure;
+                default:
+                    return Conclusion.ConclusionUnknown;
+            }
+        }
+
         public async Task UpdateWorkflowStepsAsync(Guid planId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken)
         {
             var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);

src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs

@@ -0,0 +1,63 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.Serialization.Json;
+using System.Text;
+using Xunit;
+
+namespace GitHub.Actions.RunService.WebApi.Tests;
+
+public sealed class AcquireJobRequestL0
+{
+
+    [Fact]
+    [Trait("Level", "L0")]
+    [Trait("Category", "Common")]
+    public void VerifySerialization()
+    {
+        var jobMessageId = "1526919030369-33";
+        var request = new AcquireJobRequest
+        {
+            JobMessageId = jobMessageId,
+            StreamId = jobMessageId
+        };
+        var serializer = new DataContractJsonSerializer(typeof(AcquireJobRequest));
+        using var stream = new MemoryStream();
+        serializer.WriteObject(stream, request);
+
+        stream.Position = 0;
+        using var reader = new StreamReader(stream, Encoding.UTF8);
+        string json = reader.ReadToEnd();
+        string expected = DoubleQuotify(string.Format("{{'jobMessageId':'{0}','streamId':'{0}'}}", request.JobMessageId));
+        Assert.Equal(expected, json);
+
+    }
+
+    [Fact]
+    [Trait("Level", "L0")]
+    [Trait("Category", "Common")]
+    public void VerifyDeserialization()
+    {
+        var serializer = new DataContractJsonSerializer(typeof(AcquireJobRequest));
+        var variations = new Dictionary<string, string>()
+        {
+            ["{'streamId': 'legacy', 'jobMessageId': 'new-1'}"] = "new-1",
+            ["{'jobMessageId': 'new-2', 'streamId': 'legacy'}"] = "new-2",
+            ["{'jobMessageId': 'new-3'}"] = "new-3"
+        };
+
+        foreach (var (source, expected) in variations)
+        {
+            using var stream = new MemoryStream();
+            stream.Write(Encoding.UTF8.GetBytes(DoubleQuotify(source)));
+            stream.Position = 0;
+            var recoveredRecord = serializer.ReadObject(stream) as AcquireJobRequest;
+            Assert.NotNull(recoveredRecord);
+            Assert.Equal(expected, recoveredRecord.JobMessageId);
+        }
+    }
+
+    private static string DoubleQuotify(string text)
+    {
+        return text.Replace('\'', '"');
+    }
+}

src/Test/L0/Util/WhichUtilL0.cs

@@ -1,4 +1,4 @@
-using GitHub.Runner.Common.Util;
+using GitHub.Runner.Common.Util;
 using GitHub.Runner.Sdk;
 using System;
 using System.IO;
@@ -89,5 +89,128 @@ namespace GitHub.Runner.Common.Tests.Util
                 Assert.Equal(gitPath, gitPath2);
             }
         }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void WhichHandlesSymlinkToTargetFullPath()
+        {
+            // Arrange
+            using TestHostContext hc = new TestHostContext(this);
+            Tracing trace = hc.GetTrace();
+            string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
+#if OS_WINDOWS
+            string newValue = oldValue + @$";{Path.GetTempPath()}";
+            string symlinkName = $"symlink-{Guid.NewGuid()}";
+            string symlink = Path.GetTempPath() + $"{symlinkName}.exe";
+            string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}.exe";
+#else
+            string newValue = oldValue + @$":{Path.GetTempPath()}";
+            string symlinkName = $"symlink-{Guid.NewGuid()}";
+            string symlink = Path.GetTempPath() + $"{symlinkName}";
+            string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}";
+#endif
+
+            Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue);
+
+
+            using (File.Create(target))
+            {
+                File.CreateSymbolicLink(symlink, target);
+
+                // Act.
+                var result = WhichUtil.Which(symlinkName, require: true, trace: trace);
+
+                // Assert
+                Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find symlink through: {nameof(WhichUtil.Which)}");
+
+            }
+
+
+            // Cleanup
+            File.Delete(symlink);
+            File.Delete(target);
+            Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue);
+
+        }
+
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void WhichHandlesSymlinkToTargetRelativePath()
+        {
+            // Arrange
+            using TestHostContext hc = new TestHostContext(this);
+            Tracing trace = hc.GetTrace();
+            string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
+#if OS_WINDOWS
+            string newValue = oldValue + @$";{Path.GetTempPath()}";
+            string symlinkName = $"symlink-{Guid.NewGuid()}";
+            string symlink = Path.GetTempPath() + $"{symlinkName}.exe";
+            string targetName = $"target-{Guid.NewGuid()}.exe";
+            string target = Path.GetTempPath() + targetName;
+#else
+            string newValue = oldValue + @$":{Path.GetTempPath()}";
+            string symlinkName = $"symlink-{Guid.NewGuid()}";
+            string symlink = Path.GetTempPath() + $"{symlinkName}";
+            string targetName = $"target-{Guid.NewGuid()}";
+            string target = Path.GetTempPath() + targetName;
+#endif
+            Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue);
+
+
+            using (File.Create(target))
+            {
+                File.CreateSymbolicLink(symlink, targetName);
+
+                // Act.
+                var result = WhichUtil.Which(symlinkName, require: true, trace: trace);
+
+                // Assert
+                Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find {symlinkName} through: {nameof(WhichUtil.Which)}");
+            }
+
+            // Cleanup
+            File.Delete(symlink);
+            File.Delete(target);
+            Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue);
+
+        }
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Common")]
+        public void WhichThrowsWhenSymlinkBroken()
+        {
+            // Arrange
+            using TestHostContext hc = new TestHostContext(this);
+            Tracing trace = hc.GetTrace();
+            string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
+
+#if OS_WINDOWS
+            string newValue = oldValue + @$";{Path.GetTempPath()}";
+            string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}";
+            string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}.exe";
+#else
+            string newValue = oldValue + @$":{Path.GetTempPath()}";
+            string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}";
+            string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}";
+#endif
+
+
+            string target = "no-such-file-cf7e351f";
+            Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue);
+
+            File.CreateSymbolicLink(brokenSymlink, target);
+
+            // Act.
+            var exception = Assert.Throws<FileNotFoundException>(()=>WhichUtil.Which(brokenSymlinkName, require: true, trace: trace));
+
+            // Assert
+            Assert.Equal(brokenSymlinkName, exception.FileName);
+
+            // Cleanup
+            File.Delete(brokenSymlink);
+            Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue);
+        }
     }
 }

src/runnerversion

@@ -1 +1 @@
-2.303.0
+2.304.0