diff --git a/src/Runner.Common/BrokerServer.cs b/src/Runner.Common/BrokerServer.cs index b2b20f089..5633166c6 100644 --- a/src/Runner.Common/BrokerServer.cs +++ b/src/Runner.Common/BrokerServer.cs @@ -22,6 +22,8 @@ namespace GitHub.Runner.Common Task GetRunnerMessageAsync(Guid? sessionId, TaskAgentStatus status, string version, string os, string architecture, bool disableUpdate, CancellationToken token); + Task DeleteRunnerMessageAsync(Guid sessionId, string jobMessageKey, CancellationToken cancellationToken); + Task UpdateConnectionIfNeeded(Uri serverUri, VssCredentials credentials); Task ForceRefreshConnection(VssCredentials credentials); diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index c042796b1..72821276d 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -62,7 +62,9 @@ namespace GitHub.Runner.Common CheckConnection(); return RetryRequest( async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, cancellationToken), cancellationToken, - shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException); + shouldRetry: ex => ex is not TaskOrchestrationJobNotFoundException && + ex is not TaskOrchestrationJobAlreadyAcquiredException && + ex is not TaskOrchestrationJobUnprocessableException); } public Task CompleteJobAsync( diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index d610db74a..e5a8a1138 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -544,6 +544,8 @@ namespace GitHub.Runner.Listener } else { + skipMessageDeletion = true; + var messageRef = StringUtil.ConvertFromJson(message.Body); Pipelines.AgentJobRequestMessage jobRequestMessage = null; @@ -565,9 +567,13 @@ namespace GitHub.Runner.Listener { jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token); } - catch (TaskOrchestrationJobAlreadyAcquiredException) + catch (Exception ex) when ( + ex is TaskOrchestrationJobNotFoundException || + ex is TaskOrchestrationJobAlreadyAcquiredException || + ex is TaskOrchestrationJobUnprocessableException) { - Trace.Info("Job is already acquired, skip this message."); + Trace.Error($"Skipping job: {ex.Message}"); + skipMessageDeletion = false; continue; } catch (Exception ex) diff --git a/src/Sdk/DTWebApi/WebApi/Exceptions.cs b/src/Sdk/DTWebApi/WebApi/Exceptions.cs index 536bf7550..ee47f1370 100644 --- a/src/Sdk/DTWebApi/WebApi/Exceptions.cs +++ b/src/Sdk/DTWebApi/WebApi/Exceptions.cs @@ -1539,6 +1539,26 @@ namespace GitHub.DistributedTask.WebApi } } + [Serializable] + [ExceptionMapping("0.0", "3.0", "TaskOrchestrationJobUnprocessableException", "GitHub.DistributedTask.WebApi.TaskOrchestrationJobUnprocessableException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")] + public sealed class TaskOrchestrationJobUnprocessableException : DistributedTaskException + { + public TaskOrchestrationJobUnprocessableException(String message) + : base(message) + { + } + + public TaskOrchestrationJobUnprocessableException(String message, Exception innerException) + : base(message, innerException) + { + } + + private TaskOrchestrationJobUnprocessableException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } + [Serializable] [ExceptionMapping("0.0", "3.0", "TaskOrchestrationPlanSecurityException", "GitHub.DistributedTask.WebApi.TaskOrchestrationPlanSecurityException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")] public sealed class TaskOrchestrationPlanSecurityException : DistributedTaskException diff --git a/src/Sdk/RSWebApi/Contracts/RunServiceError.cs b/src/Sdk/RSWebApi/Contracts/RunServiceError.cs new file mode 100644 index 000000000..0276acf5d --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/RunServiceError.cs @@ -0,0 +1,17 @@ +using System.Runtime.Serialization; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class RunServiceError + { + [DataMember(Name = "source", EmitDefaultValue = false)] + public string Source { get; set; } + + [DataMember(Name = "statusCode", EmitDefaultValue = false)] + public int StatusCode { get; set; } + + [DataMember(Name = "errorMessage", EmitDefaultValue = false)] + public string ErrorMessage { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index ba176ccf6..6de4333b3 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -86,6 +86,7 @@ namespace GitHub.Actions.RunService.WebApi httpMethod, requestUri: requestUri, content: requestContent, + readErrorContent: true, cancellationToken: cancellationToken); if (result.IsSuccess) @@ -93,14 +94,35 @@ namespace GitHub.Actions.RunService.WebApi return result.Value; } + if (TryParseErrorContent(result.ErrorContent, out RunServiceError error)) + { + switch ((HttpStatusCode)error.StatusCode) + { + case HttpStatusCode.NotFound: + throw new TaskOrchestrationJobNotFoundException($"Job message not found '{messageId}'. {error.ErrorMessage}"); + case HttpStatusCode.Conflict: + throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired '{messageId}'. {error.ErrorMessage}"); + case HttpStatusCode.UnprocessableEntity: + throw new TaskOrchestrationJobUnprocessableException($"Unprocessable job '{messageId}'. {error.ErrorMessage}"); + } + } + + // Temporary back compat switch (result.StatusCode) { case HttpStatusCode.NotFound: throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}"); case HttpStatusCode.Conflict: throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired: {messageId}"); - default: - throw new Exception($"Failed to get job message: {result.Error}"); + } + + if (!string.IsNullOrEmpty(result.ErrorContent)) + { + throw new Exception($"Failed to get job message: {result.Error}. {result.ErrorContent}"); + } + else + { + throw new Exception($"Failed to get job message: {result.Error}"); } } @@ -190,5 +212,26 @@ namespace GitHub.Actions.RunService.WebApi var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); return JsonConvert.DeserializeObject(json, s_serializerSettings); } + + private static bool TryParseErrorContent(string errorContent, out RunServiceError error) + { + if (!string.IsNullOrEmpty(errorContent)) + { + try + { + error = JsonUtility.FromString(errorContent); + if (error?.Source == "actions-run-service") + { + return true; + } + } + catch (Exception) + { + } + } + + error = null; + return false; + } } } diff --git a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs index 2a6ecc7eb..8f4e810d1 100644 --- a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs @@ -122,7 +122,7 @@ namespace GitHub.Actions.RunService.WebApi throw new Exception($"Failed to get job message: {result.Error}"); } - public async Task GetRunnerMessageAsync( + public async Task DeleteRunnerMessageAsync( Guid? sessionId, string jobMessageKey, CancellationToken cancellationToken = default) @@ -141,7 +141,7 @@ namespace GitHub.Actions.RunService.WebApi queryParams.Add("jobMessageKey", jobMessageKey); } - var result = await SendAsync( + var result = await SendAsync( new HttpMethod("DELETE"), requestUri: requestUri, queryParameters: queryParams, @@ -149,7 +149,7 @@ namespace GitHub.Actions.RunService.WebApi if (result.IsSuccess) { - return result.Value; + return; } throw new Exception($"Failed to get job message: StatusCode={result.StatusCode} Error={result.Error}"); diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs index de7c3bcb3..3e9407414 100644 --- a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs +++ b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs @@ -106,10 +106,11 @@ namespace Sdk.WebApi.WebApi Uri requestUri, HttpContent content = null, IEnumerable> queryParameters = null, + Boolean readErrorContent = false, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) { - return SendAsync(method, null, requestUri, content, queryParameters, userState, cancellationToken); + return SendAsync(method, null, requestUri, content, queryParameters, readErrorContent, userState, cancellationToken); } protected async Task> SendAsync( @@ -118,18 +119,20 @@ namespace Sdk.WebApi.WebApi Uri requestUri, HttpContent content = null, IEnumerable> queryParameters = null, + Boolean readErrorContent = false, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) { using (VssTraceActivity.GetOrCreate().EnterCorrelationScope()) using (HttpRequestMessage requestMessage = CreateRequestMessage(method, additionalHeaders, requestUri, content, queryParameters)) { - return await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false); + return await SendAsync(requestMessage, readErrorContent, userState, cancellationToken).ConfigureAwait(false); } } protected async Task> SendAsync( HttpRequestMessage message, + Boolean readErrorContent = false, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) { @@ -145,8 +148,14 @@ namespace Sdk.WebApi.WebApi } else { + var errorContent = default(string); + if (readErrorContent) + { + errorContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + string errorMessage = $"Error: {response.ReasonPhrase}"; - return RawHttpClientResult.Fail(errorMessage, response.StatusCode); + return RawHttpClientResult.Fail(errorMessage, response.StatusCode, errorContent); } } } diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs b/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs index 1b2dc5f06..1a6cc6075 100644 --- a/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs +++ b/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs @@ -5,15 +5,27 @@ namespace Sdk.WebApi.WebApi public class RawHttpClientResult { public bool IsSuccess { get; protected set; } + + /// + /// A description of the HTTP status code, like "Error: Unprocessable Entity" + /// public string Error { get; protected set; } + + /// + /// The raw of the HTTP response, for unsuccessful HTTP status codes + /// + public string ErrorContent { get; protected set; } + public HttpStatusCode StatusCode { get; protected set; } + public bool IsFailure => !IsSuccess; - protected RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode) + protected RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode, string errorContent = null) { IsSuccess = isSuccess; Error = error; StatusCode = statusCode; + ErrorContent = errorContent; } } @@ -21,13 +33,13 @@ namespace Sdk.WebApi.WebApi { public T Value { get; private set; } - protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode) - : base(isSuccess, error, statusCode) + protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode, string errorContent) + : base(isSuccess, error, statusCode, errorContent) { Value = value; } - public static RawHttpClientResult Fail(string message, HttpStatusCode statusCode) => new RawHttpClientResult(default(T), false, message, statusCode); - public static RawHttpClientResult Ok(T value) => new RawHttpClientResult(value, true, string.Empty, HttpStatusCode.OK); + public static RawHttpClientResult Fail(string message, HttpStatusCode statusCode, string errorContent) => new RawHttpClientResult(default(T), false, message, statusCode, errorContent); + public static RawHttpClientResult Ok(T value) => new RawHttpClientResult(value, true, string.Empty, HttpStatusCode.OK, null); } }