using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Services.Common.Diagnostics;
using GitHub.Services.Common.Internal;
namespace GitHub.Services.Common
{
///
/// Handles automatic replay of HTTP requests when errors are encountered based on a configurable set of options.
///
[EditorBrowsable(EditorBrowsableState.Never)]
public class VssHttpRetryMessageHandler : DelegatingHandler
{
public VssHttpRetryMessageHandler(Int32 maxRetries)
: this(new VssHttpRetryOptions { MaxRetries = maxRetries })
{
}
public VssHttpRetryMessageHandler(Int32 maxRetries, string clientName)
: this(new VssHttpRetryOptions { MaxRetries = maxRetries })
{
m_clientName = clientName;
}
public VssHttpRetryMessageHandler(VssHttpRetryOptions options)
{
m_retryOptions = options;
}
public VssHttpRetryMessageHandler(
VssHttpRetryOptions options,
HttpMessageHandler innerHandler)
: base(innerHandler)
{
m_retryOptions = options;
}
protected override async Task SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
Int32 attempt = 1;
HttpResponseMessage response = null;
HttpRequestException exception = null;
VssTraceActivity traceActivity = VssTraceActivity.Current;
// Allow overriding default retry options per request
VssHttpRetryOptions retryOptions = m_retryOptions;
object retryOptionsObject;
if (request.Options.TryGetValue(HttpRetryOptionsKey, out retryOptionsObject)) // NETSTANDARD compliant, TryGetValue is not
{
// Fallback to default options if object of unexpected type was passed
retryOptions = retryOptionsObject as VssHttpRetryOptions ?? m_retryOptions;
}
TimeSpan minBackoff = retryOptions.MinBackoff;
Int32 maxAttempts = retryOptions.MaxRetries + 1;
IVssHttpRetryInfo retryInfo = null;
object retryInfoObject;
if (request.Options.TryGetValue(HttpRetryInfoKey, out retryInfoObject)) // NETSTANDARD compliant, TryGetValue is not
{
retryInfo = retryInfoObject as IVssHttpRetryInfo;
}
if (IsLowPriority(request))
{
// Increase the backoff and retry count, low priority requests can be retried many times if the server is busy.
minBackoff = TimeSpan.FromSeconds(minBackoff.TotalSeconds * 2);
maxAttempts = maxAttempts * 10;
}
TimeSpan backoff = minBackoff;
while (attempt <= maxAttempts)
{
// Reset the exception so we don't have a lingering variable
exception = null;
Boolean canRetry = false;
SocketError? socketError = null;
HttpStatusCode? statusCode = null;
WebExceptionStatus? webExceptionStatus = null;
WinHttpErrorCode? winHttpErrorCode = null;
CurlErrorCode? curlErrorCode = null;
string afdRefInfo = null;
try
{
if (attempt == 1)
{
retryInfo?.InitialAttempt(request);
}
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (attempt > 1)
{
TraceHttpRequestSucceededWithRetry(traceActivity, response, attempt);
}
// Verify the response is successful or the status code is one that may be retried.
if (response.IsSuccessStatusCode)
{
break;
}
else
{
statusCode = response.StatusCode;
afdRefInfo = response.Headers.TryGetValues(HttpHeaders.AfdResponseRef, out var headers) ? headers.First() : null;
canRetry = m_retryOptions.IsRetryableResponse(response);
}
}
catch (HttpRequestException ex)
{
exception = ex;
canRetry = VssNetworkHelper.IsTransientNetworkException(exception, m_retryOptions, out statusCode, out webExceptionStatus, out socketError, out winHttpErrorCode, out curlErrorCode);
}
catch (TimeoutException)
{
throw;
}
if (attempt < maxAttempts && canRetry)
{
backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, minBackoff, m_retryOptions.MaxBackoff, m_retryOptions.BackoffCoefficient);
retryInfo?.Retry(backoff);
TraceHttpRequestRetrying(traceActivity, request, attempt, backoff, statusCode, webExceptionStatus, socketError, winHttpErrorCode, curlErrorCode, afdRefInfo);
}
else
{
if (attempt < maxAttempts)
{
if (exception == null)
{
TraceHttpRequestFailed(traceActivity, request, statusCode != null ? statusCode.Value : (HttpStatusCode)0, afdRefInfo);
}
else
{
TraceHttpRequestFailed(traceActivity, request, exception);
}
}
else
{
TraceHttpRequestFailedMaxAttempts(traceActivity, request, attempt, statusCode, webExceptionStatus, socketError, winHttpErrorCode, curlErrorCode, afdRefInfo);
}
break;
}
// Make sure to dispose of this so we don't keep the connection open
if (response != null)
{
response.Dispose();
}
attempt++;
TraceRaw(request, 100011, TraceLevel.Error,
"{{ \"Client\":\"{0}\", \"Endpoint\":\"{1}\", \"Attempt\":{2}, \"MaxAttempts\":{3}, \"Backoff\":{4} }}",
m_clientName,
request.RequestUri.Host,
attempt,
maxAttempts,
backoff.TotalMilliseconds);
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
if (exception != null)
{
throw exception;
}
return response;
}
protected virtual void TraceRaw(HttpRequestMessage request, int tracepoint, TraceLevel level, string message, params object[] args)
{
// implement in Server so retries are recorded in ProductTrace
}
protected virtual void TraceHttpRequestFailed(VssTraceActivity activity, HttpRequestMessage request, HttpStatusCode statusCode, string afdRefInfo)
{
VssHttpEventSource.Log.HttpRequestFailed(activity, request, statusCode, afdRefInfo);
}
protected virtual void TraceHttpRequestFailed(VssTraceActivity activity, HttpRequestMessage request, Exception exception)
{
VssHttpEventSource.Log.HttpRequestFailed(activity, request, exception);
}
protected virtual void TraceHttpRequestFailedMaxAttempts(VssTraceActivity activity, HttpRequestMessage request, Int32 attempt, HttpStatusCode? httpStatusCode, WebExceptionStatus? webExceptionStatus, SocketError? socketErrorCode, WinHttpErrorCode? winHttpErrorCode, CurlErrorCode? curlErrorCode, string afdRefInfo)
{
VssHttpEventSource.Log.HttpRequestFailedMaxAttempts(activity, request, attempt, httpStatusCode, webExceptionStatus, socketErrorCode, winHttpErrorCode, curlErrorCode, afdRefInfo);
}
protected virtual void TraceHttpRequestSucceededWithRetry(VssTraceActivity activity, HttpResponseMessage response, Int32 attempt)
{
VssHttpEventSource.Log.HttpRequestSucceededWithRetry(activity, response, attempt);
}
protected virtual void TraceHttpRequestRetrying(VssTraceActivity activity, HttpRequestMessage request, Int32 attempt, TimeSpan backoffDuration, HttpStatusCode? httpStatusCode, WebExceptionStatus? webExceptionStatus, SocketError? socketErrorCode, WinHttpErrorCode? winHttpErrorCode, CurlErrorCode? curlErrorCode, string afdRefInfo)
{
VssHttpEventSource.Log.HttpRequestRetrying(activity, request, attempt, backoffDuration, httpStatusCode, webExceptionStatus, socketErrorCode, winHttpErrorCode, curlErrorCode, afdRefInfo);
}
private static bool IsLowPriority(HttpRequestMessage request)
{
bool isLowPriority = false;
IEnumerable headers;
if (request.Headers.TryGetValues(HttpHeaders.VssRequestPriority, out headers) && headers != null)
{
string header = headers.FirstOrDefault();
isLowPriority = string.Equals(header, "Low", StringComparison.OrdinalIgnoreCase);
}
return isLowPriority;
}
private VssHttpRetryOptions m_retryOptions;
public const string HttpRetryInfoKey = "HttpRetryInfo";
public const string HttpRetryOptionsKey = "VssHttpRetryOptions";
private string m_clientName = "";
}
}